The class ibis::part represents a partition of a relational table. More...

#include <part.h>

Inheritance diagram for ibis::part:

Classes
class	barrel
	To read a list of variables at the same time. More...
class	cleaner
	A cleaner to be used by the fileManager::unload function. More...
struct	indexBuilderPool
	A struct to pack arguments to the function ibis_part_build_index. More...
struct	info
	A simple class to describe an ibis::part object. More...
class	mutexLock
	Provide a mutual exclusion lock on an ibis::part object. More...
class	readLock
	Provide a read lock on an ibis::part. More...
class	softWriteLock
	An non-blocking version of writeLock. More...
struct	thrArg
	A struct to pack the arguments to function startTests. More...
class	vault
	To read variables in certain order. More...
class	writeLock
	Provide a write lock on an ibis::part. More...
Public Types
typedef std::map< const char , column , lessi >	columnList
	An associative array for columns of data.
enum	TABLE_STATE { UNKNOWN_STATE = 0, STABLE_STATE, RECEIVING_STATE, PRETRANSITION_STATE, TRANSITION_STATE, POSTTRANSITION_STATE }
Public Member Functions
ibis::fileManager::ACCESS_PREFERENCE	accessHint (const ibis::bitvector &mask, unsigned elemsize=4) const
	Evaluate the strategy for accessing a data file.
long	addColumn (const char aexpr, const char cname, ibis::TYPE_T ctype=ibis::DOUBLE)
	Add a column computed with the given arithmetic expression.
long	addColumn (const ibis::math::term xpr, ibis::bitvector &mask, const char cname, ibis::TYPE_T ctype=ibis::DOUBLE)
	Add a column computed with the given arithmetic expression.
long	append (const char *dir)
	Append data from `dir`.
virtual int	buildIndexes (const char *iopt=0, int nthr=1)
	Make sure indexes for all columns are available.
void	buildSorted (const char *colname) const
	Build a sorted version of the specified column.
long	calculate (const ibis::math::term &, const ibis::bitvector &, array_t< double > &) const
	Calculate the values of an arithmetic expression.
ibis::table::stringList	columnNames () const
	Return column names in a list.
ibis::table::typeList	columnTypes () const
	Return column types in a list.
void	combineNames (ibis::table::namesTypes &metalist) const
	Update the list of columns with information in this data partition.
long	commit (const char *dir)
	Commit the append operation involving data from `dir`.
void	computeMinMax ()
	Compute the min and max for each column.
long	countHits (const ibis::qRange &cmp) const
	Count the number of hits for a single range condition.
const char *	currentDataDir () const
	Return the name of the active data directory.
long	deactivate (const std::vector< uint32_t > &rows)
	Mark the specified rows as inactive.
long	deactivate (const char *conds)
	Mark all rows satisfying the specified conditions as inactive.
const char *	description () const
	Return a text description of the partition.
void	doBackup ()
	A function to start backing up the active dir.
virtual long	doScan (const ibis::qRange &cmp, ibis::bitvector &hits) const
	Evaluate the range condition.
virtual long	doScan (const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits) const
	Evalute the range condition on the records that are marked 1 in the mask.
virtual long	doScan (const ibis::qRange &cmp, const ibis::bitvector &mask, void *res) const
	Evalute the range condition and record the values satisfying the condition in res.
virtual long	doScan (const ibis::qRange &cmp, const ibis::bitvector &mask, void *res, ibis::bitvector &hits) const
	Evalute the range condition and record the values satisfying the condition in res.
virtual long	doScan (const ibis::math::term &, const ibis::bitvector &, ibis::bitvector &) const
	Treat the arithmetic expression as true or false.
virtual long	doScan (const ibis::compRange &cmp, ibis::bitvector &hits) const
	Sequential scan without a mask.
virtual long	doScan (const ibis::compRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits) const
	Locate the records that have mark value 1 and satisfy the complex range conditions.
template<typename T >
long	doScan (const array_t< T > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, ibis::bitvector &hits)
	Evalue the range condition on the in memory values.
template<>
long	doScan (const array_t< float > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, ibis::bitvector &hits)
	Examine the range condition with in memory values.
template<>
long	doScan (const array_t< double > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, ibis::bitvector &hits)
	Examine the range condition with in memory values.
template<typename T >
long	doScan (const array_t< T > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, array_t< T > &res)
	Evalue the range condition on the in memory values.
template<typename T >
long	doScan (const array_t< T > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, array_t< T > &res, ibis::bitvector &hits)
	Evalue the range condition on the in memory values.
template<>
long	doScan (const array_t< float > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, array_t< float > &res)
	Examine the range condition with in memory values.
template<>
long	doScan (const array_t< double > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, array_t< double > &res)
	Examine the range condition with in memory values.
template<>
long	doScan (const array_t< float > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, array_t< float > &res, ibis::bitvector &hits)
	Examine the range condition with in memory values.
template<>
long	doScan (const array_t< double > &vals, const ibis::qContinuousRange &rng, const ibis::bitvector &mask, array_t< double > &res, ibis::bitvector &hits)
	Examine the range condition with in memory values.
void	emptyCache () const
	Empty all unused resources in cache.
template<>
void	equalWeightBins (const array_t< float > &vals, uint32_t nbins, array_t< float > &bounds)
	Explicit specialization for float arrays.
template<>
void	equalWeightBins (const array_t< double > &vals, uint32_t nbins, array_t< double > &bounds)
	Explicit specialization for double arrays.
virtual double	estimateCost (const ibis::qContinuousRange &cmp) const
	Estimate the cost of evaluate the query expression.
virtual double	estimateCost (const ibis::qDiscreteRange &cmp) const
virtual double	estimateCost (const ibis::qIntHod &cmp) const
virtual double	estimateCost (const ibis::qUIntHod &cmp) const
virtual double	estimateCost (const ibis::qString &cmp) const
virtual double	estimateCost (const ibis::qMultiString &cmp) const
virtual long	estimateMatchAny (const ibis::qAnyAny &cmp, ibis::bitvector &low, ibis::bitvector &high) const
	Estimate a lower bound and an upper bound on the records that are hits.
virtual long	estimateRange (const ibis::qContinuousRange &cmp) const
	Return an upper bound on the number of hits.
virtual long	estimateRange (const ibis::qDiscreteRange &cmp) const
	Return an upper bound on the number of hits.
virtual long	estimateRange (const ibis::qIntHod &cmp) const
	Return an upper bound on the number of hits.
virtual long	estimateRange (const ibis::qUIntHod &cmp) const
	Return an upper bound on the number of hits.
virtual long	estimateRange (const ibis::qContinuousRange &cmp, ibis::bitvector &low, ibis::bitvector &high) const
	Estimate a continuous range condition.
virtual long	estimateRange (const ibis::qDiscreteRange &cmp, ibis::bitvector &low, ibis::bitvector &high) const
	Estimate the discrete range condition.
virtual long	estimateRange (const ibis::qIntHod &cmp, ibis::bitvector &low, ibis::bitvector &high) const
	Estimate the discrete range condition.
virtual long	estimateRange (const ibis::qUIntHod &cmp, ibis::bitvector &low, ibis::bitvector &high) const
	Estimate the discrete range condition.
int64_t	evaluateJoin (const ibis::deprecatedJoin &cmp, const ibis::bitvector &mask, ibis::bitvector64 &pairs) const
	Evaluate a self-join.
int64_t	evaluateJoin (const ibis::deprecatedJoin &cmp, const ibis::bitvector &mask, const char *pairfile) const
	Return the number of pairs satisfying the join condition.
int64_t	evaluateJoin (const ibis::deprecatedJoin &cmp, const ibis::bitvector &mask) const
	Return only the number of pairs satisfying the join condition.
int64_t	evaluateJoin (const std::vector< const ibis::deprecatedJoin * > &cmp, const ibis::bitvector &mask, ibis::bitvector64 &pairs) const
	Evaluate a join defined with multiple (conjunctive) range join conditions.
int64_t	evaluateJoin (const std::vector< const ibis::deprecatedJoin * > &cmp, const ibis::bitvector &mask) const
int64_t	evaluateJoin (const ibis::deprecatedJoin &cmp, const ibis::bitvector64 &trial, ibis::bitvector64 &result) const
	Evaluate all pairs in `trial` to determine whether they really satisfy the range join defined in `cmp`.
int64_t	evaluateJoin (const std::vector< const ibis::deprecatedJoin * > &cmp, const ibis::bitvector64 &trial, ibis::bitvector64 &result) const
	Check a set of pairs defined in `trial`.
virtual long	evaluateRange (const ibis::qContinuousRange &cmp, const ibis::bitvector &mask, ibis::bitvector &res) const
	Evaluate a continue range expression accurately.
virtual long	evaluateRange (const ibis::qDiscreteRange &cmp, const ibis::bitvector &mask, ibis::bitvector &res) const
	Evaluate a discrete range expression accurately.
virtual long	evaluateRange (const ibis::qIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &res) const
	Evaluate a discrete range expression accurately.
virtual long	evaluateRange (const ibis::qUIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &res) const
	Evaluate a discrete range expression accurately.
long	evaluateRIDSet (const ibis::RIDSet &, ibis::bitvector &) const
	Convert a list of RIDs into a bitvector.
double	getActualMax (const char *name) const
	The actual maximum value in the named column.
double	getActualMin (const char *name) const
	The actual minimum value in the named column.
column *	getColumn (const char *name) const
	Given a name, return the associated column.
column *	getColumn (uint32_t ind) const
	Returns the pointer to the ith column.
double	getColumnSum (const char *name) const
	Sum of all value in the named column.
info *	getInfo () const
	Return an ibis::part::info object that describes the current partition.
const std::vector< std::string > &	getMeshDimensions () const
	Return the name of the dimensions corresponding to the vector returned from getMeshShape.
const std::vector< uint32_t > &	getMeshShape () const
	In many scientific applications, data are defined on meshes.
const char *	getMetaTag (const char *) const
	Return the value of the meta tag with the specified name.
const ibis::bitvector &	getNullMask () const
	Return a reference to the mask of active rows.
array_t< rid_t > *	getRIDs () const
array_t< rid_t > *	getRIDs (const ibis::bitvector &mask) const
	Retrieve the RIDs corresponding to mask[i] == 1.
uint32_t	getRowNumber (const rid_t &rid) const
	Return the row number of the row with specified RID.
TABLE_STATE	getState () const
	Retrieve the current state of data partition.
TABLE_STATE	getStateNoLocking () const
	Return the current state of data partition.
virtual float	getUndecidable (const ibis::qContinuousRange &cmp, ibis::bitvector &iffy) const
	Discover the records that can not be decided using the index.
virtual float	getUndecidable (const ibis::qDiscreteRange &cmp, ibis::bitvector &iffy) const
	Discover the records that can not be decided using the index.
virtual float	getUndecidable (const ibis::qIntHod &cmp, ibis::bitvector &iffy) const
	Discover the records that can not be decided using the index.
virtual float	getUndecidable (const ibis::qUIntHod &cmp, ibis::bitvector &iffy) const
	Discover the records that can not be decided using the index.
bool	hasRIDs () const
const char *	indexSpec () const
	Return the current index specification.
void	indexSpec (const char *)
	Replace existing index specification with a new one.
void	loadIndexes (const char *iopt=0, int ropt=0) const
	Load indexes of all columns.
void	logMessage (const char event, const char fmt,...) const
void	logWarning (const char event, const char fmt,...) const
long	lookforString (const ibis::qString &cmp, ibis::bitvector &low) const
	Find all records that has the exact string value.
long	lookforString (const ibis::qMultiString &cmp, ibis::bitvector &low) const
	Determine the records that have the exact string values.
long	lookforString (const ibis::qString &cmp) const
	Return an upper bound of the number of records that have the exact string value.
long	lookforString (const ibis::qMultiString &cmp) const
virtual long	matchAny (const ibis::qAnyAny &cmp, ibis::bitvector &hits) const
virtual long	matchAny (const ibis::qAnyAny &cmp, const ibis::bitvector &mask, ibis::bitvector &hits) const
	Perform exact match operation for an AnyAny query.
bool	matchMetaTags (const std::vector< const char * > &mtags) const
	Match multiple name-value pairs against the internally stored meta tags.
bool	matchMetaTags (const ibis::resource::vList &mtags) const
	Match multiple name-value pairs.
bool	matchNameValuePair (const char name, const char value) const
	Match a name-value pair in the meta tags.
std::string	metaTags () const
	Return the list of meta tags as a single string.
const char *	name () const
	Return the name of the partition.
uint32_t	nColumns () const
	Return the number of attributes in the partition.
virtual long	negativeScan (const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits) const
	Compute the records (marked 1 in the mask) that does not satisfy the range condition.
uint32_t	nRows () const
	Return the number of rows.
	part (const char *name=0, bool ro=false)
	Initialize a data partition object.
	part (const char adir, const char bdir, bool ro=false)
	Initialize a table from the named directories.
	part (const std::vector< const char * > &mtags, bool ro=false)
	Initialize a partition with given meta tags.
	part (const ibis::resource::vList &mtags, bool ro=false)
	Initialize a partition with given meta tags.
long	patternSearch (const ibis::qLike &cmp, ibis::bitvector &low) const
	Look for string like the given pattern.
long	patternSearch (const ibis::qLike &cmp) const
	Look for string like the given pattern.
void	print (std::ostream &out) const
	Output a description of every column of the data partition.
long	purgeInactive ()
	Purge all inactive rows from the partition.
void	purgeIndexFiles () const
	Remove existing index files! The indexes will be rebuilt next time they are needed.
void	queryTest (const char pref, long nerrors) const
	Generate and run random queries for slefTest.
void	quickTest (const char pref, long nerrors) const
	Generate and run random queries for slefTest.
long	reactivate (const std::vector< uint32_t > &rows)
	Make sure the specified rows are active.
long	reactivate (const char *conds)
	Make sure the rows satisfying the specified conditionis are active.
void	rename (const ibis::partAssoc &known)
	Rename the partition to avoid conflicts with an existing list of data partitions.
void	rename (const char *)
	Change the name of the data partition to the given name.
virtual long	reorder ()
	Sort rows with the lowest cardinality column first.
virtual long	reorder (const ibis::table::stringList &names)
virtual long	reorder (const ibis::table::stringList &names, const std::vector< bool > &directions)
	Sort rows according the values of the columns specified in `names`.
long	rollback ()
	Rollback the append operation.
array_t< signed char > *	selectBytes (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 8-bit integers.
array_t< double > *	selectDoubles (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 64-bit floating-point values.
array_t< float > *	selectFloats (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 32-bit floating-point values.
array_t< int32_t > *	selectInts (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 32-bit integers.
array_t< int64_t > *	selectLongs (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 64-bit integers.
array_t< int16_t > *	selectShorts (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 16-bit integers.
std::vector< std::string > *	selectStrings (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as strings.
array_t< unsigned char > *	selectUBytes (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 8-bit unsigned integers.
array_t< uint32_t > *	selectUInts (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 32-bit unsigned integers.
array_t< uint64_t > *	selectULongs (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 64-bit unsigned integers.
array_t< uint16_t > *	selectUShorts (const char *name, const ibis::bitvector &mask) const
	Retrieve values of the name column as 16-bit unsigned integers.
long	selectValues (const char cname, const ibis::bitvector &mask, void vals) const
	Select values of a column based on the given mask.
long	selectValues (const ibis::qContinuousRange &cond, void *vals) const
	Select values of the column based on the range condition.
virtual long	selfTest (int nth=1, const char *pref=0) const
	Perform predefined set of tests and return the number of failures.
void	setMeshShape (const char *shape)
	Digest the mesh shape stored in the string.
void	testRangeOperators (const ibis::column col, long nerrors) const
	Try a set of range conditions with different combinations of operators.
time_t	timestamp () const
	Return the time stamp on the partition.
void	unloadIndexes () const
	Unload indexes of all columns.
int	updateData ()
	Check the time stamp on the metadata files to update the in-memory metadata information.
void	updateMetaData () const
	Write the metadata file to record the changes to the partition.
virtual	~part ()
	Destuctor.

long	get1DDistribution (const char constraints, const char cname, double begin, double end, double stride, std::vector< uint32_t > &counts) const
	Histogram functions.
long	get2DDistribution (const char constraints, const char cname1, double begin1, double end1, double stride1, const char *cname2, double begin2, double end2, double stride2, std::vector< uint32_t > &counts) const
	Compute conditional 2D histogram with regularly spaced bins.
long	get3DDistribution (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, const char cname3, double begin3, double end3, double stride3, std::vector< uint32_t > &counts) const
	Compute conditional 3D histogram with regularly spaced bins.
long	get1DDistribution (const char constraints, const char cname, double begin, double end, double stride, const char *wtname, std::vector< double > &weights) const
	Compute weighted conditional 1D histogram with regularly spaced bins.
long	get2DDistribution (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, const char wtname, std::vector< double > &weights) const
	Compute weighted conditional 2D histogram with regularly spaced bins.
long	get3DDistribution (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, const char cname3, double begin3, double end3, double stride3, const char *wtname, std::vector< double > &weights) const
	Compute weighted conditional 3D histogram with regularly spaced bins.
long	get1DDistribution (const char *cname, uint32_t nbin, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
	Compute 1D histogram with adaptive bins.
long	get1DDistribution (const char constraints, const char cname, uint32_t nbin, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
	Compute conditional 1D histogram with adaptive bins.
long	get2DDistribution (const char cname1, const char cname2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts, const char *const option=0) const
	Compute 2D histogram with adaptive bins.
long	get2DDistribution (const char constraints, const char name1, const char *name2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
	Compute conditional 2D histogram with adaptive bins.
long	get3DDistribution (const char cname1, const char cname2, const char cname3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts, const char const option=0) const
	Compute 3D histogram with adaptive bins.
long	get3DDistribution (const char constraints, const char cname1, const char cname2, const char cname3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts) const
	Compute conditional 3D histogram with adaptive bins.
long	get1DBins (const char constraints, const char cname, double begin, double end, double stride, std::vector< ibis::bitvector > &bins) const
	Partition values of the named variable into regularly spaced bins.
long	get1DBins (const char constraints, const char cname, double begin, double end, double stride, std::vector< ibis::bitvector * > &bins) const
	Partition values of the named variable into regularly spaced bins.
long	get1DBins (const char constraints, const char cname, double begin, double end, double stride, const char wtname, std::vector< double > &weights, std::vector< ibis::bitvector > &bins) const
	Partition values of the named variable into regularly spaced bins.
long	get2DBins (const char constraints, const char cname1, double begin1, double end1, double stride1, const char *cname2, double begin2, double end2, double stride2, std::vector< ibis::bitvector > &bins) const
	Partition values of named variables into regularly spaced 2D bins.
long	get2DBins (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, std::vector< ibis::bitvector > &bins) const
	Partition values of named variables into regularly spaced 2D bins.
long	get2DBins (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, const char wtname, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
	Partition values of named variables into regularly spaced 2D bins.
long	get3DBins (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, const char cname3, double begin3, double end3, double stride3, std::vector< ibis::bitvector > &bins) const
	Partition values of named variables into regularly spaced 3D bins.
long	get3DBins (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, const char cname3, double begin3, double end3, double stride3, std::vector< ibis::bitvector * > &bins) const
	Partition values of named variables into regularly spaced 3D bins.
long	get3DBins (const char constraints, const char cname1, double begin1, double end1, double stride1, const char cname2, double begin2, double end2, double stride2, const char cname3, double begin3, double end3, double stride3, const char wtname, std::vector< double > &weights, std::vector< ibis::bitvector > &bins) const
	Partition values of named variables into regularly spaced 3D bins.
long	get1DBins (const char constraints, const char cname1, uint32_t nb1, std::vector< double > &bounds1, std::vector< ibis::bitvector > &bins) const
	Partition records satisfying specified conditions into bins with about the same number of records.
long	get2DBins (const char constraints, const char cname1, const char *cname2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< ibis::bitvector > &bins) const
	Partition records satisfying specified conditions into 2D bins.
long	get3DBins (const char constraints, const char cname1, const char cname2, const char cname3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< ibis::bitvector > &bins) const
	Partition records satisfying specified conditions into 3D bins.

long	getDistribution (const char *name, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
	Obsolete histogram functions.
long	getDistribution (const char constraints, const char name, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
	Compute the conditional binned data distribution.
long	getDistribution (const char name, uint32_t nbc, double bounds, uint32_t *counts) const
	Compute the binned distribution with the specified maximum number of bins.
long	getDistribution (const char name, const char constraints, uint32_t nbc, double bounds, uint32_t counts) const
	Compute the conditional binned data distribution with the specified maximum number of bins.
long	getJointDistribution (const char constraints, const char name1, const char *name2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
	Compute the joint distribution of two variables.
long	getCumulativeDistribution (const char *name, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
	Compute a cumulative distribution (a cumulative histogram).
long	getCumulativeDistribution (const char constraints, const char name, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
	Compute the cumulative distribution of the variable named `name` under the specified constraints.
long	getCumulativeDistribution (const char name, uint32_t nbc, double bounds, uint32_t *counts) const
	This version of `getCumulativeDistribution` uses two user supplied arrays `bounds` and `counts`.
long	getCumulativeDistribution (const char constraints, const char name, uint32_t nbc, double bounds, uint32_t counts) const
	Compute the conditional distribution and return the distribution in the arrays provided.
Static Public Member Functions
static uint32_t	countPages (const ibis::bitvector &mask, unsigned elemsize=4)
	Given a `bitvector`, compute the number of pages would be accessed.
template<typename E >
static long	doScan (const array_t< E > &varr, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	Locate the records that satisfy the range condition.
template<typename E >
static long	doScan (const array_t< E > &varr, const ibis::qContinuousRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
template<typename E >
static long	doScan (const array_t< E > &varr, const ibis::qContinuousRange &cmp, const ibis::bitvector &mask, array_t< E > &res)
template<typename E >
static long	doScan (const array_t< E > &varr, const ibis::qContinuousRange &cmp, const ibis::bitvector &mask, array_t< E > &res, ibis::bitvector &hits)
static void	genName (const std::vector< const char * > &mtags, std::string &name)
	Generate name for a partition based on the meta tags.
static void	genName (const ibis::resource::vList &mtags, std::string &name)
	Generate name for a partition based on the meta tags.
static char *	readMetaTags (const char *const dir)
	A class function to read the meta tags in the tdc file.
static const char *	skipPrefix (const char *)
	Skip pass all the dots in the given string.
template<typename T >
static int	writeColumn (int fdes, ibis::bitvector::word_t nold, ibis::bitvector::word_t nnew, const array_t< T > &vals, const T &fill, ibis::bitvector &totmask, const ibis::bitvector &newmask)
	Write the content of vals to an open file.
static int	writeRaw (int bdes, int sdes, ibis::bitvector::word_t nold, ibis::bitvector::word_t nnew, const ibis::array_t< unsigned char > &bytes, const ibis::array_t< int64_t > &starts, ibis::bitvector &totmask, const ibis::bitvector &newmask)
	Write raw bytes to an open file.
static int	writeString (int fdes, ibis::bitvector::word_t nold, ibis::bitvector::word_t nnew, const std::vector< std::string > &vals, ibis::bitvector &totmask, const ibis::bitvector &newmask)
	Write strings to an open file.
Protected Member Functions
long	append1 (const char *dir)
	Perform append operation using only only one data directory.
long	append2 (const char *dir)
	Perform append operation with two data directories.
long	appendToBackup (const char *dir)
	Append data in `dir` to the partition in the backup directory.
void	buildQueryList (ibis::part::thrArg &lst, unsigned nc, unsigned nq) const
	Generate a list of random query conditions.
void	checkQueryList (const ibis::part::thrArg &lst) const
	Sum up the hits from sub-divisions to verify the hits computing from the whole range.
int	coarsenBins (const ibis::column &col, uint32_t nbin, std::vector< double > &bnds, std::vector< ibis::bitvector * > &btmp) const
	Produce a set of bitmaps corresponding to a set of coarse bins.
void	composeQueryString (std::string &str, const ibis::column col1, const ibis::column col2, const double &lower1, const double &upper1, const double &lower2, const double &upper2) const
template<typename T1 , typename T2 >
long	count2DBins (array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, std::vector< uint32_t > &counts) const
	Count the number of values in 2D bins.
template<typename T1 , typename T2 >
long	count2DWeights (array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, array_t< double > &wts, std::vector< double > &weights) const
	Count the weights in 2D bins.
template<typename T1 , typename T2 , typename T3 >
long	count3DBins (const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, std::vector< uint32_t > &counts) const
	Count the number of values in 3D bins.
template<typename T1 , typename T2 , typename T3 >
long	count3DWeights (const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights) const
	Count the weights in 3D bins.
long	deactivate (const ibis::bitvector &rows)
	Rows marked 1 will become inactive.
void	deriveBackupDirName ()
void	digestMeshShape (const char *shape)
	Convert the string describing the shape into internal storage format.
template<typename T >
long	doCount (const ibis::qRange &cmp) const
	Count the number rows satisfying the range expression.
template<typename T >
long	doCount (const array_t< T > &vals, const ibis::qIntHod &cmp, const ibis::bitvector &mask) const
	Count the number rows satisfying the range expression.
template<typename T >
long	doCount (const array_t< T > &vals, const ibis::qUIntHod &cmp, const ibis::bitvector &mask) const
	Count the number rows satisfying the range expression.
template<typename T >
long	doCount (const array_t< T > &vals, const ibis::qRange &cmp, const ibis::bitvector &mask) const
	Count the number rows satisfying the range expression.
template<typename T , typename F >
long	doCount (const array_t< T > &vals, const ibis::bitvector &mask, F cmp) const
	Count the number rows satisfying the range expression.
template<typename T , typename F1 , typename F2 >
long	doCount (const array_t< T > &vals, const ibis::bitvector &mask, F1 cmp1, F2 cmp2) const
	Count the number rows satisfying the range expression.
void	extendMetaTags ()
template<typename T1 >
long	fill1DBins (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, std::vector< ibis::bitvector > &bins) const
	Fill the bitvectors representing the 1D bins.
template<typename T1 >
long	fill1DBins (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, std::vector< ibis::bitvector * > &bins) const
	Fill the bitvectors representing the 1D bins.
template<typename T1 >
long	fill1DBinsWeighted (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
	Fill the bitvectors representing the 1D bins.
template<typename T1 , typename T2 >
long	fill2DBins (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, std::vector< ibis::bitvector > &bins) const
	Fill the bitvectors representing the 2D bins.
template<typename T1 , typename T2 >
long	fill2DBins (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, std::vector< ibis::bitvector * > &bins) const
	Fill the bitvectors representing the 2D bins.
template<typename T1 >
long	fill2DBins2 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, std::vector< ibis::bitvector > &bins) const
	A template function to resolve the second variable involved in the 2D bins.
template<typename T1 >
long	fill2DBins2 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, std::vector< ibis::bitvector * > &bins) const
	This version returns a vector of pointers to bitmaps.
template<typename T1 , typename T2 >
long	fill2DBinsWeighted (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
	Fill the bitvectors representing the 2D bins.
template<typename T1 >
long	fill2DBinsWeighted2 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
	This version returns a vector of pointers to bitmaps.
template<typename T1 , typename T2 , typename T3 >
long	fill3DBins (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, std::vector< ibis::bitvector > &bins) const
	Fill the bitvectors representing the 3D bins.
template<typename T1 , typename T2 , typename T3 >
long	fill3DBins (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, std::vector< bitvector * > &bins) const
	The three triplets, (begin1, end1, stride1), (begin2, end2, stride2), and (begin3, end3, stride3), defines `(1 + floor((end1 - begin1) / stride1)) (1 + floor((end2 - begin2) / stride2)) (1 + floor((end3 - begin3) / stride3))` 3D bins.
template<typename T1 >
long	fill3DBins2 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, std::vector< bitvector > &bins) const
	Resolve the 2nd column of the 3D bins.
template<typename T1 >
long	fill3DBins2 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, std::vector< ibis::bitvector * > &bins) const
	Resolve the 2nd column of the 3D bins.
template<typename T1 , typename T2 >
long	fill3DBins3 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, std::vector< bitvector > &bins) const
	Resolve the 3rd column involved in the 3D bins.
template<typename T1 , typename T2 >
long	fill3DBins3 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, std::vector< ibis::bitvector * > &bins) const
	Resolve the 3rd column involved in the 3D bins.
template<typename T1 , typename T2 , typename T3 >
long	fill3DBinsWeighted (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const array_t< T3 > &vals3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights, std::vector< bitvector * > &bins) const
	The three triplets, (begin1, end1, stride1), (begin2, end2, stride2), and (begin3, end3, stride3), defines `(1 + floor((end1 - begin1) / stride1)) (1 + floor((end2 - begin2) / stride2)) (1 + floor((end3 - begin3) / stride3))` 3D bins.
template<typename T1 >
long	fill3DBinsWeighted2 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const ibis::column &col2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
	Resolve the 2nd column of the 3D bins.
template<typename T1 , typename T2 >
long	fill3DBinsWeighted3 (const ibis::bitvector &mask, const array_t< T1 > &vals1, const double &begin1, const double &end1, const double &stride1, const array_t< T2 > &vals2, const double &begin2, const double &end2, const double &stride2, const ibis::column &col3, const double &begin3, const double &end3, const double &stride3, const array_t< double > &wts, std::vector< double > &weights, std::vector< ibis::bitvector * > &bins) const
	Resolve the 3rd column involved in the 3D bins.
void	freeRIDs () const
	Remove the rids list from memory.
void	gatherSortKeys (ibis::table::stringList &names)
long	get1DBins_ (const ibis::bitvector &mask, const ibis::column &col, uint32_t nbin, std::vector< double > &bounds, std::vector< ibis::bitvector > &bins, const char *mesg) const
	Compute 1D histogram from raw data.
long	get1DDistribution (const ibis::column &col, uint32_t nbin, std::vector< double > &bounds, std::vector< uint32_t > &counts) const
	Compute 1D histogram from index.
long	get2DDistributionA (const ibis::column &col1, const ibis::column &col2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
	Compute 2D histogram with adaptive bins from base data.
long	get2DDistributionI (const ibis::column &col1, const ibis::column &col2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
	Compute 2D histogram from indexes.
long	get2DDistributionU (const ibis::column &col1, const ibis::column &col2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
	Compute 2D histogram with uniform bins from base data.
long	get3DDistributionA (const ibis::bitvector &mask, const ibis::column &col1, const ibis::column &col2, const ibis::column &col3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts) const
	Compute 3D histogram with adaptive bins from base data.
template<typename E1 >
long	get3DDistributionA1 (const ibis::bitvector &mask, const array_t< E1 > &vals1, const ibis::column &col2, const ibis::column &col3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts) const
	Read the value of the second column.
template<typename E1 , typename E2 >
long	get3DDistributionA2 (const ibis::bitvector &mask, const array_t< E1 > &vals1, const array_t< E2 > &vals2, const ibis::column &col3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts) const
	Read the values of the third column.
void	logError (const char event, const char fmt,...) const
	Write out a message with indication of severe error.
void	makeBackupCopy ()
	Spawn another thread to copy the content of `activeDir` to `backupDir`.
void	numbersToBitvector (const std::vector< uint32_t > &, ibis::bitvector &) const
	Turn a list of numbers into a bitvector.
long	old2DDistribution (const char constraints, const char name1, const char *name2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts) const
	The old implementation that uses binary lookup.
long	packCumulativeDistribution (const std::vector< double > &bounds, const std::vector< uint32_t > &counts, uint32_t nbc, double bptr, uint32_t cptr) const
	Pack a cumulative distribution stored in two std::vectors into two arrays provided by the caller.
long	packDistribution (const std::vector< double > &bounds, const std::vector< uint32_t > &counts, uint32_t nbc, double bptr, uint32_t cptr) const
	Pack a binned distribution.
long	reactivate (const ibis::bitvector &rows)
	Change all rows marked 1 to be active.
void	readMeshShape (const char *const dir)
	Read shape of the mesh from the metadata file.
int	readMetaData (uint32_t &nrows, columnList &plist, const char *dir)
	Read metadata file -part.txt.
void	readRIDs () const
	Read RIDs from file 'rids'.
uint32_t	recursiveQuery (const char pref, const column att, double low, double high, long *nerr) const
	Issues a query and then subdivided the range into three to check the total hits of the three sub queries matches the hits of the single query.
template<typename T >
long	reorderValues (const char *fname, array_t< uint32_t > &starts, array_t< uint32_t > &indout, const array_t< uint32_t > &indin, bool ascending)
	Write the named data file in a segmented sorted order.
void	setMetaTags (const ibis::resource::vList &mts)
	Make a deep copy of the incoming name-value pairs.
void	setMetaTags (const std::vector< const char * > &mts)
	Make a deep copy of the incoming name-value pairs.
void	stringToBitvector (const char *, ibis::bitvector &) const
	Convert a set of range conditions to an ibis::bitvector.
long	verifyBackupDir ()
void	writeMetaData (const uint32_t nrows, const columnList &plist, const char *dir) const
	Write metadata file -part.txt.
template<typename T >
long	writeValues (const char *fname, const array_t< uint32_t > &ind)
	Write the named data file with values in the given order.
Static Protected Member Functions
template<typename T1 , typename T2 >
static long	adaptive2DBins (const array_t< T1 > &vals1, const array_t< T2 > &vals2, uint32_t nb1, uint32_t nb2, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< uint32_t > &counts)
	Adaptive binning through regularly spaced bins.
template<typename T1 , typename T2 , typename T3 >
static long	adaptive3DBins (const array_t< T1 > &vals1, const array_t< T2 > &vals2, const array_t< T3 > &vals3, uint32_t nb1, uint32_t nb2, uint32_t nb3, std::vector< double > &bounds1, std::vector< double > &bounds2, std::vector< double > &bounds3, std::vector< uint32_t > &counts)
	Adaptive binning through regularly spaced bins.
template<typename T >
static long	adaptiveFloats (const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< uint32_t > &counts)
	The adaptive binning function for floats and integers in wide ranges.
template<typename T >
static long	adaptiveFloatsDetailed (const ibis::bitvector &mask, const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< ibis::bitvector > &detail)
	Bins the given values so that each each bin is nearly equal weight.
template<typename T >
static long	adaptiveInts (const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< uint32_t > &counts)
	The adaptive binning function for integer values.
template<typename T >
static long	adaptiveIntsDetailed (const ibis::bitvector &mask, const array_t< T > &vals, const T vmin, const T vmax, uint32_t nbins, std::vector< double > &bounds, std::vector< ibis::bitvector > &detail)
	Bins the given values so that each each bin is nearly equal weight.
template<typename T , typename F >
static long	doComp (const array_t< T > &vals, F cmp, const ibis::bitvector &mask, array_t< T > &res, ibis::bitvector &hits)
	Evaluate the range condition.
template<typename T , typename F >
static long	doComp (const array_t< T > &vals, F cmp, const ibis::bitvector &mask, array_t< T > &res)
	Evaluate the range condition.
template<typename T , typename F >
static long	doComp (const array_t< T > &vals, F cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	Evaluate the range condition.
template<typename T , typename F1 , typename F2 >
static long	doComp (const array_t< T > &vals, F1 cmp1, F2 cmp2, const ibis::bitvector &mask, array_t< T > &res, ibis::bitvector &hits)
	Evaluate the range condition.
template<typename T , typename F1 , typename F2 >
static long	doComp (const array_t< T > &vals, F1 cmp1, F2 cmp2, const ibis::bitvector &mask, array_t< T > &res)
	Evaluate the range condition.
template<typename T , typename F1 , typename F2 >
static long	doComp (const array_t< T > &vals, F1 cmp1, F2 cmp2, const ibis::bitvector &mask, ibis::bitvector &hits)
	Evaluate the range condition.
template<typename T , typename F >
static long	doComp0 (const array_t< T > &vals, F cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	Evaluate the range condition.
template<typename T , typename F1 , typename F2 >
static long	doComp0 (const array_t< T > &vals, F1 cmp1, F2 cmp2, const ibis::bitvector &mask, ibis::bitvector &hits)
	This version uses uncompressed bitvector to store the scan results internally.
template<typename T >
static long	doCompare (const array_t< T > &array, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::array_t< T > &res, ibis::bitvector &hits)
	The function that performs the actual comparison for range queries.
template<typename T >
static long	doCompare (const char *file, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::array_t< T > &res, ibis::bitvector &hits)
	Perform comparisons with data in the named file.
template<typename T >
static long	doCompare (const array_t< T > &array, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::array_t< T > &res)
	The function that performs the actual comparison for range queries.
template<typename T >
static long	doCompare (const char *file, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::array_t< T > &res)
	Perform comparisons with data in the named file.
template<typename T >
static long	doCompare (const array_t< T > &array, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	The function that performs the actual comparison for range queries.
template<typename T >
static long	doCompare (const char *file, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	Perform comparisons with data in the named file.
template<typename T >
static long	doCompare (const array_t< T > &array, const ibis::qIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	The function that performs the actual comparison for range queries.
template<typename T >
static long	doCompare (const char *file, const ibis::qIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	Evaluate the range condition.
template<typename T >
static long	doCompare (const array_t< T > &array, const ibis::qUIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	The function that performs the actual comparison for range queries.
template<typename T >
static long	doCompare (const char *file, const ibis::qUIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
template<typename T >
static void	equalWeightBins (const array_t< T > &vals, uint32_t nbins, array_t< T > &bounds)
template<typename E1 , typename E2 >
static void	mapValues (array_t< E1 > &val1, array_t< E2 > &val2, uint32_t nb1, uint32_t nb2, array_t< E1 > &bnd1, array_t< E2 > &bnd2, std::vector< uint32_t > &cnts)
	The templated function to decide the bin boundaries and count the number of values fall in each bin.
template<typename T >
static void	mapValues (const array_t< T > &vals, std::map< T, uint32_t > &hist)
template<typename T >
static long	negativeCompare (const array_t< T > &array, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
template<typename T >
static long	negativeCompare (const char *file, const ibis::qRange &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
template<typename T >
static long	negativeCompare (const array_t< T > &array, const ibis::qIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	Perform the negative comparison.
template<typename T >
static long	negativeCompare (const char *file, const ibis::qIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
	Perform the negative comparison.
template<typename T >
static long	negativeCompare (const array_t< T > &array, const ibis::qUIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
template<typename T >
static long	negativeCompare (const char *file, const ibis::qUIntHod &cmp, const ibis::bitvector &mask, ibis::bitvector &hits)
Protected Attributes
char *	activeDir
	The active data directory.
ibis::bitvector	amask
	Active rows are maked 1.
char *	backupDir
	The backup data directory.
std::vector< const column * >	colorder
	An ordering of columns.
columnList	columns
	List of the columns.
char *	idxstr
	Index specification.
std::string	m_desc
	Free form description of the partition.
char *	m_name
	Name of the data partition.
ibis::resource::vList	metaList
	Meta tags as name-value pairs.
ibis::part::cleaner *	myCleaner
	The cleaner for the file manager.
uint32_t	nEvents
	Number of events (rows) in the partition.
bool	readonly
	Don't change the data.
array_t< rid_t > *	rids
	The object IDs (row id).
std::vector< std::string >	shapeName
	Names of the dimensions.
std::vector< uint32_t >	shapeSize
	Sizes of the dimensions.
TABLE_STATE	state
time_t	switchTime
	Time of last switch operation.
Friends
class	cleaner
struct	info
class	mutexLock
class	readLock
class	softWriteLock
class	writeLock

Detailed Description

The class ibis::part represents a partition of a relational table.

The current implementation is designed to work with vertically partitioned data files. This class contains common information and operations on a data partition. It must have a name. Following SQL convention, the name must start with a underscore or an ASCII alphabet and may be followed by any number of ASCII alphanumeric characters or underscore.

Constructor & Destructor Documentation

ibis::part::part	(	const char *	name = `0`,
		bool	ro = `false`
	)		`[explicit]`

Initialize a data partition object.

The incoming argument can be a directory name or a data partition name.

If it contains any forward or backward slash, it is assumed to be a directory name. Otherwise, it is assumed to be the name of a data partition. In which case, this function looks for data directory names in the global parameter list under the parameters 'name.activeDir' and 'name.backupDir' or 'name.dataDir1' and 'name.dataDir2'. If the name is a directory name, then no attempt shall be made to produce a second directory name.

The default value for name is a nil pointer. In this case, it will find 'dataDir1' and 'dataDir2' from the global parameter list.

The default argument for ro is false, which allows new directory to be created and new data records to be appended. If the argument ro is true, then the specified data directory must already exist, otherwise, an exception is thrown.

Referenced by ibis::util::gatherParts().

ibis::part::part	(	const char *	adir,
		const char *	bdir,
		bool	ro = `false`
	)

Initialize a table from the named directories.

Construct a partition from the named directories.

Originally, FastBit was designed to work with a pair of directories, adir and bdir. Therefore, the constructor takes a pair of directory names. In many cases, data is stored only in one directory, in which simply give the data directory as adir and leave bdir as null. Prefer to have full and complete path.

The default argument for ro is false, which allows new directory to be created and new data records to be appended. If the argument ro is true, then the specified data directory must already exist, otherwise, an exception is thrown.

References activeDir, ibis::bitvector::adjustSize(), amask, backupDir, ibis::bitvector::cnt(), columns, ibis::util::envLock, ibis::fileManager::flushFile(), ibis::gParameters(), ibis::gVerbose, ibis::fileManager::instance(), ibis::resource::isStringTrue(), m_desc, m_name, makeBackupCopy(), ibis::util::makeDir(), myCleaner, name(), nEvents, print(), ibis::bitvector::read(), readMetaData(), readonly, readRIDs(), ibis::util::removeDir(), ibis::util::removeTail(), rids, ibis::bitvector::set(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::util::strnewdup(), switchTime, and ibis::bitvector::write().

ibis::part::part	(	const std::vector< const char * > &	mtags,
		bool	ro = `false`
	)

Initialize a partition with given meta tags.

The meta tags are specified as a list of name-value strings, where each string in one name-value pair.

References genName(), and setMetaTags().

ibis::part::part	(	const ibis::resource::vList &	mtags,
		bool	ro = `false`
	)

Initialize a partition with given meta tags.

The name-value pairs are specified in a structured form.

References genName(), and setMetaTags().

Member Function Documentation

template<typename T1 , typename T2 >

template long ibis::part::adaptive2DBins	(	const array_t< T1 > &	vals1,
		const array_t< T2 > &	vals2,
		uint32_t	nb1,
		uint32_t	nb2,
		std::vector< double > &	bounds1,
		std::vector< double > &	bounds2,
		std::vector< uint32_t > &	counts
	)		`[static, protected]`

Adaptive binning through regularly spaced bins.

It goes through the arrays twice, once to compute the actual minimum and maximum values and once to count the entries in each bins. It produces three sets of bins: the 1-D bins for vals1 and vals2, and a 2-D bin at a high resolution. It then combine the 1-D bins to form nearly equal-weight bins and use that grouping to decide how to combine the 2-D bins to form the final output.

Note:

The number of fine bins used internally is dynamically determined based on the number of desired bins on output, nb1 and nb2, as well as the number of records in vals1 and vals2.

The output number of bins may not be exactly nb1*nb2. Here are some of the reasons.

If either nb1 or nb2 is less than or equal to one, it is set 100.
If either nb1 or nb2 is larger than 2048, it may be reset to a smaller value so that the number of records in each bin might be about cubic root of the total number of records on input.
It may be necessary to use a few more or less bins (along each dimension) to avoid grouping very popular values with very unpopular values into the same bin.

References ibis::index::divideCounts(), ibis::gVerbose, ibis::util::incrDouble(), ibis::array_t< T >::resize(), and ibis::array_t< T >::size().

template<typename T1 , typename T2 , typename T3 >

long ibis::part::adaptive3DBins	(	const array_t< T1 > &	vals1,
		const array_t< T2 > &	vals2,
		const array_t< T3 > &	vals3,
		uint32_t	nb1,
		uint32_t	nb2,
		uint32_t	nb3,
		std::vector< double > &	bounds1,
		std::vector< double > &	bounds2,
		std::vector< double > &	bounds3,
		std::vector< uint32_t > &	counts
	)		`[static, protected]`

Adaptive binning through regularly spaced bins.

Note:

Here are the special cases that are different from ibis::part::adaptive2DBins.

If the number of desired bins along any of the three dimensions, nb1, nb2, or nb3, is zero (0) or one (1), it is set to 32. If all three dimensions are using 32 bins, there is a total of 32,768 bins altogether.
If the number of desired bins along any of the three dimensions, nb1, nb2, or nb3, is greater than 128, it may be reduced to about fourth root of the number of records in input.

See also:: ibis::part::adaptive2DBins

References ibis::index::divideCounts(), ibis::gVerbose, ibis::util::incrDouble(), ibis::array_t< T >::resize(), and ibis::array_t< T >::size().

template<typename T >

template long ibis::part::adaptiveFloats	(	const array_t< T > &	vals,
		const T	vmin,
		const T	vmax,
		uint32_t	nbins,
		std::vector< double > &	bounds,
		std::vector< uint32_t > &	counts
	)		`[static, protected]`

The adaptive binning function for floats and integers in wide ranges.

This function first constructs a number of fine uniform bins and then merge the fine bins to generate nearly equal-weight bins. This is likely to produce final bins that are not as equal in their weights as those produced from ibis::part::adaptiveInts, but because it usually does less work and takes less time.

The number of fine bins used is the larger one of 8 times the number of desired bins and the geometric mean of the number of desired bins and the number of records in vals.

Note:

This function still relies on the caller to compute vmin and vmax, but it assumes there are many distinct values in each bin.

The output number of bins may not be the input value nbins for the following reasons.

If nbins is 0 or 1, it is reset to 1000 in this function;
if nbins is greater than 1/4 of vals.size(), it is set ot vals.size()/4;
in all other cases, the final number of bins is determine by the function that partitions the fine bins into coarse bins, ibis::index::divideCounts. This partition process may not produce exactly nbins bins.

See also:: ibis::part::adaptiveInts.

References ibis::index::divideCounts(), ibis::gVerbose, ibis::util::incrDouble(), and ibis::array_t< T >::size().

template<typename T >

long ibis::part::adaptiveFloatsDetailed	(	const ibis::bitvector &	mask,
		const array_t< T > &	vals,
		const T	vmin,
		const T	vmax,
		uint32_t	nbins,
		std::vector< double > &	bounds,
		std::vector< ibis::bitvector > &	detail
	)		`[static, protected]`

Bins the given values so that each each bin is nearly equal weight.

Instead of counting the number entries in each bin return bitvectors that mark the positions of the records. This version is for floating-point values and integer values with wide ranges. This function first bins the values into a relatively large number of fine equal-width bins and then coalesce nearby fines bins to for nearly equal-weight bins. The final bins produced this way are less likely to be very uniform in their weights, but it requires less internal work space and therefore may be faster than ibis::part::adaptiveIntsDetailed.

See also:: ibis::part::adapativeFloats

References ibis::bitvector::cnt(), ibis::index::divideCounts(), ibis::gVerbose, ibis::util::incrDouble(), ibis::array_t< T >::size(), ibis::bitvector::size(), and ibis::index::sumBits().

template<typename T >

template long ibis::part::adaptiveInts	(	const array_t< T > &	vals,
		const T	vmin,
		const T	vmax,
		uint32_t	nbins,
		std::vector< double > &	bounds,
		std::vector< uint32_t > &	counts
	)		`[static, protected]`

The adaptive binning function for integer values.

It is intended for values within a relatively narrow range. The input arguments vmin and vmax must be the correct minimum and maximum values -- it uses the minimum and maximum valuse to decided whether an exact histogram can be used internally; incorrect values for vmin or vmax may cuase this function to misbehave!

It counts the frequency of each distinct value before deciding how to produce the equal-weight bins for output. Because it has the most detailed information possible, the output bins are mostly to be about equal. This comes with a cost of a detailed frequency count, which takes time and memory space to compute.

Note:

The output number of bins may not be the input value nbins because of following reasons.

If nbins is 0 or 1, it is set to 1000 in this function.
If nbins is larger than 2/3rds of the number of distinct values as indicated by vmin and vmax, each value will have its own bin.
In other cases, this function calls the function ibis::index::divideCounts to determine how to coalesce different fine bins into nbins bins on output. However, it is possible that the function ibis::index::divideCounts may have trouble produce exactly nbins as requested.

References ibis::index::divideCounts(), ibis::gVerbose, ibis::array_t< T >::resize(), and ibis::array_t< T >::size().

template<typename T >

long ibis::part::adaptiveIntsDetailed	(	const ibis::bitvector &	mask,
		const array_t< T > &	vals,
		const T	vmin,
		const T	vmax,
		uint32_t	nbins,
		std::vector< double > &	bounds,
		std::vector< ibis::bitvector > &	detail
	)		`[static, protected]`

Bins the given values so that each each bin is nearly equal weight.

Instead of counting the number entries in each bin return bitvectors that mark the positions of the records. This version is for integer values in relatively narrow ranges. It will count each distinct value separately, which gives the most accurate information for deciding how to produce equal-weight bins. If there are many dictinct values, this function will require considerable amount of internal memory to count each distinct value.

On successful completion of this function, the return value is the number of bins used. If the input array is empty, it returns 0 without modifying the content of the output arrays, bounds and detail. Either mask and vals have the same number of elements, or vals has as many elements as the number of ones (1) in mask, otherwise this function will return -51.

See also:: ibis::part::adaptiveInts

References ibis::bitvector::cnt(), ibis::index::divideCounts(), ibis::gVerbose, ibis::array_t< T >::resize(), ibis::array_t< T >::size(), ibis::bitvector::size(), and ibis::index::sumBits().

long ibis::part::addColumn	(	const char *	aexpr,
		const char *	cname,
		ibis::TYPE_T	ctype = `ibis::DOUBLE`
	)

Add a column computed with the given arithmetic expression.

The arithmetic expression is evaluated in double and casted to the specified type.

References ibis::selectClause::aggExpr(), ibis::selectClause::aggSize(), and ibis::gVerbose.

long ibis::part::append ( const char * dir )

Append data from dir.

Append data in dir to the current database.

Return the number of rows actually added.

Note:: If there is a backup data directory, it is possible to rollback the append operation before commit.

References ibis::fileManager::flushDir(), and ibis::fileManager::instance().

long ibis::part::append1 ( const char * dir ) [protected]

Perform append operation using only only one data directory.

Must wait for all queries on the partition to finish before preceding.

References ibis::fileManager::flushDir(), ibis::fileManager::flushFile(), ibis::gParameters(), ibis::gVerbose, and ibis::fileManager::instance().

long ibis::part::append2 ( const char * dir ) [protected]

Perform append operation with two data directories.

It appends the data to the backup directory first, then swap the roles of the two directories.

References ibis::fileManager::flushDir(), ibis::fileManager::flushFile(), ibis::util::getFileSize(), ibis::gParameters(), ibis::gVerbose, ibis::fileManager::instance(), and ibis::util::removeDir().

long ibis::part::appendToBackup ( const char * dir ) [protected]

Append data in dir to the partition in the backup directory.

Return the number of rows actually appended.

References ibis::fileManager::buffer< T >::address(), ibis::column::append(), ibis::CATEGORY, ibis::horometer::CPUTime(), ibis::fileManager::flushDir(), ibis::util::getFileSize(), ibis::gVerbose, ibis::fileManager::instance(), ibis::column::name(), ibis::OID, ibis::horometer::realTime(), ibis::fileManager::buffer< T >::size(), ibis::horometer::start(), ibis::horometer::stop(), ibis::TEXT, ibis::TYPESTRING, and ibis::column::upperBound().

int ibis::part::buildIndexes	(	const char *	iopt = `0`,
		int	nthr = `1`
	)		`[virtual]`

Make sure indexes for all columns are available.

May use nthr threads to build indexes. The argument iopt is used to build new indexes if the corresponding columns do not already have indexes.

See also:: ibis::part::loadIndexes

Reimplemented in ibis::bord.

References ibis::horometer::CPUTime(), ibis::gVerbose, ibis::horometer::realTime(), ibis::horometer::start(), and ibis::horometer::stop().

Referenced by fastbit_build_indexes().

void ibis::part::buildQueryList	(	ibis::part::thrArg &	lst,
		unsigned	nc,
		unsigned	nq
	)		const `[protected]`

Generate a list of random query conditions.

It select nc columns from the list of all columns and fills the array lst.conds and lst.super. The array lst.hits is resized to the correct size, but left to be filled with other functions. It generates at least nc-1 queries. When nq > nc, it may generate one nq+1 queries because it always adds two subranges as queries together. This is to ensure that two sub-ranges of any given range is present together for checkQueryList.

References ibis::part::thrArg::conds, ibis::gVerbose, ibis::part::thrArg::hits, ibis::part::thrArg::nerrors, ibis::util::rand(), and ibis::part::thrArg::super.

void ibis::part::buildSorted ( const char * cname ) const

Build a sorted version of the specified column.

Will sort the base data of the named column if needed.

References ibis::gVerbose.

long ibis::part::calculate	(	const ibis::math::term &	trm,
		const ibis::bitvector &	msk,
		array_t< double > &	res
	)		const

Calculate the values of an arithmetic expression.

The arithmetic expression is applied to each row that are marked 1 in the mask, msk, with names in the arithmetic expression interpretted as column names. The resulting values are packed into the array res as doubles. Upon the successful completion of this function, the return value should be the number of records examined, which should be same as msk.cnt() and res.size().

References ibis::array_t< T >::clear(), ibis::bitvector::cnt(), ibis::math::term::eval(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), ibis::part::barrel::open(), ibis::array_t< T >::push_back(), ibis::part::barrel::read(), ibis::horometer::realTime(), ibis::math::barrel::recordVariable(), ibis::array_t< T >::reserve(), ibis::array_t< T >::resize(), ibis::part::barrel::seek(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), ibis::horometer::stop(), and ibis::array_t< T >::swap().

Referenced by ibis::bord::append(), and ibis::bord::groupbyc().

void ibis::part::checkQueryList ( const ibis::part::thrArg & lst ) const [protected]

Sum up the hits from sub-divisions to verify the hits computing from the whole range.

Based on the construction of buildQueryList, each query condition knows which conditions contains it.

References ibis::part::thrArg::conds, ibis::gVerbose, ibis::part::thrArg::hits, ibis::part::thrArg::nerrors, and ibis::part::thrArg::super.

int ibis::part::coarsenBins	(	const ibis::column &	col,
		uint32_t	nbin,
		std::vector< double > &	bnds,
		std::vector< ibis::bitvector * > &	btmp
	)		const `[protected]`

Produce a set of bitmaps corresponding to a set of coarse bins.

This function makes use of an existing index to produce bitmaps representing a set of bins defined by bnds.

Following the private convention used in FastBit, there are two open bins at the two ends.

References ibis::index::binBoundaries(), ibis::util::copy(), ibis::index::divideCounts(), ibis::index::evaluate(), ibis::column::getActualMax(), ibis::gVerbose, ibis::column::name(), ibis::array_t< T >::resize(), and ibis::array_t< T >::size().

ibis::table::stringList ibis::part::columnNames ( ) const

Return column names in a list.

The list contains raw pointers. These pointers are valid as long as the part objects are present in memory.

Reimplemented in ibis::bord.

References ibis::array_t< T >::clear().

Referenced by ibis::bord::columnNames().

long ibis::part::commit ( const char * dir )

Commit the append operation involving data from dir.

Commit the active database.

Note:: The content of dir must match the content of directory passed to append. Clearly, the easiest way to achieve this is to use the same directory.

No longer able to rollback afterward. Return the number of records committed.

References ibis::fileManager::flushDir(), ibis::fileManager::flushFile(), ibis::gVerbose, and ibis::fileManager::instance().

void ibis::part::computeMinMax ( )

Compute the min and max for each column.

Actually compute the min and max of each attribute and write out a new metadata file for the data partition.

uint32_t ibis::part::countPages	(	const ibis::bitvector &	mask,
		unsigned	elemsize = `4`
	)		`[static]`

Given a bitvector, compute the number of pages would be accessed.

Assumes the pages are packed with values.

References ibis::bitvector::cnt(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), and ibis::fileManager::pageSize().

long ibis::part::deactivate ( const std::vector< uint32_t > & rows )

Mark the specified rows as inactive.

The integers in array rows are simply the row numbers. Note rows are numbered starting from 0. Return the number of rows inactive or error code.

Note:: Inactive rows will no longer participate in future query evaluations.

References ibis::bitvector::cnt().

long ibis::part::deactivate ( const char * conds )

Mark all rows satisfying the specified conditions as inactive.

All rows satisfying the specified conditions will be made inactive. Return the number of rows inactive or error code.

Note:: All inactive rows will no longer participate in any future query processing.

References ibis::bitvector::cnt(), and ibis::gVerbose.

long ibis::part::deactivate ( const ibis::bitvector & rows ) [protected]

Rows marked 1 will become inactive.

Mark the rows identified in rows as inactive.

Return the number of rows inactive or error code.

Note:: Inactive rows will no longer participate in future query evaluations.

References ibis::bitvector::cnt(), ibis::fileManager::flushFile(), ibis::gVerbose, ibis::fileManager::instance(), ibis::bitvector::size(), and ibis::bitvector::write().

void ibis::part::digestMeshShape ( const char * shape ) [protected]

Convert the string describing the shape into internal storage format.

Digest the column shape string read from metadata file.

References ibis::gVerbose.

Referenced by setMeshShape().

void ibis::part::doBackup ( )

A function to start backing up the active dir.

The routine to perform the actual copying for making a backup copy.

References backupDir, ibis::util::envLock, ibis::gVerbose, logError(), and ibis::util::removeDir().

template<typename T , typename F >

long ibis::part::doComp	(	const array_t< T > &	vals,
		F	cmp,
		const ibis::bitvector &	mask,
		array_t< T > &	res,
		ibis::bitvector &	hits
	)		`[static, protected]`

Evaluate the range condition.

Accepts an externally passed comparison operator. It chooses whether the bitvector hits will be compressed internally based on the number of set bits in the mask.

References ibis::array_t< T >::capacity(), ibis::array_t< T >::clear(), ibis::bitvector::clear(), ibis::bitvector::cnt(), ibis::bitvector::copy(), ibis::gVerbose, name(), ibis::array_t< T >::nosharing(), ibis::array_t< T >::push_back(), ibis::array_t< T >::reserve(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

Referenced by doScan().

template<typename T , typename F >

long ibis::part::doComp	(	const array_t< T > &	vals,
		F	cmp,
		const ibis::bitvector &	mask,
		array_t< T > &	res
	)		`[static, protected]`

Evaluate the range condition.

Accepts an externally passed comparison operator. It chooses whether the bitvector hits will be compressed internally based on the number of set bits in the mask.

References ibis::array_t< T >::capacity(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), ibis::gVerbose, name(), ibis::array_t< T >::nosharing(), ibis::array_t< T >::push_back(), ibis::array_t< T >::reserve(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T , typename F >

long ibis::part::doComp	(	const array_t< T > &	vals,
		F	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

Evaluate the range condition.

Accepts an externally passed comparison operator. It chooses whether the bitvector hits will be compressed internally based on the number of set bits in the mask.

References ibis::bitvector::adjustSize(), ibis::bitvector::clear(), ibis::bitvector::cnt(), ibis::bitvector::compress(), ibis::bitvector::decompress(), ibis::gVerbose, name(), ibis::bitvector::reserve(), ibis::bitvector::set(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T , typename F1 , typename F2 >

long ibis::part::doComp	(	const array_t< T > &	vals,
		F1	cmp1,
		F2	cmp2,
		const ibis::bitvector &	mask,
		array_t< T > &	res,
		ibis::bitvector &	hits
	)		`[static, protected]`

Evaluate the range condition.

The actual comparison functions are only applied on rows with mask == 1. The values satisfying the comparison operators are stored in res. This function reserves enough space in res for about half of the set bits in mask to avoid repeat reallocation of space for res. This space reservation will likely increase memory usage.

References ibis::bitvector::adjustSize(), ibis::array_t< T >::capacity(), ibis::array_t< T >::clear(), ibis::bitvector::clear(), ibis::bitvector::cnt(), ibis::bitvector::copy(), ibis::gVerbose, name(), ibis::array_t< T >::nosharing(), ibis::array_t< T >::push_back(), ibis::array_t< T >::reserve(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T , typename F1 , typename F2 >

long ibis::part::doComp	(	const array_t< T > &	vals,
		F1	cmp1,
		F2	cmp2,
		const ibis::bitvector &	mask,
		array_t< T > &	res
	)		`[static, protected]`

Evaluate the range condition.

The actual comparison functions are only applied on rows with mask == 1. The values satisfying the comparison operators are stored in res. This function reserves enough space in res for about half of the set bits in mask to avoid repeat reallocation of space for res. This space reservation will likely increase memory usage.

References ibis::array_t< T >::capacity(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), ibis::gVerbose, name(), ibis::array_t< T >::nosharing(), ibis::array_t< T >::push_back(), ibis::array_t< T >::reserve(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T , typename F1 , typename F2 >

long ibis::part::doComp	(	const array_t< T > &	vals,
		F1	cmp1,
		F2	cmp2,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

Evaluate the range condition.

The actual comparison functions are only applied on rows with mask == 1. The actual scan function. This one chooses whether the internal bitvector for storing the scan results will be compressed or not. It always returns a compressed bitvector.

References ibis::bitvector::adjustSize(), ibis::bitvector::clear(), ibis::bitvector::cnt(), ibis::bitvector::compress(), ibis::bitvector::decompress(), ibis::gVerbose, name(), ibis::bitvector::reserve(), ibis::bitvector::set(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T , typename F >

long ibis::part::doComp0	(	const array_t< T > &	vals,
		F	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

Evaluate the range condition.

The actual comparison function is only applied on rows with mask == 1. This version uses a uncompressed bitvector to store the scan results internally.

References ibis::bitvector::cnt(), ibis::bitvector::compress(), ibis::bitvector::decompress(), ibis::gVerbose, name(), ibis::bitvector::set(), ibis::array_t< T >::size(), ibis::bitvector::size(), and ibis::bitvector::turnOnRawBit().

Referenced by doScan().

template<typename T , typename F1 , typename F2 >

long ibis::part::doComp0	(	const array_t< T > &	vals,
		F1	cmp1,
		F2	cmp2,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

This version uses uncompressed bitvector to store the scan results internally.

References ibis::bitvector::cnt(), ibis::bitvector::compress(), ibis::bitvector::decompress(), ibis::gVerbose, name(), ibis::bitvector::set(), ibis::array_t< T >::size(), ibis::bitvector::size(), and ibis::bitvector::turnOnRawBit().

template<typename T >

long ibis::part::doCompare	(	const array_t< T > &	array,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::array_t< T > &	res,
		ibis::bitvector &	hits
	)		`[static, protected]`

The function that performs the actual comparison for range queries.

The size of array may either match the number of bits in mask or the number of set bits in mask. This allows one to either use the whole array or the only the elements need for this operation. The values satisfying the query condition is stored in res.

The return value is the number of elements in res or a negative number to indicate error.

References ibis::array_t< T >::capacity(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), ibis::qRange::colName(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::qRange::inRange(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), ibis::array_t< T >::nosharing(), ibis::array_t< T >::push_back(), ibis::horometer::realTime(), ibis::array_t< T >::reserve(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

template<typename T >

long ibis::part::doCompare	(	const char *	file,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::array_t< T > &	res,
		ibis::bitvector &	hits
	)		`[static, protected]`

Perform comparisons with data in the named file.

Place the values satisfying the specified condition into res. This function attempts to allocate a buffer so the reading can be done in relatively large-size chunks. If it is unable to allocate a useful buffer, it will read one value at a time.

template<typename T >

long ibis::part::doCompare	(	const array_t< T > &	array,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::array_t< T > &	res
	)		`[static, protected]`

The function that performs the actual comparison for range queries.

The size of array may either match the number of bits in mask or the number of set bits in mask. This allows one to either use the whole array or the only the elements need for this operation. The values satisfying the query condition is stored in res.

The return value is the number of elements in res or a negative number to indicate error.

References ibis::array_t< T >::capacity(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), ibis::qRange::colName(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::qRange::inRange(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), ibis::array_t< T >::nosharing(), ibis::array_t< T >::push_back(), ibis::horometer::realTime(), ibis::array_t< T >::reserve(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

template<typename T >

long ibis::part::doCompare	(	const char *	file,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::array_t< T > &	res
	)		`[static, protected]`

Perform comparisons with data in the named file.

Place the values satisfying the specified condition into res. This function attempts to allocate a buffer so the reading can be done in relatively large-size chunks. If it is unable to allocate a useful buffer, it will read one value at a time.

References ibis::fileManager::buffer< T >::address(), ibis::array_t< T >::capacity(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), ibis::qRange::colName(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::qRange::inRange(), ibis::fileManager::instance(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), ibis::array_t< T >::nosharing(), ibis::array_t< T >::push_back(), ibis::horometer::realTime(), ibis::fileManager::recordPages(), ibis::array_t< T >::reserve(), ibis::array_t< T >::size(), ibis::fileManager::buffer< T >::size(), ibis::horometer::start(), ibis::horometer::stop(), and UnixOpen.

template<typename T >

long ibis::part::doCompare	(	const array_t< T > &	array,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

The function that performs the actual comparison for range queries.

The size of array may either match the number of bits in mask or the number of set bits in mask. This allows one to either use the whole array or the only the elements need for this operation. In either case, only mask.cnt() elements of array are checked but position of the bits that need to be set in the output bitvector hits have to be handled differently.

template<typename T >

template long ibis::part::doCompare< signed char >	(	const char *	file,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

Perform comparisons with data in the named file.

It attempts to allocate a buffer so the reading can be done in relatively large-size chunks. If it is unable to allocate a useful buffer, it will read one value at a time.

template<typename T >

long ibis::part::doCompare	(	const array_t< T > &	array,
		const ibis::qIntHod &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

The function that performs the actual comparison for range queries.

The size of array may either match the number of bits in mask or the number of set bits in mask. This allows one to either use the whole array or the only the elements need for this operation. In either case, only mask.cnt() elements of array are checked but position of the bits that need to be set in the output bitvector hits have to be handled differently.

template<typename T >

long ibis::part::doCompare	(	const char *	file,
		const ibis::qIntHod &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

Evaluate the range condition.

The actual comparison function is only applied on rows with mask == 1.

References ibis::fileManager::buffer< T >::address(), ibis::bitvector::adjustSize(), ibis::bitvector::clear(), ibis::bitvector::cnt(), ibis::qIntHod::colName(), ibis::bitvector::compress(), ibis::bitvector::decompress(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::qIntHod::inRange(), ibis::fileManager::instance(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), ibis::horometer::realTime(), ibis::fileManager::recordPages(), ibis::bitvector::reserve(), ibis::bitvector::set(), ibis::bitvector::setBit(), ibis::fileManager::buffer< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), ibis::horometer::stop(), and UnixOpen.

template<typename T >

long ibis::part::doCompare	(	const array_t< T > &	array,
		const ibis::qUIntHod &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static, protected]`

The function that performs the actual comparison for range queries.

The size of array may either match the number of bits in mask or the number of set bits in mask. This allows one to either use the whole array or the only the elements need for this operation. In either case, only mask.cnt() elements of array are checked but position of the bits that need to be set in the output bitvector hits have to be handled differently.

long ibis::part::doCount< double > ( const ibis::qRange & cmp ) const [protected]

Count the number rows satisfying the range expression.

A specialization of template part::doCount for double values.

A specialization of template part::doCount for float values.

Note that the comparison are performed as doubles.

References ibis::bitvector::adjustSize(), ibis::qRange::colName(), getColumn(), ibis::column::getNullMask(), ibis::qExpr::getType(), ibis::column::getValuesArray(), ibis::qContinuousRange::leftBound(), ibis::qContinuousRange::rightBound(), ibis::util::round_down(), and ibis::array_t< T >::size().

template<typename T >

long ibis::part::doCount	(	const array_t< T > &	vals,
		const ibis::qIntHod &	cmp,
		const ibis::bitvector &	mask
	)		const `[protected]`

Count the number rows satisfying the range expression.

T is an elementary type.

References ibis::qIntHod::inRange().

template<typename T >

long ibis::part::doCount	(	const array_t< T > &	vals,
		const ibis::qUIntHod &	cmp,
		const ibis::bitvector &	mask
	)		const `[protected]`

Count the number rows satisfying the range expression.

T is an elementary type.

References ibis::qUIntHod::inRange().

template<typename T >

long ibis::part::doCount	(	const array_t< T > &	vals,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask
	)		const `[protected]`

Count the number rows satisfying the range expression.

T is an elementary type.

References ibis::qRange::inRange().

long ibis::part::doScan	(	const ibis::qRange &	cmp,
		ibis::bitvector &	hits
	)		const `[virtual]`

Evaluate the range condition.

Scan the base data to resolve the range condition. Without a user specified mask, all non-NULL values are examined.

References ibis::bitvector::clear(), ibis::qRange::colName(), ibis::column::getNullMask(), ibis::gVerbose, and ibis::bitvector::size().

Referenced by ibis::bord::column::evaluateRange().

long ibis::part::doScan	(	const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		const `[virtual]`

Evalute the range condition on the records that are marked 1 in the mask.

The i'th element of the column is examined if mask[i] is set (mask[i] == 1).

References ibis::bitvector::adjustSize(), ibis::BYTE, ibis::CATEGORY, ibis::bitvector::cnt(), ibis::qRange::colName(), ibis::column::dataFileName(), ibis::DOUBLE, ibis::column::elementSize(), ibis::column::estimateRange(), ibis::FLOAT, ibis::util::getFileSize(), ibis::column::getRawData(), ibis::qExpr::getType(), ibis::gVerbose, ibis::INT, ibis::LONG, ibis::bitvector::set(), ibis::bitvector::setBit(), ibis::SHORT, ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::TEXT, ibis::column::type(), ibis::TYPESTRING, ibis::UBYTE, ibis::UINT, ibis::ULONG, and ibis::USHORT.

long ibis::part::doScan	(	const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		void *	res
	)		const `[virtual]`

Evalute the range condition and record the values satisfying the condition in res.

The tests are only performed on the records that are marked 1 in the mask (mask[i] == 1). This function only works for integers and floating-point numbers.

References ibis::BYTE, ibis::bitvector::cnt(), ibis::qRange::colName(), ibis::column::dataFileName(), ibis::DOUBLE, ibis::column::elementSize(), ibis::FLOAT, ibis::util::getFileSize(), ibis::column::getRawData(), ibis::qExpr::getType(), ibis::gVerbose, ibis::INT, ibis::LONG, ibis::column::name(), ibis::SHORT, ibis::bitvector::size(), ibis::column::type(), ibis::TYPESTRING, ibis::UBYTE, ibis::UINT, ibis::ULONG, and ibis::USHORT.

long ibis::part::doScan	(	const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		void *	res,
		ibis::bitvector &	hits
	)		const `[virtual]`

Evalute the range condition and record the values satisfying the condition in res.

The tests are only performed on the records that are marked 1 in the mask (mask[i] == 1). This function only works for integers and floating-point numbers.

References ibis::BYTE, ibis::bitvector::cnt(), ibis::qRange::colName(), ibis::column::dataFileName(), ibis::DOUBLE, ibis::column::elementSize(), ibis::FLOAT, ibis::util::getFileSize(), ibis::qExpr::getType(), ibis::gVerbose, ibis::fileManager::instance(), ibis::INT, ibis::LONG, ibis::column::name(), ibis::SHORT, ibis::bitvector::size(), ibis::column::type(), ibis::TYPESTRING, ibis::UBYTE, ibis::UINT, ibis::ULONG, and ibis::USHORT.

long ibis::part::doScan	(	const ibis::math::term &	trm,
		const ibis::bitvector &	msk,
		ibis::bitvector &	res
	)		const `[virtual]`

Treat the arithmetic expression as true or false.

The arithmetic expression is evaluated, nonzero values are treated as true and others are treated as false. This function only uses the test 'eval() != 0', which will treat all NaN as false.

References ibis::bitvector::adjustSize(), ibis::bitvector::clear(), ibis::bitvector::cnt(), ibis::bitvector::copy(), ibis::math::term::eval(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), ibis::part::barrel::open(), ibis::part::barrel::read(), ibis::horometer::realTime(), ibis::math::barrel::recordVariable(), ibis::part::barrel::seek(), ibis::bitvector::set(), ibis::bitvector::setBit(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::doScan	(	const ibis::compRange &	cmp,
		ibis::bitvector &	hits
	)		const `[virtual]`

Sequential scan without a mask.

It assumes that every valid row is to be examined.

References ibis::bitvector::set().

long ibis::part::doScan	(	const ibis::compRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		const `[virtual]`

Locate the records that have mark value 1 and satisfy the complex range conditions.

This implementation uses ibis::part::barrel for handling actual values needed.

References ibis::bitvector::clear(), ibis::bitvector::cnt(), ibis::bitvector::compress(), ibis::bitvector::copy(), ibis::bitvector::decompress(), ibis::qExpr::getLeft(), ibis::qExpr::getRight(), ibis::gVerbose, ibis::bitvector::indexSet::indices(), ibis::compRange::inRange(), ibis::bitvector::indexSet::isRange(), ibis::bitvector::indexSet::nIndices(), ibis::part::barrel::open(), ibis::part::barrel::read(), ibis::horometer::realTime(), ibis::math::barrel::recordVariable(), ibis::bitvector::reserve(), ibis::part::barrel::seek(), ibis::bitvector::set(), ibis::bitvector::setBit(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

template<typename E >

long ibis::part::doScan	(	const array_t< E > &	varr,
		const ibis::qRange &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)		`[static]`

Locate the records that satisfy the range condition.

A generic scan function that rely on the virtual function ibis::range::inRange. This static member function works on an array provided by the caller. Since the values are provided, this function does not check the name of the variable involved in the range condition.

Note:: The incoming values in varr is an ibis::array_t type. This limits the type of data supported by this function.

References ibis::bitvector::adjustSize(), ibis::bitvector::cnt(), ibis::gVerbose, ibis::qRange::inRange(), ibis::horometer::realTime(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

template<typename T >

long ibis::part::doScan	(	const array_t< T > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)

Evalue the range condition on the in memory values.

This static member function works on an array that is provided by the caller. Since the values are provided, this function does not check the name of the variable involved in the range condition.

References ibis::bitvector::cnt(), doComp(), doComp0(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), name(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::util::round_down(), ibis::bitvector::set(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::doScan	(	const array_t< float > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)

Examine the range condition with in memory values.

A specialization of the template for float arrays. All comparisons are performed as doubles.

References ibis::bitvector::cnt(), doComp(), doComp0(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::bitvector::set(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::doScan	(	const array_t< double > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		ibis::bitvector &	hits
	)

Examine the range condition with in memory values.

A specialization of the template for double values.

References ibis::bitvector::cnt(), doComp(), doComp0(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::bitvector::set(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

template<typename T >

long ibis::part::doScan	(	const array_t< T > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		array_t< T > &	res
	)

Evalue the range condition on the in memory values.

This static member function works on integer data provided by the caller. Since the values are provided, this function does not check the name of the variable involved in the range condition.

References ibis::array_t< T >::clear(), ibis::bitvector::cnt(), doComp(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), name(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::util::round_down(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

template<typename T >

long ibis::part::doScan	(	const array_t< T > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		array_t< T > &	res,
		ibis::bitvector &	hits
	)

Evalue the range condition on the in memory values.

This static member function works on integer data provided by the caller. Since the values are provided, this function does not check the name of the variable involved in the range condition.

References ibis::bitvector::adjustSize(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), doComp(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), name(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::util::round_down(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::doScan	(	const array_t< float > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		array_t< float > &	res
	)

Examine the range condition with in memory values.

A specialization of the template for float arrays. All comparisons are performed as doubles.

References ibis::array_t< T >::clear(), ibis::bitvector::cnt(), doComp(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::doScan	(	const array_t< double > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		array_t< double > &	res
	)

Examine the range condition with in memory values.

A specialization of the template for double values.

References ibis::array_t< T >::clear(), ibis::bitvector::cnt(), doComp(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::doScan	(	const array_t< float > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		array_t< float > &	res,
		ibis::bitvector &	hits
	)

Examine the range condition with in memory values.

A specialization of the template for float arrays. All comparisons are performed as doubles.

References ibis::bitvector::adjustSize(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), doComp(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::doScan	(	const array_t< double > &	vals,
		const ibis::qContinuousRange &	rng,
		const ibis::bitvector &	mask,
		array_t< double > &	res,
		ibis::bitvector &	hits
	)

Examine the range condition with in memory values.

A specialization of the template for double values.

References ibis::bitvector::adjustSize(), ibis::array_t< T >::clear(), ibis::bitvector::cnt(), doComp(), ibis::gVerbose, ibis::qContinuousRange::leftBound(), ibis::horometer::realTime(), ibis::qContinuousRange::rightBound(), ibis::array_t< T >::size(), ibis::bitvector::size(), ibis::horometer::start(), and ibis::horometer::stop().

void ibis::part::emptyCache ( ) const

Empty all unused resources in cache.

This function attempts to unload all the indexes and then remove all unused files from the file manager.

References ibis::fileManager::flushDir(), and ibis::fileManager::instance().

Referenced by ibis::query::evaluate(), and ibis::tafel::write().

void ibis::part::equalWeightBins	(	const array_t< float > &	vals,
		uint32_t	nbins,
		array_t< float > &	bounds
	)

Explicit specialization for float arrays.

Goes through the data twice, once to find the actual min and max values, and once to place the values in ten times as many bins as desired. It then coalesces the finer bins into desired number of bins.

References ibis::array_t< T >::clear(), ibis::util::compactValue(), ibis::util::compactValue2(), ibis::index::divideCounts(), ibis::array_t< T >::push_back(), ibis::array_t< T >::reserve(), ibis::array_t< T >::resize(), and ibis::array_t< T >::size().

void ibis::part::equalWeightBins	(	const array_t< double > &	vals,
		uint32_t	nbins,
		array_t< double > &	bounds
	)

Explicit specialization for double arrays.

Goes through the data twice, once to find the actual min and max values, and once to place the values in ten times as many bins as desired. It then coalesces the finer bins into desired number of bins.

References ibis::array_t< T >::clear(), ibis::util::compactValue(), ibis::util::compactValue2(), ibis::index::divideCounts(), ibis::array_t< T >::push_back(), ibis::array_t< T >::reserve(), ibis::array_t< T >::resize(), and ibis::array_t< T >::size().

long ibis::part::estimateMatchAny	(	const ibis::qAnyAny &	cmp,
		ibis::bitvector &	low,
		ibis::bitvector &	high
	)		const `[virtual]`

Estimate a lower bound and an upper bound on the records that are hits.

The bitvector low contains records that are hits (for sure) and the bitvector high contains records that are possible hits.

References ibis::bitvector::set(), ibis::array_t< T >::size(), and ibis::bitvector::size().

long ibis::part::estimateRange	(	const ibis::qContinuousRange &	cmp,
		ibis::bitvector &	low,
		ibis::bitvector &	high
	)		const `[virtual]`

Estimate a continuous range condition.

Return sure hits in bitvector low, and sure hits plus candidates in bitvector high.

An alternative view is that low and high represent an lower bound and an upper bound of the actual hits.

References ibis::bitvector::cnt(), ibis::qContinuousRange::colName(), ibis::column::estimateRange(), ibis::gVerbose, ibis::bitvector::set(), and ibis::bitvector::size().

int64_t ibis::part::evaluateJoin	(	const ibis::deprecatedJoin &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector64 &	pairs
	)		const `[inline]`

Evaluate a self-join.

Return the number of pairs satisfying join condition. Only records marked with mask=1 are considered. The result pairs are stored in the bitvector pairs. A pair <i, j> would be marked at position i*nRows() + j in pairs.

int64_t ibis::part::evaluateJoin	(	const ibis::deprecatedJoin &	cmp,
		const ibis::bitvector &	mask,
		const char *	pairfile
	)		const `[inline]`

Return the number of pairs satisfying the join condition.

In addition, write the pairs into the file named pairfile.

int64_t ibis::part::evaluateJoin	(	const std::vector< const ibis::deprecatedJoin * > &	cmp,
		const ibis::bitvector &	mask,
		ibis::bitvector64 &	pairs
	)		const `[inline]`

Evaluate a join defined with multiple (conjunctive) range join conditions.

int64_t ibis::part::evaluateJoin	(	const ibis::deprecatedJoin &	cmp,
		const ibis::bitvector64 &	trial,
		ibis::bitvector64 &	result
	)		const

Evaluate all pairs in trial to determine whether they really satisfy the range join defined in cmp.

The result is stored in the argument result. This function returns the number of hits found.

References ibis::bitvector64::cnt(), ibis::math::term::eval(), ibis::bitvector64::set(), and ibis::bitvector64::size().

int64_t ibis::part::evaluateJoin	(	const std::vector< const ibis::deprecatedJoin * > &	cmp,
		const ibis::bitvector64 &	trial,
		ibis::bitvector64 &	result
	)		const

Check a set of pairs defined in trial.

This version works on multiple (conjunctive) join conditions.

References ibis::bitvector64::adjustSize(), ibis::bitvector64::clear(), ibis::bitvector64::cnt(), ibis::horometer::CPUTime(), ibis::gVerbose, ibis::part::barrel::open(), ibis::part::barrel::read(), ibis::horometer::realTime(), ibis::math::barrel::recordVariable(), ibis::part::barrel::seek(), ibis::bitvector64::set(), ibis::bitvector64::setBit(), ibis::bitvector64::size(), ibis::horometer::start(), and ibis::horometer::stop().

long ibis::part::evaluateRIDSet	(	const ibis::RIDSet &	in,
		ibis::bitvector &	hits
	)		const

Convert a list of RIDs into a bitvector.

If an list of external RIDs is available, sort those RIDS and search through them, otherwise, assume the incoming numbers are row numbers and mark the corresponding positions of hits.

References ibis::bitvector::adjustSize(), ibis::bitvector::cnt(), ibis::gVerbose, ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::util::sortRIDs().

Referenced by ibis::query::limit().

template<typename T1 >

long ibis::part::fill1DBins	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals,
		const double &	begin,
		const double &	end,
		const double &	stride,
		std::vector< ibis::bitvector > &	bins
	)		const `[protected]`

Fill the bitvectors representing the 1D bins.

The number of bins defined by the given (begin, end, stride)-triplet is 1 + floor((end-begin)/stride) , with the following bin boundaries,.

 [begin, begin+stride)
 [begin+stride, begin+stride*2)
 ...
 [begin+stride*floor((end-begin)/stride), end].

This function detects two error conditions. It returns -11 to indicate that mask and the number of values do not match. Normally, the number of elements in vals is either mask.size() or mask.cnt(). It returns -10 if the triplet (begin, end, stride) does not define a valid set of bins or defines more than 1 billion bins. Upon successful completion of this function, the return value is the number of bins, i.e. bins.size().

Note:: All bitmaps that are empty are left with size() = 0. All other bitmaps have the same size() as mask.size(). When use these returned bitmaps, please make sure to NOT mix empty bitmaps with non-empty bitmaps in bitwise logical operations!

References ibis::bitvector::cnt(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 >

long ibis::part::fill1DBins	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals,
		const double &	begin,
		const double &	end,
		const double &	stride,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

Fill the bitvectors representing the 1D bins.

This version returns a vector of pointers to bitvectors.

It can reduce the memory usage and reduce execution time if the majority of the bins are empty.

References ibis::bitvector::cnt(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 >

long ibis::part::fill1DBinsWeighted	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals,
		const double &	begin,
		const double &	end,
		const double &	stride,
		const array_t< double > &	wts,
		std::vector< double > &	weights,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

Fill the bitvectors representing the 1D bins.

Mark the positions of records falling in each bin and computed the total weight in each bins.

This version returns a vector of pointers to bitvectors. It can reduce the memory usage and reduce execution time if the majority of the bins are empty.

Note:: Assumes wts.size() == vals.size().

References ibis::bitvector::cnt(), ibis::array_t< T >::resize(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 , typename T2 >

long ibis::part::fill2DBins	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	vals2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		std::vector< ibis::bitvector > &	bins
	)		const `[protected]`

Fill the bitvectors representing the 2D bins.

The pair of triplets, (begin1, end1, stride1) and (begin2, end2, stride2) define (1 + floor((end1-begin1)/stride1)) (1 + floor((end2-begin2)/stride2)) 2D bins.

The 2D bins are packed into the 1D array bins in raster scan order, with the second dimension as the faster varying dimensioin.

Note:: All bitmaps that are empty are left with size() = 0. All other bitmaps have the same size() as mask.size(). When use these returned bitmaps, please make sure to NOT mix empty bitmaps with non-empty bitmaps in bitwise logical operations!

See also:: ibis::part::file1DBins.

References ibis::bitvector::cnt(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 , typename T2 >

long ibis::part::fill2DBins	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	vals2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

Fill the bitvectors representing the 2D bins.

This version returns a vector of pointers to bitmaps.

Because the empty bitmaps are left as null pointers, it can reduce the memory usage and the execution time if the majority of the bins are empty.

References ibis::bitvector::cnt(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 >

long ibis::part::fill2DBins2	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	val1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const ibis::column &	col2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		std::vector< ibis::bitvector > &	bins
	)		const `[protected]`

A template function to resolve the second variable involved in the 2D bins.

The actual binning work done in ibis::part::fill2DBins.

template<typename T1 >

long ibis::part::fill2DBins2	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	val1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const ibis::column &	col2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

This version returns a vector of pointers to bitmaps.

Because the empty bitmaps are left as null pointers, it can reduce the memory usage and the execution time if the majority of the bins are empty.

template<typename T1 , typename T2 >

long ibis::part::fill2DBinsWeighted	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	vals2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const array_t< double > &	wts,
		std::vector< double > &	weights,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

Fill the bitvectors representing the 2D bins.

This version returns a vector of pointers to bitmaps.

Because the empty bitmaps are left as null pointers, it can reduce the memory usage and the execution time if the majority of the bins are empty.

References ibis::bitvector::cnt(), ibis::array_t< T >::resize(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 >

long ibis::part::fill2DBinsWeighted2	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	val1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const ibis::column &	col2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const array_t< double > &	wts,
		std::vector< double > &	weights,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

This version returns a vector of pointers to bitmaps.

Because the empty bitmaps are left as null pointers, it can reduce the memory usage and the execution time if the majority of the bins are empty.

template<typename T1 , typename T2 , typename T3 >

long ibis::part::fill3DBins	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	vals2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const array_t< T3 > &	vals3,
		const double &	begin3,
		const double &	end3,
		const double &	stride3,
		std::vector< ibis::bitvector > &	bins
	)		const `[protected]`

Fill the bitvectors representing the 3D bins.

The three triplets, (begin1, end1, stride1), (begin2, end2, stride2), and (begin3, end3, stride3), defines (1 + floor((end1 - begin1) / stride1)) (1 + floor((end2 - begin2) / stride2)) (1 + floor((end3 - begin3) / stride3)) 3D bins.

The 3D bins are packed into the 1D array bins in raster scan order, with the 3rd dimension as the fastest varying dimension and the 1st dimension as the slowest varying dimension.

Note:: All bitmaps that are empty are left with size() = 0. All other bitmaps have the same size() as mask.size(). When use these returned bitmaps, please make sure to NOT mix empty bitmaps with non-empty bitmaps in bitwise logical operations!

See also:: ibis::part::fill1DBins, ibis::part::fill2DBins.

References ibis::bitvector::cnt(), ibis::gVerbose, name(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 , typename T2 , typename T3 >

long ibis::part::fill3DBins	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	vals2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const array_t< T3 > &	vals3,
		const double &	begin3,
		const double &	end3,
		const double &	stride3,
		std::vector< bitvector * > &	bins
	)		const `[protected]`

The three triplets, (begin1, end1, stride1), (begin2, end2, stride2), and (begin3, end3, stride3), defines (1 + floor((end1 - begin1) / stride1)) (1 + floor((end2 - begin2) / stride2)) (1 + floor((end3 - begin3) / stride3)) 3D bins.

The 3D bins are packed into the 1D array bins in raster scan order, with the 3rd dimension as the fastest varying dimension and the 1st dimension as the slowest varying dimension.

Note:: All bitmaps that are empty are left as nil (0). All other bitmaps have the same size() as mask.size(). When use these returned bitmaps, please make sure to NOT mix empty bitmaps with non-empty bitmaps in bitwise logical operations!

See also:: ibis::part::fill1DBins, ibis::part::fill2DBins.

References ibis::bitvector::cnt(), ibis::gVerbose, name(), ibis::bitvector::setBit(), ibis::array_t< T >::size(), and ibis::bitvector::size().

template<typename T1 >

long ibis::part::fill3DBins2	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	val1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const ibis::column &	col2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const ibis::column &	col3,
		const double &	begin3,
		const double &	end3,
		const double &	stride3,
		std::vector< bitvector > &	bins
	)		const `[protected]`

Resolve the 2nd column of the 3D bins.

It invokes ibis::part::fill3DBins3 to resolve the 3rd dimension and finally ibis::part::fill3DBins to perform the actual binning.

template<typename T1 >

long ibis::part::fill3DBins2	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	val1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const ibis::column &	col2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const ibis::column &	col3,
		const double &	begin3,
		const double &	end3,
		const double &	stride3,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

Resolve the 2nd column of the 3D bins.

It invokes ibis::part::fill3DBins3 to resolve the 3rd dimension and finally ibis::part::fill3DBins to perform the actual binning.

template<typename T1 , typename T2 >

long ibis::part::fill3DBins3	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	val1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	val2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const ibis::column &	col3,
		const double &	begin3,
		const double &	end3,
		const double &	stride3,
		std::vector< bitvector > &	bins
	)		const `[protected]`

Resolve the 3rd column involved in the 3D bins.

The finally binning work is performed by ibis::part::fill3DBins.

template<typename T1 , typename T2 >

long ibis::part::fill3DBins3	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	val1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	val2,
		const double &	begin2,
		const double &	end2,
		const double &	stride2,
		const ibis::column &	col3,
		const double &	begin3,
		const double &	end3,
		const double &	stride3,
		std::vector< ibis::bitvector * > &	bins
	)		const `[protected]`

Resolve the 3rd column involved in the 3D bins.

The finally binning work is performed by ibis::part::fill3DBins.

template<typename T1 , typename T2 , typename T3 >

long ibis::part::fill3DBinsWeighted	(	const ibis::bitvector &	mask,
		const array_t< T1 > &	vals1,
		const double &	begin1,
		const double &	end1,
		const double &	stride1,
		const array_t< T2 > &	vals2,
		const double &	begin2,

Classes

Public Types

Public Member Functions

Static Public Member Functions

Protected Member Functions

Static Protected Member Functions

Protected Attributes

Friends

Detailed Description

Constructor & Destructor Documentation

Member Function Documentation