dataset.h

Go to the documentation of this file.

/* Copyright (C) 2017-2018 Chris Piker <chris-piker@uiowa.edu>
 *
 * This file is part of das2C, the Core Das2 C Library.
 *
 * das2C is free software; you can redistribute it and/or modify it under
 * the terms of the GNU Lesser General Public License version 2.1 as published
 * by the Free Software Foundation.
 *
 * das2C is distributed in the hope that it will be useful, but WITHOUT ANY
 * WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for
 * more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * version 2.1 along with das2C; if not, see <http://www.gnu.org/licenses/>.
 */
 
 
#ifndef _das_dataset_h_
#define _das_dataset_h_
 
#include <das2/dimension.h>
#include <das2/codec.h>
 
#ifdef __cplusplus
extern "C" {
#endif
 
/* Old initial comment that kicked off the entire das2 data model design...
 *
 * The structures below are the start of an idea on how to get independent
 * parameters for data at any particular index.  These are just thoughts
 * at the moment and don't affect any working code.  There are many ways
 * to do this.  The CDF and QStream assumption is that there are the same
 * number of parameters locating a data point in parameter space as there
 * are indices to the dataset.  Because of this x,y,z scatter data are 
 * hard to handle.
 *  
 * For x,y,z scatter lists there is 1 index for any point in the dataset, 
 * but for each index there are 2 independent parameters.  Basically QStream
 * and CDF assume that all datasets are CUBEs in parameter space but this
 * is not the case for a great many sets. 
 *
 * To adequately handle these 'path' datasets a parameter map is required.
 * The mapping takes 1 index value per data rank and returns 1 to N parameter
 * values.
 *
 * These structures start to handle this idea but are just doodles at this
 * point. -cwp 2017-07-25
 */
 
/* Second comment that added desire for flexible data types ...
 * 
 * Thinking about coordinate returns, how about a data set of thefts / month
 * in 5 cities...  Won't usually come up, but should be possible
 * to handle.  Here's the data set:
 *
 *              2016-01 2016-02 2016-03 2016-04 2016-05 2016-06
 * Baltimore       2351    3789    4625    5525    6135    5902
 * Bogotaà      109065  110365   99625   98265   43850   33892
 * Chicago         4789    5764    8901   10145   13456   22678
 * Des Moines         4      10      33      35      44     107
 *
 * Properties:  Title -> "Thefts/Month for selected cities"
 *
 * Okay, the X axis data type is text[12] (need null char)
 *           Y axis data type is datetime
 *           Z axis data type is datum, "thefts month**-1"
 *
 * So what is the return value from pDs->bin(pDs, 0, 0) ?
 *
 * The bin is defined on the space of all UTC times, and on the space of all
 * cities in the data set.
 *
 *
 * So, what about this common data set, interference events:
 *
 * |<----------     Bin      ------>|  |<----- Value --->|
 * 2016-01-01T14:00  2016-01-02T02:20  Mag Roll
 * 2016-01-01T15:40  2016-01-01T15:41  Stabilization Pulse
 * 2016-01-01T15:43  2016-01-01T15:44  Stabilization Pulse
 * 2016-01-01T15:45  2016-01-01T15:47  Stabilization Pulse
 * 2016-01-01T15:48  2016-01-01T15:50  Stabilization Pulse
 *
 * So what is the return value from pDs->bin(pDs, 0) ?
 *
 * The space is UTC time,  So each bin start and stop is defined on the space
 * of all UTC times.
 * -cwp 2017-??-??
 */
 
/* Number of encoders that can be stored internally, more then this and they
 * have to be allocated on the heap.  This is the common "small vector" 
 * optimization */
#define DASDS_LOC_ENC_SZ 32
 
typedef struct dataset {
   DasDesc base;        /* This would be equivalent to the properties for 
                           a packet descriptor.  Typically in das 2.2 packets
                           don't have a descriptor, only streams and planes
                           but access to the stream descriptor forwards through
                           here. */
   
   int nRank;           /* The number of whole-dataset index dimenions. 
                         * Variables can define internal dimensions but they
                         * can't use indices in the first nRank positions for
                         * internal use, as these are used to correlate values
                         * across the dataset. */
   
   /* A text identifier for this instance of a data set */
   char sId[DAS_MAX_ID_BUFSZ];
   
   /* A text identifier for the join group for this dataset.  Datasets with
    * the same groupID should be joined automatically by display clients. 
    */
   char sGroupId[DAS_MAX_ID_BUFSZ];
                        
   size_t uDims;        /* Number of dimensions, das datasets are 
                          * implicitly bundles in qdataset terms. */
   
   DasDim** lDims;      /* The data variable object arrays */
   size_t uSzDims;      /* Current size of dimension array */
   
   size_t uArrays;      /* The number of low-level arrays */
   DasAry** lArrays;    /* An array of array objects */
   size_t uSzArrays;
   
   ptrdiff_t _shape[DASIDX_MAX];  /* cache shape calls for speed */
   
   bool _dynamic;      /* If true, the dataset may still be changing and all
                          bulk properties such as the iteration shape should be
                          recalculated instead of using cached values. 
                          If false, cached values are expected to already be 
                          available */
 
   /* dataset arrays can be written in chunks to output buffers. The number of
    * elements in each chuck, the encoding of each element any separators are
    * defined below. */
   /* DasCodec** lEncs; */
 
   /* Use a fixed size for now */
   size_t uSzEncs;
   DasCodec aPktEncs[DASDS_LOC_ENC_SZ];
   int nPktItems[DASDS_LOC_ENC_SZ];
 
   void* pUser;
 
} DasDs;
 
DAS_API DasDs* new_DasDs(
   const char* sId, const char* sGroupId, int nRank
);
 
DAS_API void del_DasDs(DasDs* pThis);
 
DAS_API void DasDs_setMutable(DasDs* pThis, bool bChangeAllowed);
 
 
#define DasDs_mutable(P) P->_mutable
 
DAS_API int DasDs_shape(const DasDs* pThis, ptrdiff_t* pShape);
 
DAS_API ptrdiff_t DasDs_lengthIn(const DasDs* pThis, int nIdx, ptrdiff_t* pLoc);
 
typedef struct dasds_iterator_t{
   
   bool       done;
   
   ptrdiff_t index[DASIDX_MAX];
   
   int        rank;
   ptrdiff_t  shape[DASIDX_MAX];  /* Used for CUBIC datasets */
   ptrdiff_t  nLenIn;            /* Used for ragged datasets */
   bool      ragged;
   const DasDs* pDs;
} dasds_iterator;
 
DAS_API void dasds_iter_init(dasds_iterator* pIter, const DasDs* pDs);
 
DAS_API bool dasds_iter_next(dasds_iterator* pIter);
 
 
#define DasDs_group(P) ((const char*)(P)->sGroupId)
 
#define DasDs_id(P) ((const char*)(P)->sId)
 
 
#define DasDs_rank(P) ((P)->nRank)
 
DAS_API DasErrCode DasDs_addAry(DasDs* pThis, DasAry* pAry);
 
 
#define DasDs_numAry(P) ((P)->uArrays)
 
#define DasDs_getAry(P, I) ((P)->lArrays[(I)])
 
 
DAS_API DasAry* DasDs_getAryById(DasDs* pThis, const char* sAryId);
 
DAS_API size_t DasDs_memUsed(const DasDs* pThis);
 
DAS_API size_t DasDs_memIndexed(const DasDs* pThis);
 
DAS_API size_t DasDs_memOwned(const DasDs* pThis);
 
 
DAS_API DasErrCode DasDs_addFixedCodec(
   DasDs* pThis, const char* sAryId, const char* sSemantic, 
   const char* sEncType, int nItemBytes, int nNumItems
);
 
DAS_API DasErrCode DasDs_addRaggedCodec(
   DasDs* pThis, const char* sAryId, const char* sEncType, 
   int nItemBytes, int nSeps, ubyte uSepLen, const ubyte* pSepByIdx
);
 
DAS_API size_t DasDs_clearRagged0Arrays(DasDs* pThis);
 
 
DAS_API DasDim* DasDs_makeDim(
    DasDs* pThis, enum dim_type dType, const char* sDim, const char* sId
);
 
DAS_API DasErrCode DasDs_addDim(DasDs* pThis, DasDim* pDim);
 
DAS_API size_t DasDs_numDims(const DasDs* pThis, enum dim_type vt);
 
 
DAS_API const DasDim* DasDs_getDim(const DasDs* pThis, const char* sDim);
 
DAS_API const DasDim* DasDs_getDimByIdx(
   const DasDs* pThis, size_t idx, enum dim_type vt
);
 
DAS_API const DasDim* DasDs_getDimById(const DasDs* pThis, const char* sId);
 
 
DAS_API char* DasDs_toStr(const DasDs* pThis, char* sBuf, int nLen);
 
 
 
/* Ideas I'm still working on...
 
 
/ * The two functions below are really useful but I'll need to crack open
   a double pack of Flex and Bison to get it done so I'm punting for now. * /
   
/ * Ex Expression:  $spec_dens[i][j][k] * /
const Function* Dataset_evalDataExp(Dataset* pThis, const char* sExpression);
 
/ * Ex Expression:  $craft_alt[i][j] - 0.5 * $delay_time[k] * 299792 * /
const Function* Dataset_evalCoordExp(Dataset* pThis, const char* sExpression);
 
 
/ ** 
 * 
 * This function answers the question by either provided the spanning set of
 * coordinates or returning nothing.  
 * For a dataset to be defined
 * on a coordinate grid there must exist one coordinate set for each index in
 * the data set and each coordinate must be a function of only one index.  
 * 
 * Non-gridded data can still be sliced but coordintate slices will need to be
 * produced as well in order to plot the slice. See Dataset_orthogonal()
 * 
 * @param pThis A correlated dataset object
 * @param sDs The string id of the dataset in question
 * @param[out] psCoords a pointer to a const char* array to recived the 
 *              coordinate ID's forming the spanning set.  Note that every
 *              combination of returned coordinates satisfies the orthogonal
 *              condition and would return true from Dataset_orthogonal().
 * 
 * @return     The number of spanning coordinates.  Will be equal to the 
 *              rank of the dataset.
 * /
size_t Dataset_gridCoords(
   const Dataset* pThis, const char* sDs, const char** psCoords
);
 
bool DataGen_grid(const DataSet* pDataset);
 
const DataSet** Dg_griddedIn(const DataSet* pDataset);
 
 
 
/ ** Get the coefficients for iterating over a 1-D slice of a regular (i.e. 
 * non-ragged) dataset.
 *
 * This function dose not work for ragged datasets and merely returns NULL if
 * asked for iteration coefficents for such a set.  In such a case use 
 * Dataset_copySlice1D().
 *
 * /
const void* Dataset_slice1D(
   const Dataset* pThis, const char* sDs, const char* sCoord, int iCoordIdx,
   int* pCoeff
);
 
/ ** Increment the reference count on any array objects that are part of 
 * a data space.
 *
 * This is useful in instances where the underlying data arrays are going
 * to be represented by an organizational structure other than datasets
 * and DataSets since Das array objects only free data memory if thier
 * feference count is zero.
 *
 * @param pThis
 * /
void DataSpace_incAryRef(Dataset* pThis);
 
/ * Need a way to trigger callbacks from datasets changing, not just
   packets changing.  It could be useful to work on items from the
   dataset level instead of just the packet level * /
 bool DataSpace_stream(Dataset* pThis); * /
 
 
 
/ ** Indicate the physical degrees of freedom for a dataset by denoting a
 * complete list of coordinate sets.  
 *
 * A list of coordinates over which an entire dataset is defined is called
 * a span.  Datasets may have 1-N spans.
 * /
int DataSpace_addSpan(const char* sDsId, const char** lCoords, size_t nCoords);
*/
 
#ifdef __cplusplus
}
#endif
 
#endif /* _das_dataset_h */