Big-Data-HPC-Platform/ecl/hqlcpp/hqltcppc.ipp

/*##############################################################################

    Copyright (C) 2011 HPCC Systems.

    All rights reserved. This program is free software: you can redistribute it and/or modify
    it under the terms of the GNU Affero General Public License as
    published by the Free Software Foundation, either version 3 of the
    License, or (at your option) any later version.

    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU Affero General Public License for more details.

    You should have received a copy of the GNU Affero General Public License
    along with this program.  If not, see <http://www.gnu.org/licenses/>.
############################################################################## */
#ifndef __HQLCPPC_IPP_
#define __HQLCPPC_IPP_

#include "hqlutil.hpp"
#include "hqlhtcpp.ipp"
#include "hqltcppc.hpp"
#include "hqlcppc.hpp"

class HqlCppTranslator;

class SizeStruct
{
public:
    SizeStruct() { fixedSize = 0; varMinSize = 0; }
    SizeStruct(IHqlExpression * _self) { self.set(_self); fixedSize = 0; varMinSize = 0; }
    SizeStruct(const SizeStruct & other)                    { fixedSize = other.fixedSize; varMinSize = other.varMinSize; varSize.set(other.varSize); self.set(other.self); }

    void add(const SizeStruct & other);
    inline void addFixed(unsigned delta)                            
    { 
        fixedSize += delta; 
    }
    void addVariableExpr(unsigned _varMinSize, IHqlExpression * expr);
    void addVariable(unsigned _varMinSize, IHqlExpression * expr);
    void buildSizeExpr(HqlCppTranslator & translator, BuildCtx & ctx, BoundRow * row, CHqlBoundExpr & bound);
    void clear(IHqlExpression * _self)                      { fixedSize = 0; varSize.clear(); self.set(_self); }
    void forceToTemp(node_operator op, IHqlExpression * selector);
    unsigned getFixedSize() const                           { return fixedSize; }
    unsigned getMinimumSize()   const                       { return fixedSize+varMinSize; }
    IHqlExpression * getSizeExpr(BoundRow * cursor);
    bool isEmpty() const                                    { return fixedSize == 0 && varSize == NULL; }
    bool isFixedSize() const                                { return varSize == NULL; }
    bool isWorthCommoning() const;
    IHqlExpression * queryVarSize() const                   { return varSize; }
    IHqlExpression * querySelf() const                      { return self; }
    void set(const SizeStruct & other)                      { fixedSize = other.fixedSize; varSize.set(other.varSize); self.set(other.self); }
    void set(unsigned _fixedSize, IHqlExpression * _varSize) { fixedSize = _fixedSize; varSize.set(_varSize); }

protected:
    unsigned    fixedSize;
    unsigned    varMinSize;
    HqlExprAttr varSize;
    HqlExprAttr self;
};


//---------------------------------------------------------------------------

typedef MapOf<IHqlExpression *, AColumnInfo> ColumnToInfoMap;

class CContainerInfo;
class HQLCPP_API CMemberInfo : public AColumnInfo
{
public:
    CMemberInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//IMappingBase
    virtual const void * getKey() const { return &column; }

//AColumnInfo
    virtual void buildAddress(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildOffset(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual bool hasFixedOffset();
    virtual AColumnInfo * lookupColumn(IHqlExpression * search);
    virtual bool requiresTemp();

//null implementation
    virtual bool isConditional();

// implementation virtuals
    virtual void calcAllCachedOffsets();
            void calcCachedOffsets(const SizeStruct & offset, SizeStruct & sizeSelf);
    virtual void calcCachedSize(const SizeStruct & offset, SizeStruct & sizeSelf);
    virtual IHqlExpression * createSelectorExpr();
    virtual void gatherSize(SizeStruct & target);
            void gatherOffset(SizeStruct & target, IHqlExpression * selector);
            void getSizeExpr(SizeStruct & target);
    virtual IReferenceSelector * getSelector(BuildCtx & ctx, IReferenceSelector * parentSelector);
    virtual void noteLastBitfield() {}
    virtual IHqlExpression * queryParentSelector(IHqlExpression * selector);
    virtual IHqlExpression * queryRootSelf();
    virtual ITypeInfo * queryType() const;
    virtual unsigned getTotalFixedSize();
    virtual unsigned getTotalMinimumSize();
    virtual unsigned getContainerTrailingFixed();
    virtual bool modifyColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value, node_operator op) { return false; }

    void addVariableSize(size32_t varMinSize, SizeStruct & size);
    void getXPath(StringBuffer & out);
    StringBuffer & expandSelectPathText(StringBuffer & out, bool isLast) const;

public:
    IHqlExpression * getCondition(BuildCtx & ctx);
    IHqlExpression * getConditionSelect(HqlCppTranslator & translator, BuildCtx & ctx, BoundRow * row);
    IHqlExpression * makeConditional(HqlCppTranslator & translator, BuildCtx & ctx, BoundRow * row, IHqlExpression * value);
    void setOffset(bool _hasVarOffset);     // to avoid loads of arguments to constructor

protected:
    virtual ITypeInfo * queryPhysicalType();
    void associateSizeOf(BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * rawSize, size32_t extraSize);
    void buildConditionFilter(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    void doBuildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper, IHqlExpression * boundSize);
    void doBuildSkipInput(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper, size32_t size);
    void checkAssignOk(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * newLength, unsigned fixedExtra);
    void defaultSetColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
    void ensureTargetAvailable(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, size32_t thisSize);
    IHqlExpression * createSelfPeekDeserializer(HqlCppTranslator & translator, IHqlExpression * helper);
    IHqlExpression * getColumnAddress(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, ITypeInfo * columnType, size32_t delta = 0);
    void getColumnOffset(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & tgt);
    IHqlExpression * getColumnRef(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    void gatherMaxRowSize(SizeStruct & totalSize, IHqlExpression * newSize, size32_t fixedExtra, IReferenceSelector * selector);

    IHqlExpression * addDatasetLimits(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * _value);
    bool hasDatasetLimits() const;

protected:
    CContainerInfo *    container;
    CMemberInfo *       prior;
    HqlExprAttr         column;
    HqlExprAttr         varSize;
    SizeStruct          cachedOffset;
    SizeStruct          cachedSize;
//  SizeStruct          cachedMaxSize;
    bool                hasVarOffset;
    bool                isOffsetCached;
};


typedef CIArrayOf<CMemberInfo> CMemberInfoArray;

struct ReadAheadState
{
public:
    ReadAheadState(IHqlExpression * _helper) : helper(_helper) {};

    void addDummyMappings()
    {
        ForEachItemIn(i, requiredValues)
            mappedValues.append(OLINK(requiredValues.item(i)));
    }

    void setMapping(unsigned setIdx, IHqlExpression * expr)
    {
        mappedValues.replace(*LINK(expr), setIdx);
    }

    LinkedHqlExpr helper;
    HqlExprArray requiredValues;
    HqlExprArray mappedValues;
};

class HQLCPP_API CContainerInfo : public CMemberInfo
{
public:
    CContainerInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

// AColumnInfo
    virtual void buildSizeOf(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildClear(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, int direction);
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
    virtual IHqlExpression * createSelectorExpr();
    virtual void setRow(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IReferenceSelector * value);

//  kind of interface....
    virtual void calcAllCachedOffsets();
    virtual void calcCachedSize(const SizeStruct & offset, SizeStruct & sizeSelf);
    virtual IHqlExpression * getCondition(BuildCtx & ctx);
    virtual IHqlExpression * getRelativeSelf() = 0;
    virtual unsigned getTotalFixedSize();
    virtual unsigned getTotalMinimumSize();
    virtual bool isConditional();
    virtual bool isFixedSize()              { return fixedSize && !isDynamic; }

    void addTrailingFixed(SizeStruct & size, CMemberInfo * cur);
    void subLeadingFixed(SizeStruct & size, CMemberInfo * cur);
    void getContainerXPath(StringBuffer & out);
    inline unsigned numChildren() const     { return children.ordinality(); }
            
public:
    void addChild(CMemberInfo * child);
    void setFixedSize(bool _fixed)          { fixedSize = _fixed; }
    void setDynamic()                       { isDynamic = true; }

protected:
    virtual void registerChild(CMemberInfo * child);
    void calcCachedChildrenOffsets(SizeStruct & offset, SizeStruct & sizeSelf);
//  void gatherChildrenFixedSize(SizeStruct & target);


protected:
    CMemberInfoArray    children;
    bool                fixedSize;
    bool                isDynamic;
};


class HQLCPP_API CRecordInfo : public CContainerInfo
{
public:
    CRecordInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
    virtual void buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual AColumnInfo * lookupColumn(IHqlExpression * search);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual IHqlExpression * getRelativeSelf();
    virtual IHqlExpression * queryRootSelf();

protected:
    virtual void registerChild(CMemberInfo * child);

protected:
    ColumnToInfoMap map;
    OwnedHqlExpr cachedSelf;
};


class HQLCPP_API CIfBlockInfo : public CContainerInfo
{
public:
    CIfBlockInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
    virtual void buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void calcCachedSize(const SizeStruct & offset, SizeStruct & sizeSelf);
    virtual IHqlExpression * getCondition(BuildCtx & ctx);
    virtual bool isConditional();
    virtual bool isFixedSize()              { return false; }
    virtual IReferenceSelector * getSelector(BuildCtx & ctx, IReferenceSelector * parentSelector);
    virtual IHqlExpression * queryParentSelector(IHqlExpression * selector);
    virtual IHqlExpression * getRelativeSelf();
    virtual unsigned getTotalFixedSize();
    virtual unsigned getTotalMinimumSize();

private:
    OwnedHqlExpr condition;
    bool alwaysPresent;
};


class HQLCPP_API CColumnInfo : public CMemberInfo
{
public:
    CColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
    virtual void buildSizeOf(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildClear(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, int direction);
    virtual void buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
    virtual void setRow(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IReferenceSelector * value) { throwUnexpected(); }

//Virtuals for column implementation
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool isFixedSize();
    virtual void gatherSize(SizeStruct & target);

protected:
    virtual ITypeInfo * queryLogicalType()          { return queryType(); }
    virtual void buildColumnAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
    virtual void calcCurrentOffset(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);

    void buildDeserializeToBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IHqlCppDatasetBuilder * builder, IReferenceSelector * selector, IHqlExpression * helper);
    void buildDeserializeDatasetUsingBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual void buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper);
};

class HQLCPP_API CSpecialIntColumnInfo : public CColumnInfo
{
public:
    CSpecialIntColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
};

class HQLCPP_API CPackedIntColumnInfo : public CColumnInfo
{
public:
    CPackedIntColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual ITypeInfo * queryLogicalType()          { return queryType()->queryPromotedType(); }
    virtual void gatherSize(SizeStruct & target);
    virtual bool isFixedSize()              { return false; }
};

class HQLCPP_API CSpecialStringColumnInfo : public CColumnInfo
{
public:
    CSpecialStringColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void gatherSize(SizeStruct & target);
};

class HQLCPP_API CSpecialVStringColumnInfo : public CColumnInfo
{
public:
    CSpecialVStringColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void gatherSize(SizeStruct & target);
};

class HQLCPP_API CAlienColumnInfo : public CColumnInfo
{
public:
    CAlienColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

//Virtuals for column implementation
    virtual void buildAddress(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool isFixedSize();
    virtual void gatherSize(SizeStruct & target);

protected:
    virtual ITypeInfo * queryLogicalType()          { return queryType()->queryPromotedType(); }
    virtual ITypeInfo * queryPhysicalType();
    virtual void buildColumnAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);

protected:
    IHqlExpression * doBuildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    IHqlExpression * getAlienGetFunction(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    ITypeInfo * getPhysicalSourceType();
    unsigned getPhysicalSize();

protected:
    OwnedHqlExpr self;          // expression that represents self - different if nested.
};

class HQLCPP_API CBitfieldContainerInfo : public CContainerInfo
{
public:
    CBitfieldContainerInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
    virtual void buildClear(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, int direction);
    virtual void buildExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void calcCachedSize(const SizeStruct & offset, SizeStruct & sizeSelf);
    virtual IHqlExpression * getRelativeSelf();
    virtual IReferenceSelector * getSelector(BuildCtx & ctx, IReferenceSelector * parentSelector);
    virtual void noteLastBitfield();
    virtual IHqlExpression * queryParentSelector(IHqlExpression * selector);

public:
    CMemberInfo * lastBitfield();
    ITypeInfo * queryStorageType() const        { return column->queryType(); }

protected:
    virtual void gatherSize(SizeStruct & target);
    virtual bool isFixedSize();
};


class HQLCPP_API CBitfieldInfo : public CColumnInfo
{
public:
    CBitfieldInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

//column implementation
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual void gatherSize(SizeStruct & target);
    virtual bool isFixedSize();

//public helper functions
    virtual ITypeInfo * queryLogicalType()          { return queryType()->queryPromotedType(); }
    ITypeInfo * queryStorageType()                  { return ((CBitfieldContainerInfo *)container)->queryStorageType(); }
    void noteLastBitfield()                         { isLastBitfield = true; }
    void setBitOffset(unsigned _bitOffset);

    inline bool queryIsLastBitfield() const { return isLastBitfield; }
    inline unsigned queryBitfieldOffset() const { return bitOffset; }
    unsigned queryBitfieldPackSize() const;

private:
    unsigned            bitOffset;
    bool                isLastBitfield;
};

class HQLCPP_API CVirtualColumnInfo : public CColumnInfo
{
public:
    CVirtualColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

//Virtuals for column implementation
    virtual void buildAddress(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool isFixedSize();
    virtual void gatherSize(SizeStruct & target);

protected:
    virtual void calcCurrentOffset(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector) {}
};

class HQLCPP_API CCsvColumnInfo : public CColumnInfo
{
public:
    CCsvColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column, _ATOM _encoding);

//AColumnInfo
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

//Virtuals for column implementation
    virtual void buildAddress(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildColumnAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool isFixedSize();
    virtual void gatherSize(SizeStruct & target);

protected:
    void getName(HqlCppTranslator & translator, BuildCtx & ctx, StringBuffer & out, const char * prefix, IReferenceSelector * selector);
    IHqlExpression * getColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);

protected:
    _ATOM encoding;
};

class HQLCPP_API CXmlColumnInfo : public CColumnInfo
{
public:
    CXmlColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

//Virtuals for column implementation
    virtual void buildAddress(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);
    virtual void buildColumnAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target);
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool isFixedSize();
    virtual void gatherSize(SizeStruct & target);

protected:
    virtual void calcCurrentOffset(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector) {}
    void buildFixedStringAssign(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, const CHqlBoundTarget & target, IHqlExpression * defaultValue, _ATOM func);
    IHqlExpression * getCallExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    IHqlExpression * getXmlDatasetExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    IHqlExpression * getXmlSetExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);

};

//---------------------------------------------------------------------------

enum MapFormat { MapFormatBinary, MapFormatCsv, MapFormatXml };

class HQLCPP_API ColumnToOffsetMap : public MappingBase
{
public:
    ColumnToOffsetMap(IHqlExpression * record, unsigned _packing, unsigned _maxRecordSize, bool _translateVirtuals);

    virtual const void * getKey() const { return &record; }

    void init(RecordOffsetMap & map);
    void checkValidMaxSize();
    unsigned getFixedRecordSize();
    bool isFixedWidth() const                       { return fixedSizeRecord; }
    unsigned getMaxSize();
    bool isMaxSizeSpecified();
    unsigned numRootFields() const                  { return root.numChildren(); }
    size32_t getTotalFixedSize()                    { return root.getTotalFixedSize(); }
    size32_t getTotalMinimumSize()                  { return root.getTotalMinimumSize(); }
    virtual MapFormat getFormat()                   { return MapFormatBinary; }
    bool queryContainsIfBlock()                     { return containsIfBlock; }

    AColumnInfo * queryRootColumn();
    bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper);

protected:
    virtual CMemberInfo * addColumn(CContainerInfo * container, IHqlExpression * column, RecordOffsetMap & map);
    virtual CMemberInfo * createColumn(CContainerInfo * container, IHqlExpression * column, RecordOffsetMap & map);
    CMemberInfo * expandRecord(IHqlExpression * record, CContainerInfo * container, RecordOffsetMap & map);
    void completeActiveBitfields();
    void ensureMaxSizeCached();

protected:
    IHqlExpression * record;
    CMemberInfo * prior;
    BitfieldPacker packer;
    unsigned maxAlign;
    unsigned packing;
    unsigned defaultMaxRecordSize;
    unsigned cachedMaxSize;
    bool cachedDefaultMaxSizeUsed;
    bool fixedSizeRecord;
    bool translateVirtuals;
    bool containsIfBlock;
    CRecordInfo root;
};


class HQLCPP_API DynamicColumnToOffsetMap : public ColumnToOffsetMap
{
public:
    DynamicColumnToOffsetMap(unsigned _maxRecordSize);

    void addColumn(IHqlExpression * column, RecordOffsetMap & map);
};


class DelayedSizeGenerator : public CInterface, public IHqlDelayedCodeGenerator
{
public:
    DelayedSizeGenerator(ColumnToOffsetMap * _map)
    {
        map.set(_map);
    }
    IMPLEMENT_IINTERFACE

//IHqlDelayedCodeGenerator
    virtual void generateCpp(StringBuffer & out) { out.append(map->getTotalFixedSize()); }

public:
    Owned<ColumnToOffsetMap> map;
};


class DelayedUnsignedGenerator : public CInterface, public IHqlDelayedCodeGenerator
{
public:
    DelayedUnsignedGenerator(unsigned & _value)
    {
        value = &_value;
    }
    IMPLEMENT_IINTERFACE

//IHqlDelayedCodeGenerator
    virtual void generateCpp(StringBuffer & out) { out.append(*value); }

public:
    unsigned * value;
};


class HQLCPP_API CsvColumnToOffsetMap : public ColumnToOffsetMap
{
public:
    CsvColumnToOffsetMap(IHqlExpression * record, unsigned _maxRecordSize, bool _translateVirtuals, _ATOM _encoding);

    virtual MapFormat getFormat()                   { return MapFormatCsv; }
    virtual CMemberInfo * createColumn(CContainerInfo * container, IHqlExpression * column, RecordOffsetMap & map);

protected:
    _ATOM encoding;
};

class HQLCPP_API XmlColumnToOffsetMap : public ColumnToOffsetMap
{
public:
    XmlColumnToOffsetMap(IHqlExpression * record, unsigned _maxRecordSize, bool _translateVirtuals);

    virtual MapFormat getFormat()                   { return MapFormatXml; }
    virtual CMemberInfo * createColumn(CContainerInfo * container, IHqlExpression * column, RecordOffsetMap & map);
};

//---------------------------------------------------------------------------

class HQLCPP_API CChildSetColumnInfo : public CColumnInfo
{
public:
    CChildSetColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column);

//AColumnInfo
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void gatherSize(SizeStruct & target);
};


class HQLCPP_API CChildDatasetColumnInfo : public CColumnInfo
{
public:
    CChildDatasetColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column, RecordOffsetMap & map, unsigned defaultMaxRecordSize);

//AColumnInfo
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void gatherSize(SizeStruct & target);
    virtual AColumnInfo * lookupColumn(IHqlExpression * search);

protected:
    unsigned maxChildSize;
};


class HQLCPP_API CChildLimitedDatasetColumnInfo : public CColumnInfo
{
public:
    CChildLimitedDatasetColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column, RecordOffsetMap & map, unsigned defaultMaxRecordSize);

//AColumnInfo
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool prepareReadAhead(HqlCppTranslator & translator, ReadAheadState & state);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void gatherSize(SizeStruct & target);
    virtual bool isFixedSize();
    virtual AColumnInfo * lookupColumn(IHqlExpression * search);

protected:
    void setColumnFromBuilder(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlCppDatasetBuilder * builder);
    virtual void buildDeserializeChildLoop(HqlCppTranslator & translator, BuildCtx & loopctx, IReferenceSelector * selector, IHqlExpression * helper);

protected:
    OwnedHqlExpr countField;
    OwnedHqlExpr sizeField;
    unsigned fixedChildSize;
    unsigned maxChildSize;
};

class HQLCPP_API CChildLinkedDatasetColumnInfo : public CColumnInfo
{
public:
    CChildLinkedDatasetColumnInfo(CContainerInfo * _container, CMemberInfo * _prior, IHqlExpression * _column, RecordOffsetMap & map, unsigned defaultMaxRecordSize);

//AColumnInfo
    virtual void buildColumnExpr(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, CHqlBoundExpr & bound);  // get() after conditions.
    virtual void buildDeserialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual void buildSerialize(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * helper);
    virtual bool buildReadAhead(HqlCppTranslator & translator, BuildCtx & ctx, ReadAheadState & state);
    virtual IHqlExpression * buildSizeOfUnbound(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector);
    virtual bool modifyColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value, node_operator op);
    virtual void setColumn(HqlCppTranslator & translator, BuildCtx & ctx, IReferenceSelector * selector, IHqlExpression * value);

    virtual void gatherSize(SizeStruct & target);
    virtual AColumnInfo * lookupColumn(IHqlExpression * search);
    virtual bool isFixedSize()              { return true; }

protected:
    unsigned maxChildSize;
};


void ensureSimpleLength(HqlCppTranslator & translator, BuildCtx & ctx, CHqlBoundExpr & bound);
void callDeserializeGetN(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper, IHqlExpression * boundSize, IHqlExpression * address);
IHqlExpression * callDeserializerGetSize(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper);
void callDeserializerSkipInputSize(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper, IHqlExpression * size);
void callDeserializerSkipInputTranslatedSize(HqlCppTranslator & translator, BuildCtx & ctx, IHqlExpression * helper, IHqlExpression * size);

struct HQLCPP_API HqlStmtExprAssociation : public HqlExprAssociation
{
public:
    HqlStmtExprAssociation(IHqlExpression * _represents, IHqlStmt * _stmt) :
    HqlExprAssociation(_represents), stmt(_stmt) {}

    virtual AssocKind getKind() { return AssocStmt; }

public:
    IHqlStmt * stmt;
};

#endif