/*##############################################################################
Copyright (C) 2011 HPCC Systems.
All rights reserved. This program is free software: you can redistribute it and/or modify
it under the terms of the GNU Affero General Public License as
published by the Free Software Foundation, either version 3 of the
License, or (at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License
along with this program. If not, see .
############################################################################## */
#include "jliball.hpp"
#include "hql.hpp"
#include "platform.h"
#include "jlib.hpp"
#include "jmisc.hpp"
#include "jstream.ipp"
#include "jdebug.hpp"
#include "hql.hpp"
#include "hqlthql.hpp"
#include "hqlhtcpp.ipp"
#include "hqlttcpp.ipp"
#include "hqlutil.hpp"
#include "hqlthql.hpp"
#include "hqlpmap.hpp"
#include "hqlattr.hpp"
#include "hqlwcpp.hpp"
#include "hqlcpputil.hpp"
#include "hqltcppc.ipp"
#include "hqlopt.hpp"
#include "hqlfold.hpp"
#include "hqlcerrors.hpp"
#include "hqlcatom.hpp"
#include "hqlresource.hpp"
#include "hqlregex.ipp"
#include "hqlsource.ipp"
#include "hqlcse.ipp"
#include "hqlgraph.ipp"
#include "hqlccommon.hpp"
#include "hqliter.ipp"
#include "hqlinline.hpp"
#include "hqlusage.hpp"
#include "hqlcppds.hpp"
#define MAX_FIXED_SIZE_RAW 1024
#define INLINE_TABLE_EXPAND_LIMIT 4
void addGraphIdAttribute(ActivityInstance * instance, BuildCtx & ctx, IHqlExpression * graphId)
{
SubGraphInfo * match = matchActiveGraph(ctx, graphId);
assertex(match);
instance->addAttributeInt("_graphId", match->graphId);
}
//===========================================================================
void HqlCppTranslator::doBuildRowIfBranch(BuildCtx & initctx, BuildCtx & ctx, BoundRow * targetRow, IHqlExpression * branchExpr)
{
IHqlExpression * targetRowExpr = targetRow->queryBound();
Owned rowSelector = buildNewRow(ctx, branchExpr);
Owned boundRow = rowSelector->getRow(ctx);
OwnedHqlExpr rowExpr = getPointer(boundRow->queryBound());
OwnedHqlExpr castRow = createValue(no_implicitcast, targetRowExpr->getType(), LINK(rowExpr));
ctx.addAssign(targetRowExpr, castRow);
if (rowSelector->isConditional())
targetRow->setConditional(true);
}
IReferenceSelector * HqlCppTranslator::doBuildRowIf(BuildCtx & ctx, IHqlExpression * expr)
{
OwnedHqlExpr foldedCond = foldHqlExpression(expr->queryChild(0));
if (foldedCond->queryValue())
{
unsigned branch = (foldedCond->queryValue()->getBoolValue()) ? 1 : 2;
return buildNewRow(ctx, expr->queryChild(branch));
}
IHqlExpression * trueBranch = expr->queryChild(1);
IHqlExpression * falseBranch = expr->queryChild(2);
//Ideally should have a constant modifier on the following row...
Owned rowType = makeReferenceModifier(expr->getType());
OwnedHqlExpr rowExpr = ctx.getTempDeclare(rowType, NULL);
Owned row = createBoundRow(expr->queryBody(), rowExpr);
//MORE: Need casts because cursor may be (probably are) constant, but temporary isn't
//should find out by looking at the a const modifier on the cursor.
BuildCtx condctx(ctx);
IHqlStmt * cond = buildFilterViaExpr(condctx, foldedCond);
//Mark the context as conditional after the filter test, so any temporaries from the filter aren't affected.
condctx.associateExpr(queryConditionalRowMarker(), rowExpr);
doBuildRowIfBranch(ctx, condctx, row, trueBranch);
condctx.selectElse(cond);
condctx.associateExpr(queryConditionalRowMarker(), rowExpr);
doBuildRowIfBranch(ctx, condctx, row, falseBranch);
ctx.associate(*row);
return createReferenceSelector(row);
}
IReferenceSelector * HqlCppTranslator::doBuildRowDeserializeRow(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * srcRow = expr->queryChild(0);
IHqlExpression * record = expr->queryRecord();
Owned tempRow = declareLinkedRow(ctx, expr, false);
CHqlBoundTarget target;
target.expr.set(tempRow->queryBound());
HqlExprArray args;
args.append(*createRowAllocator(ctx, record));
args.append(*createRowSerializer(ctx, record, deserializerAtom));
args.append(*LINK(srcRow));
Owned resultType = makeReferenceModifier(makeAttributeModifier(makeRowType(record->getType()), getLinkCountedAttr()));
OwnedHqlExpr call = bindFunctionCall(rtlDeserializeRowAtom, args, resultType);
buildExprAssign(ctx, target, call);
ctx.associate(*tempRow);
return createReferenceSelector(tempRow);
}
void HqlCppTranslator::buildConstRow(IHqlExpression * record, IHqlExpression * rowData, CHqlBoundExpr & bound)
{
OwnedHqlExpr marker = createAttribute(rowAtom, LINK(record), LINK(rowData));
BuildCtx declareCtx(*code, literalAtom);
if (declareCtx.getMatchExpr(marker, bound))
return;
//MORE: This probably needs to go in the header as well...
Owned rowType = makeConstantModifier(makeRowType(record->getType()));
StringBuffer rowName;
getUniqueId(rowName.append("r"));
//Generate two variables to cope with the different ways the data is interpreted.
//Would prefer it to be cleaner... row value would need an associated size
unsigned dataSize = rowData->queryType()->getSize();
OwnedITypeInfo declareType;
OwnedHqlExpr initializer;
if (options.staticRowsUseStringInitializer)
{
//Generates smaller code (and probably more efficient representation in the c++ compiler
//const byte[5+1] = "Hello"; need an extra byte for the implicit \0
declareType.setown(makeDataType(dataSize+1));
initializer.set(rowData);
}
else
{
//Following is strictly correct, but much larger.
//const byte[5] = { 'H','e','l','l','o' };
declareType.set(rowData->queryType());
initializer.setown(createValue(no_create_initializer, rowData->getType(), LINK(rowData)));
}
//MORE: Currently these are marked as const rows, but not generated as such
OwnedHqlExpr boundDeclare = createVariable(rowName, makeConstantModifier(LINK(declareType)));
OwnedHqlExpr boundRow = createVariable(rowName, LINK(rowType));
declareCtx.addDeclare(boundDeclare, initializer);
if (options.spanMultipleCpp)
{
BuildCtx protoctx(*code, mainprototypesAtom);
protoctx.addDeclareExternal(boundDeclare);
}
bound.length.setown(getSizetConstant(dataSize));
bound.expr.set(boundRow);
declareCtx.associateExpr(marker, bound);
}
bool HqlCppTranslator::doBuildRowConstantTransform(IHqlExpression * transform, CHqlBoundExpr & bound)
{
if (!transform->isConstant() || !options.generateStaticInlineTables)
return false;
OwnedHqlExpr constRow = createConstantRowExpr(transform);
if (!constRow || !canGenerateStringInline(constRow->queryType()->getSize()))
return false;
buildConstRow(transform->queryRecord(), constRow, bound);
return true;
}
IReferenceSelector * HqlCppTranslator::doBuildRowCreateRow(BuildCtx & ctx, IHqlExpression * expr)
{
CHqlBoundExpr bound;
if (!doBuildRowConstantTransform(expr->queryChild(0), bound))
return doBuildRowViaTemp(ctx, expr);
BoundRow * row = bindConstantRow(ctx, expr, bound);
return createReferenceSelector(row);
}
BoundRow * HqlCppTranslator::bindConstantRow(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & bound)
{
BoundRow * row = bindRow(ctx, expr, bound.expr);
//MORE: This should be done more cleanly
OwnedHqlExpr sizeOfRow = createSizeof(row->querySelector());
ctx.associateExpr(sizeOfRow, bound.length);
return row;
}
bool HqlCppTranslator::doBuildRowConstantNull(IHqlExpression * expr, CHqlBoundExpr & bound)
{
if (!options.generateStaticInlineTables)
return false;
IHqlExpression * record = expr->queryRecord();
OwnedHqlExpr constRow = createConstantNullRowExpr(record);
if (!constRow)
return false;
buildConstRow(record, constRow, bound);
return true;
}
IReferenceSelector * HqlCppTranslator::doBuildRowNull(BuildCtx & ctx, IHqlExpression * expr)
{
CHqlBoundExpr bound;
if (!doBuildRowConstantNull(expr, bound))
return doBuildRowViaTemp(ctx, expr);
BoundRow * row = bindRow(ctx, expr, bound.expr);
return createReferenceSelector(row);
}
IReferenceSelector * HqlCppTranslator::doBuildRowViaTemp(BuildCtx & ctx, IHqlExpression * expr)
{
HqlExprAssociation * match;
if (expr->isDataset())
match = ctx.queryAssociation(expr->queryNormalizedSelector(), AssocCursor, NULL);
else
match = ctx.queryAssociation(expr, AssocRow, NULL);
if (match)
{
BoundRow * row = (BoundRow *)match;
return createReferenceSelector(row, expr);
}
Owned tempRow = declareTempRow(ctx, ctx, expr);
Owned rowBuilder = createRowBuilder(ctx, tempRow);
Owned createdRef = createReferenceSelector(rowBuilder);
buildRowAssign(ctx, createdRef, expr);
finalizeTempRow(ctx, tempRow, rowBuilder);
ctx.associate(*tempRow);
return createReferenceSelector(tempRow);
}
void HqlCppTranslator::buildDefaultRow(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & bound)
{
OwnedHqlExpr clearExpr = createRow(no_null, LINK(expr->queryRecord()), createAttribute(clearAtom));
BoundRow * matchedRow = (BoundRow *)ctx.queryAssociation(clearExpr, AssocRow, NULL);
if (!matchedRow)
{
if (doBuildRowConstantNull(expr, bound))
{
bindRow(ctx, clearExpr, bound.expr);
}
else
{
BuildCtx * declarectx = &ctx;
BuildCtx * callCtx = &ctx;
getInvariantMemberContext(ctx, &declarectx, &callCtx, true, false);
Owned tempRow = declareTempRow(*declarectx, *callCtx, clearExpr);
Owned rowBuilder = createRowBuilder(*callCtx, tempRow);
OwnedHqlExpr size = createVariable(LINK(sizetType));
OwnedHqlExpr clearCall = createClearRowCall(*callCtx, rowBuilder);
callCtx->addDeclare(size, clearCall);
OwnedHqlExpr sizeOfRow = createSizeof(rowBuilder->querySelector());
callCtx->associateExpr(sizeOfRow, size);
finalizeTempRow(*callCtx, tempRow, rowBuilder);
declarectx->associate(*tempRow);
bound.expr.set(tempRow->queryBound());
}
}
else
bound.expr.set(matchedRow->queryBound());
//yuk yuk, hack. If called from a const context then need to make the reference unconst.
//The real fix is to implement real const tracking throughout the code generator, but that is far from trivial.
//rkc39.hql is an example...
if (ctx.queryMatchExpr(constantMemberMarkerExpr))
bound.expr.setown(createValue(no_cast, makeReferenceModifier(bound.expr->getType()), getPointer(bound.expr)));
}
void HqlCppTranslator::buildNullRow(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & bound)
{
bound.expr.setown(createValue(no_nullptr, makeRowReferenceType(expr)));
}
IReferenceSelector * HqlCppTranslator::doBuildRowFromXML(BuildCtx & ctx, IHqlExpression * expr)
{
// assertex(supportsLinkCountedRows);
Owned overrideType = setLinkCountedAttr(expr->queryType(), true);
Owned utf8Type = makeUtf8Type(UNKNOWN_LENGTH, NULL);
IHqlExpression * record = expr->queryRecord();
OwnedHqlExpr ds = createDataset(no_anon, LINK(record));
StringBuffer xmlInstanceName, xmlFactoryName, s;
bool usesContents = false;
getUniqueId(xmlInstanceName.append("xml"));
buildXmlReadTransform(ds, xmlFactoryName, usesContents);
OwnedHqlExpr curActivityId = getCurrentActivityId(ctx);
s.append("Owned ").append(xmlInstanceName).append(" = ").append(xmlFactoryName).append("(ctx,");
generateExprCpp(s, curActivityId).append(");");
ctx.addQuoted(s);
HqlExprArray args;
args.append(*createRowAllocator(ctx, record));
args.append(*ensureExprType(expr->queryChild(1), utf8Type));
args.append(*createQuoted(xmlInstanceName, makeBoolType()));
args.append(*createConstant(expr->hasProperty(trimAtom)));
OwnedHqlExpr function = bindFunctionCall(createRowFromXmlAtom, args, overrideType);
CHqlBoundExpr bound;
buildExpr(ctx, function, bound);
Owned rowType = makeReferenceModifier(LINK(overrideType));
OwnedHqlExpr rowExpr = ctx.getTempDeclare(rowType, NULL);
Owned row = createBoundRow(expr, rowExpr);
ctx.associate(*row); // associate here because it is compared inside the loop
OwnedHqlExpr defaultRowPtr = getPointer(bound.expr);
ctx.addAssign(rowExpr, defaultRowPtr);
return createReferenceSelector(row);
}
//NB: If this is a dataset operation, this function assumes that any parent datasets are already in scope
// i.e. processed in buildDatasetAssign()
// the one exception is aggregate because it needs to treat its input differently.
IReferenceSelector * HqlCppTranslator::buildNewOrActiveRow(BuildCtx & ctx, IHqlExpression * expr, bool isNew)
{
if (isNew)
return buildNewRow(ctx, expr);
else
return buildActiveRow(ctx, expr);
}
IReferenceSelector * HqlCppTranslator::buildNewRow(BuildCtx & ctx, IHqlExpression * expr)
{
assertex(!expr->isDataset());
BoundRow * match = static_cast(ctx.queryAssociation(expr, AssocRow, NULL));
if (match)
return createReferenceSelector(match, expr);
BoundRow * row = NULL;
node_operator op = expr->getOperator();
switch (op)
{
case no_activerow:
return buildActiveRow(ctx, expr->queryChild(0));
case no_if:
return doBuildRowIf(ctx, expr);
case no_id2blob:
return doBuildRowIdToBlob(ctx, expr, true);
case no_index:
case no_selectnth:
return buildDatasetIndex(ctx, expr);
case no_selectmap:
return buildDatasetSelectMap(ctx, expr);
case no_left:
case no_right:
case no_self:
case no_top:
case no_activetable:
return buildActiveRow(ctx, expr);
case no_fromxml:
return doBuildRowFromXML(ctx, expr);
case no_serialize:
if (isDummySerializeDeserialize(expr))
return buildNewRow(ctx, expr->queryChild(0)->queryChild(0));
return doBuildRowViaTemp(ctx, expr);
case no_deserialize:
if (isDummySerializeDeserialize(expr))
return buildNewRow(ctx, expr->queryChild(0)->queryChild(0));
return doBuildRowDeserializeRow(ctx, expr);
case no_deref:
{
//Untested
CHqlBoundExpr bound;
buildExpr(ctx, expr, bound);
row = bindRow(ctx, expr, bound.expr);
break;
}
case no_createrow:
return doBuildRowCreateRow(ctx, expr);
case no_newusertable:
case no_hqlproject:
case no_temprow:
case no_projectrow:
return doBuildRowViaTemp(ctx, expr);
case no_null:
return doBuildRowNull(ctx, expr);
case no_typetransfer:
{
CHqlBoundExpr bound;
IHqlExpression * value = expr->queryChild(1);
if (value->isDatarow())
buildAddress(ctx, value, bound);
else
buildExpr(ctx, value, bound);
OwnedHqlExpr cursorExpr = createValue(no_implicitcast, makeReferenceModifier(expr->getType()), LINK(bound.expr));
row = bindRow(ctx, expr, cursorExpr);
break;
}
case no_getresult:
{
IHqlExpression * seqAttr = expr->queryProperty(sequenceAtom);
IHqlExpression * nameAttr = expr->queryProperty(namedAtom);
IHqlExpression * record = expr->queryRecord();
OwnedHqlExpr serializedRecord = getSerializedForm(record);
OwnedHqlExpr temp = createDatasetF(no_getresult, LINK(serializedRecord), LINK(seqAttr), LINK(nameAttr), NULL);
OwnedHqlExpr row = createRow(no_selectnth, LINK(temp), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom)));
row.setown(ensureDeserialized(row, expr->queryType()));
return buildNewRow(ctx, row);
}
case no_matchattr:
return doBuildRowMatchAttr(ctx, expr);
case no_matchrow:
return doBuildRowMatchRow(ctx, expr, true);
case no_getgraphresult:
case no_call:
case no_externalcall:
case no_alias:
case no_translated:
case no_libraryinput:
{
CHqlBoundExpr bound;
buildExpr(ctx, expr, bound);
Owned rawType = removeModifier(expr->queryType(), typemod_ref);
OwnedHqlExpr cursorExpr = createValue(no_implicitcast, makeReferenceModifier(LINK(rawType)), LINK(bound.expr));
row = bindRow(ctx, expr, cursorExpr);
if (bound.length)
{
OwnedHqlExpr sizeOfRow = createSizeof(row->querySelector());
ctx.associateExpr(sizeOfRow, bound.length);
}
//We could associate the original expression to allow better cse for child datasets in transforms, but it doesn't actually improve any examples
//IHqlExpression * original = queryPropertyChild(expr, _original_Atom, 0);
//if (original)
// bindRow(ctx, original, cursorExpr)->setResultAlias();
break;//return createReferenceSelector(cursor);
}
case no_comma:
case no_compound:
buildStmt(ctx, expr->queryChild(0));
return buildNewRow(ctx, expr->queryChild(1));
case no_select:
{
#ifdef _DEBUG
IHqlExpression * field = expr->queryChild(1);
#endif
Owned selector;
if (isNewSelector(expr))
selector.setown(buildNewRow(ctx, expr->queryChild(0)));
else
selector.setown(buildActiveRow(ctx, expr->queryChild(0)));
return selector->select(ctx, expr);
}
//If called because known to be a single row.
case no_datasetfromrow:
case no_nofold:
case no_nohoist:
case no_section:
case no_sectioninput:
return buildNewRow(ctx, expr->queryChild(0));
case no_skip:
{
buildStmt(ctx, expr);
OwnedHqlExpr null = createNullExpr(expr);
return buildNewRow(ctx, null);
}
case no_alias_scope:
{
expandAliasScope(ctx, expr);
return buildNewRow(ctx, expr->queryChild(0));
}
case no_split:
throwUnexpected();
//not at all sure about this.....
return buildNewRow(ctx, expr->queryChild(0));
default:
{
HqlExprAssociation * match;
if (expr->isDataset())
match = ctx.queryAssociation(expr->queryNormalizedSelector(), AssocCursor, NULL);
else
match = ctx.queryAssociation(expr, AssocRow, NULL);
if (match)
{
BoundRow * row = (BoundRow *)match;
IReferenceSelector * alias = row->queryAlias();
if (alias)
return LINK(alias);
return createReferenceSelector(row, expr);
}
UNIMPLEMENTED_XY("row", getOpString(expr->getOperator()));
}
}
assertex(row);
return createReferenceSelector(row);
}
IReferenceSelector * HqlCppTranslator::buildActiveRow(BuildCtx & ctx, IHqlExpression * expr)
{
node_operator op = expr->getOperator();
switch (op)
{
case no_left:
case no_right:
case no_self:
case no_top:
case no_activetable:
case no_selfref: // shouldn't ever occur...
//All selectors should be listed here...
break;
case no_activerow:
return buildActiveRow(ctx, expr->queryChild(0));
default:
if (!expr->isDataset() && !expr->isDictionary())
return buildNewRow(ctx, expr);
break;
}
HqlExprAssociation * match = ctx.queryAssociation(expr->queryNormalizedSelector(), AssocCursor, NULL);
if (match)
{
BoundRow * row = (BoundRow *)match;
IReferenceSelector * alias = row->queryAlias();
if (alias)
return LINK(alias);
return createReferenceSelector(row, expr);
}
switch (op)
{
case no_select:
{
#ifdef _DEBUG
IHqlExpression * field = expr->queryChild(1);
#endif
Owned selector = buildNewOrActiveRow(ctx, expr->queryChild(0), isNewSelector(expr));
return selector->select(ctx, expr);
}
case no_id2blob:
return doBuildRowIdToBlob(ctx, expr, false);
}
StringBuffer tablename;
getExprIdentifier(tablename, expr);
traceExpression("Dataset not found", expr);
RowAssociationIterator iter(ctx);
ForEach(iter)
{
BoundRow & cur = iter.get();
traceExpression("BoundCursor:", cur.querySelector());
}
throwError1(HQLERR_DatasetNotActive, tablename.str());
return NULL; //remove warning about control paths
}
//---------------------------------------------------------------------------
void HqlCppTranslator::doBuildExprAggregate(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
{
OwnedHqlExpr normalized = normalizeAnyDatasetAliases(expr);
if (expr != normalized)
{
buildExpr(ctx, normalized, tgt);
return;
}
node_operator op = expr->getOperator();
ITypeInfo * type = expr->queryType();
ITypeInfo * tempType = op == no_count ? unsignedType : type;
LoopInvariantHelper helper;
BuildCtx aggctx(ctx);
if (options.optimizeLoopInvariant)
helper.getBestContext(aggctx, expr);
CHqlBoundTarget result;
createTempFor(aggctx, tempType, result, typemod_none, FormatNatural);
doBuildAssignAggregate(aggctx, result, expr);
tgt.setFromTarget(result);
if (!isSameBasicType(type, tempType))
tgt.expr.setown(createValue(no_implicitcast, LINK(type), tgt.expr.getClear()));
if (expr->isPure())
aggctx.associateExpr(expr, tgt);
}
void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * dataset, IHqlExpression * doneFirstVar)
{
node_operator op = expr->getOperator();
switch (op)
{
case no_exists:
{
OwnedHqlExpr optimized = queryOptimizedExists(ctx, expr, dataset);
if (optimized)
{
assignBound(ctx, target, optimized);
return;
}
break;
}
case no_count:
{
CHqlBoundExpr temp;
if (canBuildOptimizedCount(ctx, dataset, temp))
{
OwnedHqlExpr thisCount = temp.getTranslatedExpr();
buildIncrementAssign(ctx, target, thisCount);
return;
}
break;
}
}
switch (dataset->getOperator())
{
case no_if:
{
BuildCtx subctx(ctx);
IHqlStmt * stmt = buildFilterViaExpr(subctx, dataset->queryChild(0));
doBuildAssignAggregateLoop(subctx, target, expr, dataset->queryChild(1), doneFirstVar);
subctx.selectElse(stmt);
doBuildAssignAggregateLoop(subctx, target, expr, dataset->queryChild(2), doneFirstVar);
return;
}
case no_addfiles:
{
doBuildAssignAggregateLoop(ctx, target, expr, dataset->queryChild(0), doneFirstVar);
doBuildAssignAggregateLoop(ctx, target, expr, dataset->queryChild(1), doneFirstVar);
return;
}
case no_null:
return;
}
LinkedHqlExpr arg = expr->queryChild(1);
IHqlExpression * oldDataset = expr->queryChild(0);
//If no_if or no_addfiles has been optimized above then the selector for the argument will have changed => map it.
if (arg && (dataset != oldDataset))
arg.setown(replaceSelector(arg, oldDataset, dataset));
bool needToBreak = (op == no_exists);
if (needToBreak)
{
//if it can have at most one row (fairly strange code!) then don't add a break
//unless it was deliberately a choosen to restrict the number of iterations.
if (hasNoMoreRowsThan(dataset, 1) && (dataset->getOperator() != no_choosen))
needToBreak = false;
}
BuildCtx loopctx(ctx);
buildDatasetIterate(loopctx, dataset, needToBreak);
switch (op)
{
case no_exists:
buildExprAssign(loopctx, target, queryBoolExpr(true));
if (needToBreak)
loopctx.addBreak();
break;
case no_count:
{
OwnedHqlExpr inc = createValue(no_postinc, makeVoidType(), LINK(target.expr));
loopctx.addExpr(inc);
break;
}
case no_sum:
{
OwnedHqlExpr cseArg = options.spotCSE ? spotScalarCSE(arg) : LINK(arg);
buildIncrementAssign(loopctx, target, cseArg);
break;
}
case no_min:
case no_max:
{
BuildCtx maxctx(loopctx);
OwnedHqlExpr resultExpr = target.getTranslatedExpr();
OwnedHqlExpr cseArg = options.spotCSE ? spotScalarCSE(arg) : LINK(arg);
OwnedHqlExpr simpleArg = buildSimplifyExpr(loopctx, cseArg);
OwnedHqlExpr test = createBoolExpr((op == no_min) ? no_lt : no_gt, LINK(simpleArg), LINK(resultExpr));
if (doneFirstVar)
{
IHqlExpression * firstTest = createValue(no_not, makeBoolType(), LINK(doneFirstVar));
test.setown(createBoolExpr(no_or, firstTest, test.getClear()));
}
buildFilter(maxctx, test);
buildExprAssign(maxctx, target, simpleArg);
if (doneFirstVar)
buildAssignToTemp(maxctx, doneFirstVar, queryBoolExpr(true));
break;
}
default:
assertex(!"unknown aggregate on child datasets");
break;
}
}
bool assignAggregateDirect(const CHqlBoundTarget & target, IHqlExpression * expr)
{
node_operator op = expr->getOperator();
ITypeInfo * type = expr->queryType();
ITypeInfo * tempType = op == no_count ? unsignedType : type;
if (!isSameUnqualifiedType(target.queryType(), tempType))
return false;
//For exists/count/sum use a temporary variable, and then assign rather than accumulating directly in the target
switch (op)
{
case no_sum:
if (type->getTypeCode() != type_int)
break;
//fall through
case no_exists:
case no_count:
if (target.expr->getOperator() != no_variable)
return false;
break;
}
return true;
}
static bool isNullValueMinimumValue(ITypeInfo * type)
{
switch (type->getTypeCode())
{
case type_int:
case type_swapint:
case type_decimal:
return !type->isSigned();
case type_data:
case type_qstring:
return true;
}
return false;
}
void HqlCppTranslator::doBuildAssignAggregate(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * _expr)
{
OwnedHqlExpr expr = normalizeAnyDatasetAliases(_expr);
if (assignAggregateDirect(target, expr))
{
IHqlExpression * dataset = expr->queryChild(0);
OwnedHqlExpr resultExpr = target.getTranslatedExpr();
node_operator op = expr->getOperator();
switch (op)
{
case no_exists:
buildExprAssign(ctx, target, queryBoolExpr(false));
break;
default:
{
OwnedHqlExpr null = createNullExpr(target.queryType());
buildExprAssign(ctx, target, null);
break;
}
}
OwnedHqlExpr doneFirstVar;
if ((op == no_min) || ((op == no_max) && !isNullValueMinimumValue(target.queryType())))
{
doneFirstVar.setown(ctx.getTempDeclare(queryBoolType(), queryBoolExpr(false)));
}
doBuildAssignAggregateLoop(ctx, target, expr, dataset, doneFirstVar);
}
else
{
doBuildExprAssign(ctx, target, expr);
}
}
//---------------------------------------------------------------------------
bool HqlCppTranslator::canBuildOptimizedCount(BuildCtx & ctx, IHqlExpression * dataset, CHqlBoundExpr & tgt)
{
switch (dataset->getOperator())
{
case no_select:
{
if (isMultiLevelDatasetSelector(dataset, false))
return false;
Owned selector = buildReference(ctx, dataset);
CHqlBoundExpr temp;
selector->get(ctx, temp);
tgt.expr.setown(getBoundCount(temp));
return true;
}
break;
default:
if (!options.tempDatasetsUseLinkedRows)
break;
if (!alwaysEvaluatesToBound(dataset))
break;
//fall through
case no_rows:
case no_null:
{
CHqlBoundExpr temp;
buildDataset(ctx, dataset, temp, FormatNatural);
tgt.expr.setown(getBoundCount(temp));
return true;
}
}
#if 0
//This is improves a few obscure cases (normally in the global context). I'm not convinced it is worth the extra cycles.
//Could also remove the bound.count test.
CHqlBoundExpr bound;
if (ctx.getMatchExpr(dataset, bound) && bound.count)
{
tgt.expr.setown(getBoundCount(bound));
return true;
}
#endif
return false;
}
void HqlCppTranslator::doBuildExprCount(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
{
if (expr->hasProperty(keyedAtom))
throwError(HQLERR_KeyedCountNonKeyable);
IHqlExpression * dataset = expr->queryChild(0);
CHqlBoundExpr temp;
if (canBuildOptimizedCount(ctx, dataset, temp))
{
OwnedHqlExpr translated = temp.getTranslatedExpr();
OwnedHqlExpr cast = ensureExprType(translated, expr->queryType());
buildExpr(ctx, cast, tgt);
}
else
doBuildExprAggregate(ctx, expr, tgt);
}
//---------------------------------------------------------------------------
IHqlExpression * HqlCppTranslator::queryOptimizedExists(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * dataset)
{
node_operator dsOp = dataset->getOperator();
//really this is isSimple()
CHqlBoundExpr optimized;
bool canOptimizeCount = canBuildOptimizedCount(ctx, dataset, optimized);
node_operator op = (expr->getOperator() == no_exists) ? no_ne : no_eq;
bool specialCase = false;
switch (dsOp)
{
case no_select:
specialCase = canOptimizeCount;
break;
default:
specialCase = !canOptimizeCount && alwaysEvaluatesToBound(dataset);
break;
}
if (specialCase)
{
CHqlBoundExpr temp;
buildDataset(ctx, dataset, temp, FormatNatural);
IHqlExpression * test;
if (temp.count)
test = LINK(temp.count);
else
test = getBoundSize(temp);
return createBoolExpr(op, test, createConstant(test->queryType()->castFrom(false, 0)));
}
else if (canOptimizeCount)
{
IHqlExpression * count = optimized.expr;
return createBoolExpr(op, LINK(count), createConstant(count->queryType()->castFrom(false, 0)));
}
return NULL;
}
void HqlCppTranslator::doBuildExprExists(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
{
OwnedHqlExpr optimized = queryOptimizedExists(ctx, expr, expr->queryChild(0));
if (optimized)
tgt.expr.setown(optimized.getClear());
else
doBuildExprAggregate(ctx, expr, tgt);
}
//---------------------------------------------------------------------------
static IHqlExpression * createMinMax(node_operator compareOp, ITypeInfo * type, IHqlExpression * left, IHqlExpression * right)
{
return createValue(no_if, LINK(type),
createBoolExpr(compareOp, LINK(left), LINK(right)),
LINK(left), LINK(right));
}
bool HqlCppTranslator::doBuildAggregateMinMaxList(BuildCtx & ctx, const CHqlBoundTarget * target, IHqlExpression * expr, IHqlExpression * list, CHqlBoundExpr * tgt, node_operator compareOp)
{
OwnedHqlExpr value;
unsigned max = list->numChildren();
switch(max)
{
case 0:
value.setown(createNullExpr(expr));
break;
case 1:
value.set(list->queryChild(0));
break;
case 2:
case 3:
{
OwnedHqlExpr simple[3];
for (unsigned i=0; i < max; i++)
{
CHqlBoundExpr bound;
buildSimpleExpr(ctx, list->queryChild(i), bound);
simple[i].setown(bound.getTranslatedExpr());
}
ITypeInfo * type = expr->queryType();
if (max == 2)
value.setown(createMinMax(compareOp, type, simple[0], simple[1]));
else
{
OwnedHqlExpr cmp02 = createMinMax(compareOp, type, simple[0], simple[2]);
OwnedHqlExpr cmp12 = createMinMax(compareOp, type, simple[1], simple[2]);
value.setown(createValue(no_if, expr->getType(),
createBoolExpr(compareOp, LINK(simple[0]), LINK(simple[1])),
LINK(cmp02), LINK(cmp12)));
}
}
}
if (value)
{
buildExprOrAssign(ctx, target, value, tgt);
return true;
}
{
CHqlBoundTarget temp;
createTempFor(ctx, expr, temp);
buildExprAssign(ctx, temp, list->queryChild(0));
OwnedHqlExpr best = temp.getTranslatedExpr();
for (unsigned i=1; i < list->numChildren(); i++)
{
CHqlBoundExpr bound;
buildSimpleExpr(ctx, list->queryChild(i), bound);
OwnedHqlExpr simple = bound.getTranslatedExpr();
BuildCtx subctx(ctx);
OwnedHqlExpr cond = createBoolExpr(compareOp, LINK(simple), LINK(best));
buildFilter(subctx, cond);
buildExprAssign(subctx, temp, simple);
}
buildExprOrAssign(ctx, target, best, tgt);
return true;
}
}
void HqlCppTranslator::doBuildAggregateList(BuildCtx & ctx, const CHqlBoundTarget * target, IHqlExpression * expr, CHqlBoundExpr * tgt)
{
OwnedHqlExpr list = normalizeListCasts(expr->queryChild(0));
if (list->getOperator() == no_alias_scope)
{
expandAliasScope(ctx, list);
list.set(list->queryChild(0));
}
node_operator aggOp;
switch (expr->getOperator())
{
case no_existslist:
{
//Fixed length lists should have been optimized away
CHqlBoundExpr bound;
buildExpr(ctx, list, bound);
OwnedHqlExpr test;
if (bound.count)
test.set(bound.count);
else
test.setown(getBoundLength(bound));
OwnedHqlExpr value = createValue(no_ne, makeBoolType(), LINK(test), ensureExprType(queryZero(), test->queryType()));
OwnedHqlExpr translated = createTranslated(value);
buildExprOrAssign(ctx, target, translated, tgt);
return;
}
case no_countlist:
{
//Fixed length lists should have been optimized away
CHqlBoundExpr bound;
buildExpr(ctx, list, bound);
OwnedHqlExpr test = getBoundCount(bound);
OwnedHqlExpr value = ensureExprType(test, expr->queryType());
OwnedHqlExpr translated = createTranslated(value);
buildExprOrAssign(ctx, target, translated, tgt);
return;
}
case no_sumlist:
aggOp = no_sumgroup;
if (list->getOperator() == no_list)
{
HqlExprArray args;
ITypeInfo * exprType = expr->queryType();
ForEachChild(i, list)
args.append(*ensureExprType(list->queryChild(i), exprType));
OwnedHqlExpr balanced = createBalanced(no_add, exprType, args);
if (!balanced)
balanced.setown(ensureExprType(queryZero(), exprType));
buildExprOrAssign(ctx, target, balanced, tgt);
return;
}
//special case fixed length lists
break;
case no_minlist:
aggOp = no_mingroup;
if (list->getOperator() == no_list)
{
if (doBuildAggregateMinMaxList(ctx, target, expr, list, tgt, no_lt))
return;
}
break;
case no_maxlist:
aggOp = no_maxgroup;
if (list->getOperator() == no_list)
{
if (doBuildAggregateMinMaxList(ctx, target, expr, list, tgt, no_gt))
return;
}
break;
default:
throwUnexpectedOp(expr->getOperator());
}
ITypeInfo * elemType = list->queryType()->queryChildType();
if (!elemType) elemType = defaultIntegralType;
//Default implementation in terms of a dataset
OwnedHqlExpr field = createField(valueAtom, LINK(elemType), NULL);
OwnedHqlExpr record = createRecord(field);
OwnedHqlExpr ds = createDataset(no_temptable, LINK(list), LINK(record));
OwnedHqlExpr aggField = createField(valueAtom, expr->getType(), NULL);
OwnedHqlExpr aggRecord = createRecord(aggField);
OwnedHqlExpr self = createSelector(no_self, aggRecord, NULL);
OwnedHqlExpr aggExpr = createValue(aggOp, expr->getType(), createSelectExpr(LINK(ds), LINK(field)));
OwnedHqlExpr aggAssign = createAssign(createSelectExpr(LINK(self), LINK(aggField)), LINK(aggExpr));
OwnedHqlExpr aggTransform = createValue(no_newtransform, makeTransformType(aggRecord->getType()), LINK(aggAssign));
OwnedHqlExpr agg = createDataset(no_newaggregate, LINK(ds), createComma(LINK(aggRecord), LINK(aggTransform)));
OwnedHqlExpr result = createNewSelectExpr(createRow(no_selectnth, LINK(agg), createConstantOne()), LINK(aggField));
buildExprOrAssign(ctx, target, result, tgt);
}
//---------------------------------------------------------------------------
static HqlTransformerInfo graphIndependanceCheckerInfo("GraphIndependanceChecker");
class GraphIndependanceChecker : public NewHqlTransformer
{
public:
GraphIndependanceChecker(IHqlExpression * _graph) : NewHqlTransformer(graphIndependanceCheckerInfo), graph(_graph) { independent = true; }
void analyseExpr(IHqlExpression * expr)
{
if (!independent || alreadyVisited(expr))
return;
switch (expr->getOperator())
{
case no_getgraphresult:
case no_getgraphloopresultset:
case no_getgraphloopresult:
if (expr->queryChild(1) == graph)
{
independent = false;
return;
}
break;
}
NewHqlTransformer::analyseExpr(expr);
}
inline bool isIndependent() const { return independent; }
protected:
LinkedHqlExpr graph;
bool independent;
};
bool isGraphIndependent(IHqlExpression * expr, IHqlExpression * graph)
{
switch (expr->getOperator())
{
case no_constant:
return true;
}
GraphIndependanceChecker checker(graph);
checker.analyse(expr, 0);
return checker.isIndependent();
}
///--------------------------------------------------------------------------------------------------------------------
IHqlExpression * createCounterAsGraphResult(IHqlExpression * counter, IHqlExpression * represents, unsigned seq)
{
OwnedHqlExpr value = createScalarFromGraphResult(counter->queryType(), unsignedType, represents, seq);
OwnedHqlExpr internalAttr = createAttribute(internalAtom);
return createAlias(value, internalAttr);
}
ChildGraphExprBuilder::ChildGraphExprBuilder(unsigned _numInputs)
: numInputs(_numInputs)
{
numOutputs=0;
represents.setown(createAttribute(graphAtom, createUniqueId()));
resultsExpr.setown(createAttribute(resultsAtom, LINK(represents)));
}
IHqlExpression * ChildGraphExprBuilder::addDataset(IHqlExpression * expr)
{
OwnedHqlExpr resultNumExpr;
ForEachItemIn(i, results)
{
IHqlExpression & curSetResult = results.item(i);
if (expr->queryBody() == curSetResult.queryChild(0)->queryBody())
{
resultNumExpr.set(curSetResult.queryChild(2));
break;
}
}
if (!resultNumExpr)
{
resultNumExpr.setown(getSizetConstant(numResults()));
results.append(*createValue(no_setgraphresult, makeVoidType(), LINK(expr), LINK(represents), LINK(resultNumExpr)));
numOutputs++;
}
HqlExprArray args;
args.append(*LINK(expr->queryRecord()));
args.append(*LINK(represents));
args.append(*LINK(resultNumExpr));
if (isGrouped(expr))
args.append(*createAttribute(groupedAtom));
if (!expr->isDataset())
args.append(*createAttribute(rowAtom));
args.append(*createAttribute(externalAtom, LINK(resultsExpr)));
args.append(*createAttribute(_original_Atom, LINK(expr)));
IHqlExpression * recordCountAttr = queryRecordCountInfo(expr);
if (recordCountAttr)
args.append(*LINK(recordCountAttr));
OwnedHqlExpr ret = createDataset(no_getgraphresult, args);
if (expr->isDatarow())
ret.setown(createRow(no_selectnth, LINK(ret), createComma(getSizetConstant(1), createAttribute(noBoundCheckAtom))));
return ret.getClear();
}
void ChildGraphExprBuilder::addAction(IHqlExpression * expr)
{
results.append(*LINK(expr));
}
unsigned ChildGraphExprBuilder::addInput()
{
unsigned id = numResults();
numInputs++;
return id;
}
IHqlExpression * ChildGraphExprBuilder::getGraph()
{
HqlExprArray args;
args.append(*LINK(represents));
args.append(*getSizetConstant(numResults()));
args.append(*createActionList(results));
return createValue(no_childquery, makeVoidType(), args);
}
//---------------------------------------------------------------------------
// Child dataset processing
ChildGraphBuilder::ChildGraphBuilder(HqlCppTranslator & _translator, IHqlExpression * subgraph)
: translator(_translator)
{
represents.set(subgraph->queryChild(0));
id = translator.nextActivityId();
appendUniqueId(instanceName.append("child"), id);
instanceExpr.setown(createQuoted(instanceName, makeBoolType()));
resultsExpr.setown(createAttribute(resultsAtom, LINK(represents)));
StringBuffer s;
resultInstanceExpr.setown(createQuoted(appendUniqueId(s.append("res"), id), makeBoolType()));
numResults = (unsigned)getIntValue(subgraph->queryChild(1));
IHqlExpression * actions = subgraph->queryChild(2);
actions->unwindList(results, no_actionlist);
}
void ChildGraphBuilder::generateGraph(BuildCtx & ctx)
{
BuildCtx graphctx(ctx);
//Make sure at least one results - because currently that's how we determine if new resourcing is being used
//Remove this line once all engines use the new child queries exclusively
if (numResults == 0) numResults++;
OwnedHqlExpr query = createActionList(results);
OwnedHqlExpr resourced = translator.getResourcedChildGraph(graphctx, query, represents, numResults, no_none);
Owned extractBuilder = translator.createExtractBuilder(graphctx, PETchild, represents, resourced, true);
if (!translator.queryOptions().serializeRowsetInExtract)
extractBuilder->setAllowDestructor();
translator.beginExtract(graphctx, extractBuilder);
translator.doBuildThorSubGraph(graphctx, resourced, SubGraphChild, id, represents);
EvalContext * instance = translator.queryEvalContext(graphctx);
OwnedHqlExpr retInstanceExpr;
if (instance && !translator.insideOnCreate(graphctx))
retInstanceExpr.setown(instance->createGraphLookup(id, true));
else
retInstanceExpr.setown(translator.doCreateGraphLookup(graphctx, graphctx, id, "this", true));
assertex(retInstanceExpr == instanceExpr);
CHqlBoundExpr boundExtract;
extractBuilder->endCreateExtract(boundExtract);
HqlExprArray args;
args.append(*LINK(instanceExpr));
args.append(*createTranslated(boundExtract.length));
args.append(*boundExtract.getTranslatedExpr());
OwnedHqlExpr call = translator.bindFunctionCall(evaluateChildQueryInstanceAtom, args);
CHqlBoundExpr bound;
translator.buildExpr(graphctx, call, bound);
StringBuffer s;
s.append("Owned ");
translator.generateExprCpp(s, resultInstanceExpr);
s.append(" = ");
translator.generateExprCpp(s, bound.expr);
s.append(";");
graphctx.addQuoted(s);
translator.endExtract(graphctx, extractBuilder);
ctx.associateExpr(resultsExpr, resultInstanceExpr);
}
void ChildGraphBuilder::generatePrefetchGraph(BuildCtx & _ctx, OwnedHqlExpr * retGraphExpr)
{
BuildCtx ctx(_ctx);
ctx.addGroup();
BuildCtx aliasctx(ctx);
aliasctx.addGroup();
OwnedHqlExpr query = createActionList(results);
OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_none);
Owned extractBuilder = translator.createExtractBuilder(ctx, PETchild, represents, resourced, false);
createBuilderAlias(aliasctx, extractBuilder);
translator.beginExtract(ctx, extractBuilder);
translator.doBuildThorSubGraph(ctx, resourced, SubGraphChild, id, represents);
EvalContext * instance = translator.queryEvalContext(ctx);
OwnedHqlExpr retInstanceExpr;
assertex(instance && !translator.insideOnCreate(ctx));
retInstanceExpr.setown(instance->createGraphLookup(id, true));
assertex(retInstanceExpr == instanceExpr);
retGraphExpr->setown(retInstanceExpr.getClear());
}
void ChildGraphBuilder::createBuilderAlias(BuildCtx & ctx, ParentExtract * extractBuilder)
{
StringBuffer s;
s.append("rtlRowBuilder & ");
translator.generateExprCpp(s, extractBuilder->queryExtractName());
s.append(" = builder;");
ctx.addQuoted(s);
}
unique_id_t ChildGraphBuilder::buildLoopBody(BuildCtx & ctx, bool multiInstance)
{
BuildCtx subctx(ctx);
subctx.addGroup();
OwnedHqlExpr query = createActionList(results);
OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_loop);
//Add a flag to indicate multi instance
if (multiInstance)
resourced.setown(appendOwnedOperand(resourced, createAttribute(multiInstanceAtom)));
bool isGlobalThorLoop = translator.targetThor() && !translator.insideChildQuery(ctx);
Owned extractBuilder = isGlobalThorLoop ? translator.createExtractBuilder(ctx, PETloop, represents, GraphRemote, false)
: translator.createExtractBuilder(ctx, PETloop, represents, resourced, false);
createBuilderAlias(subctx, extractBuilder);
translator.beginExtract(ctx, extractBuilder);
translator.doBuildThorSubGraph(ctx, resourced, SubGraphChild, id, represents);
translator.endExtract(ctx, extractBuilder);
return id;
}
static HqlTransformerInfo graphLoopReplacerInfo("GraphLoopReplacer");
class GraphLoopReplacer : public NewHqlTransformer
{
public:
GraphLoopReplacer(IHqlExpression * _rowsid, IHqlExpression * _represents, IHqlExpression * _counter, bool _isParallel) :
NewHqlTransformer(graphLoopReplacerInfo), rowsid(_rowsid), represents(_represents), counter(_counter), isParallel(_isParallel)
{
}
virtual IHqlExpression * createTransformed(IHqlExpression * expr)
{
switch (expr->getOperator())
{
case no_counter:
if (expr->queryBody() == counter)
{
if (isParallel)
{
HqlExprArray args;
args.append(*LINK(represents));
// unwindChildren(args, expr);
OwnedHqlExpr ret = createValue(no_loopcounter, expr->getType(), args);
//Yuk: Wrap this in an alias to ensure it is evaluated at the correct place.
//there has to be a better way..... We could...
//a) strictly defined when it can be evaluated - e.g., ctx->defines(graph) && (!parentctx || !parentctx->definesGraph)
//b) set a flag in the expression to indicate forced evaluation (even worse than the alias)
//c) add the code to evaluate no_loopcounter inside evaluateInContext
return createAlias(ret, internalAttrExpr);
}
else
{
counterResult.setown(createCounterAsGraphResult(counter, represents, 0));
return LINK(counterResult);
}
}
break;
case no_rowsetindex:
{
IHqlExpression * rowset = expr->queryChild(0);
if (rowset->getOperator() != no_rowset)
break;
IHqlExpression * rows = rowset->queryChild(0);
if (rows->queryChild(1) != rowsid)
break;
HqlExprArray args;
args.append(*LINK(rows->queryChild(0)->queryRecord()));
args.append(*LINK(represents));
args.append(*transform(expr->queryChild(1)));
return createDataset(no_getgraphloopresult, args);
}
case no_rowset:
{
IHqlExpression * rows = expr->queryChild(0);
if (rows->queryChild(1) != rowsid)
break;
HqlExprArray args;
args.append(*LINK(rows->queryChild(0)->queryRecord()));
args.append(*LINK(represents));
return createValue(no_getgraphloopresultset, expr->getType(), args);
}
}
return NewHqlTransformer::createTransformed(expr);
}
inline IHqlExpression * queryCounterResult() { return counterResult; }
protected:
IHqlExpression * rowsid;
IHqlExpression * represents;
IHqlExpression * counter;
OwnedHqlExpr counterResult;
bool isParallel;
};
unique_id_t ChildGraphBuilder::buildGraphLoopBody(BuildCtx & ctx, bool isParallel)
{
BuildCtx subctx(ctx);
subctx.addGroup();
OwnedHqlExpr query = createActionList(results);
translator.traceExpression("Before Loop resource", query);
OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_loop);
translator.traceExpression("After Loop resource", resourced);
//Add a flag to indicate multi instance
if (isParallel)
{
HqlExprArray args;
unwindChildren(args, resourced);
args.append(*createAttribute(multiInstanceAtom));
args.append(*createAttribute(delayedAtom));
resourced.setown(resourced->clone(args));
}
bool isGlobalThorLoop = translator.targetThor() && !translator.insideChildQuery(ctx);
Owned extractBuilder = isGlobalThorLoop ? translator.createExtractBuilder(ctx, PETloop, represents, GraphRemote, false)
: translator.createExtractBuilder(ctx, PETloop, represents, resourced, false);
createBuilderAlias(subctx, extractBuilder);
translator.beginExtract(ctx, extractBuilder);
translator.doBuildThorSubGraph(ctx, resourced, SubGraphChild, id, represents);
translator.endExtract(ctx, extractBuilder);
return id;
}
unique_id_t ChildGraphBuilder::buildRemoteGraph(BuildCtx & ctx)
{
BuildCtx subctx(ctx);
subctx.addGroup();
OwnedHqlExpr query = createActionList(results);
OwnedHqlExpr resourced = translator.getResourcedChildGraph(ctx, query, represents, numResults, no_allnodes);
Owned extractBuilder = translator.createExtractBuilder(ctx, PETremote, represents, GraphRemote, false);
createBuilderAlias(subctx, extractBuilder);
translator.beginExtract(ctx, extractBuilder);
translator.doBuildThorSubGraph(ctx, resourced, SubGraphChild, id, represents);
translator.endExtract(ctx, extractBuilder);
return id;
}
void HqlCppTranslator::buildChildGraph(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * represents= expr->queryChild(0);
OwnedHqlExpr resultsExpr = createAttribute(resultsAtom, LINK(represents));
//Shouldn't really happen, but if this graph has already benn called just use the results
if (ctx.queryMatchExpr(resultsExpr))
return;
ChildGraphBuilder graphBuilder(*this, expr);
graphBuilder.generateGraph(ctx);
}
void HqlCppTranslator::beginExtract(BuildCtx & ctx, ParentExtract * extractBuilder)
{
ctx.associate(*extractBuilder);
}
void HqlCppTranslator::endExtract(BuildCtx & ctx, ParentExtract * extractBuilder)
{
extractBuilder->endUseExtract(ctx);
ctx.removeAssociation(extractBuilder);
}
void HqlCppTranslator::buildAssignChildDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
{
switch (expr->getOperator())
{
case no_call:
case no_externalcall:
case no_libraryinput:
buildDatasetAssign(ctx, target, expr);
return;
}
OwnedHqlExpr call;
{
ChildGraphExprBuilder builder(0);
call.setown(builder.addDataset(expr));
OwnedHqlExpr subquery = builder.getGraph();
buildStmt(ctx, subquery);
}
buildExprAssign(ctx, target, call);
}
IHqlExpression * HqlCppTranslator::getResourcedChildGraph(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * graphIdExpr, unsigned numResults, node_operator graphKind)
{
if (options.paranoidCheckNormalized || options.paranoidCheckDependencies)
DBGLOG("Before resourcing a child graph");
LinkedHqlExpr resourced = expr;
checkNormalized(ctx, resourced);
unsigned csfFlags = CSFindex|options.optimizeDiskFlag;
switch (targetClusterType)
{
case HThorCluster:
csfFlags |= CSFcompoundSpill;
break;
case ThorCluster:
case ThorLCRCluster:
//Don't compound spills inside a child query because it can cause non remote projects to become remote
//And we'll also probably be using the roxie code to implement
break;
case RoxieCluster:
break;
}
{
unsigned time = msTick();
CompoundSourceTransformer transformer(*this, CSFpreload|csfFlags);
resourced.setown(transformer.process(resourced));
checkNormalized(ctx, resourced);
DEBUG_TIMER("EclServer: tree transform: optimize disk read", msTick()-time);
}
if (options.optimizeChildGraph)
{
unsigned time = msTick();
traceExpression("BeforeOptimizeSub", resourced);
resourced.setown(optimizeHqlExpression(resourced, getOptimizeFlags()|HOOcompoundproject));
traceExpression("AfterOptimizeSub", resourced);
DEBUG_TIMER("EclServer: optimize graph", msTick()-time);
}
traceExpression("BeforeResourcing Child", resourced);
cycle_t time = msTick();
HqlExprCopyArray activeRows;
gatherActiveCursors(ctx, activeRows);
if (graphKind == no_loop)
{
bool insideChild = insideChildQuery(ctx);
resourced.setown(resourceLoopGraph(*this, activeRows, resourced, targetClusterType, graphIdExpr, &numResults, insideChild));
}
else
resourced.setown(resourceNewChildGraph(*this, activeRows, resourced, targetClusterType, graphIdExpr, &numResults));
DEBUG_TIMER("EclServer: resource graph", msTick()-time);
checkNormalized(ctx, resourced);
traceExpression("AfterResourcing Child", resourced);
//Convert queries on preloaded into compound activities - before resourcing so keyed gets done correctly
// Second attempt to spot compound disk reads - this time of spill files. Since resourcing has removed
// any sharing we don't need to bother about sharing.
{
unsigned time = msTick();
CompoundSourceTransformer transformer(*this, csfFlags);
resourced.setown(transformer.process(resourced));
DEBUG_TIMER("EclServer: tree transform: optimize disk read", msTick()-time);
}
//Now call the optimizer again - the main purpose is to move projects over limits and into compound index/disk reads
if (options.optimizeChildGraph)
{
unsigned time = msTick();
traceExpression("BeforeOptimize2", resourced);
resourced.setown(optimizeHqlExpression(resourced, getOptimizeFlags()|HOOcompoundproject));
traceExpression("AfterOptimize2", resourced);
DEBUG_TIMER("EclServer: optimize graph", msTick()-time);
}
if (numResults == 0) numResults++;
HqlExprArray children;
resourced->unwindList(children, no_actionlist);
children.append(*createAttribute(numResultsAtom, getSizetConstant(numResults)));
children.append(*LINK(graphIdExpr));
resourced.setown(createValue(no_subgraph, makeVoidType(), children));
if (options.paranoidCheckNormalized || options.paranoidCheckDependencies)
DBGLOG("After resourcing a child graph");
return resourced.getClear();
}
void HqlCppTranslator::buildChildDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
{
if (expr->isPure() && ctx.getMatchExpr(expr, tgt))
return;
LoopInvariantHelper helper;
BuildCtx bestctx(ctx);
if (options.optimizeLoopInvariant)
helper.getBestContext(bestctx, expr);
CHqlBoundTarget temp;
//MORE: Should have similar code to buildTempExpr()
createTempFor(bestctx, expr, temp);
buildAssignChildDataset(bestctx, temp, expr);
tgt.setFromTarget(temp);
if (expr->isPure())
bestctx.associateExpr(expr, tgt);
}
unique_id_t HqlCppTranslator::buildGraphLoopSubgraph(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * selSeq, IHqlExpression * rowsid, IHqlExpression * body, IHqlExpression * counter, bool multiInstance)
{
ChildGraphExprBuilder graphBuilder(0);
OwnedHqlExpr transformedBody;
OwnedHqlExpr counterResult;
IHqlExpression * graphid = graphBuilder.queryRepresents();
{
const bool isParallel = multiInstance;
GraphLoopReplacer replacer(rowsid, graphid, counter, isParallel);
transformedBody.setown(replacer.transformRoot(body));
counterResult.set(replacer.queryCounterResult());
}
if (counterResult)
graphBuilder.addInput();
OwnedHqlExpr result = createValue(no_setgraphloopresult, makeVoidType(), LINK(transformedBody), LINK(graphid));
graphBuilder.addAction(result);
OwnedHqlExpr subquery = graphBuilder.getGraph();
ChildGraphBuilder builder(*this, subquery);
return builder.buildGraphLoopBody(ctx, multiInstance);
}
unique_id_t HqlCppTranslator::buildRemoteSubgraph(BuildCtx & ctx, IHqlExpression * dataset)
{
ChildGraphExprBuilder graphBuilder(0);
if (dataset->isAction())
{
graphBuilder.addAction(dataset);
}
else
{
OwnedHqlExpr ignoredResult = graphBuilder.addDataset(dataset);
}
OwnedHqlExpr subquery = graphBuilder.getGraph();
ChildGraphBuilder builder(*this, subquery);
return builder.buildRemoteGraph(ctx);
}
//---------------------------------------------------------------------------
// Functions to check whether a dataset can be evaluated inline or not.
//MORE: These should probably be split out into an hqlinline.cpp
bool HqlCppTranslator::canIterateInline(BuildCtx * ctx, IHqlExpression * expr)
{
return (isInlineOk() && ::canIterateInline(ctx, expr));
}
bool HqlCppTranslator::canAssignInline(BuildCtx * ctx, IHqlExpression * expr)
{
if (!isInlineOk())
return false;
return options.allowInlineSpill ? ::canProcessInline(ctx, expr) : ::canAssignInline(ctx, expr);
}
bool HqlCppTranslator::canEvaluateInline(BuildCtx * ctx, IHqlExpression * expr)
{
if (!isInlineOk())
return false;
return options.allowInlineSpill ? ::canProcessInline(ctx, expr) : ::canEvaluateInline(ctx, expr);
}
bool HqlCppTranslator::canProcessInline(BuildCtx * ctx, IHqlExpression * expr)
{
if (!isInlineOk())
return false;
return ::canProcessInline(ctx, expr);
}
bool HqlCppTranslator::isInlineOk()
{
if (!activeGraphCtx)
return true;
return true;
}
IHqlExpression * HqlCppTranslator::buildSpillChildDataset(BuildCtx & ctx, IHqlExpression * expr)
{
CHqlBoundExpr bound;
buildChildDataset(ctx, expr, bound);
return bound.getTranslatedExpr();
}
IHqlExpression * HqlCppTranslator::forceInlineAssignDataset(BuildCtx & ctx, IHqlExpression * expr)
{
loop
{
CHqlBoundExpr bound;
if (expr->isPure() && ctx.getMatchExpr(expr, bound))
return bound.getTranslatedExpr();
if (canProcessInline(&ctx, expr) || (expr->getOperator() == no_translated))
return LINK(expr);
switch (expr->getOperator())
{
case no_compound:
buildStmt(ctx, expr->queryChild(0));
expr = expr->queryChild(1);
break;
default:
return buildSpillChildDataset(ctx, expr);
}
}
}
//---------------------------------------------------------------------------
// Dataset temp creation
IHqlExpression * createGetResultFromWorkunitDataset(IHqlExpression * expr)
{
IHqlExpression * name = queryPropertyChild(expr, nameAtom, 0);
if (name)
name = createExprAttribute(namedAtom, LINK(name));
assertex(expr->isDataset());
return createDataset(no_getresult, LINK(expr->queryRecord()), createComma(LINK(expr->queryProperty(sequenceAtom)), name));
}
void HqlCppTranslator::buildAssignSerializedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
{
OwnedITypeInfo serializedType = getSerializedForm(expr->queryType());
assertex(recordTypesMatch(target.queryType(), serializedType));
HqlExprArray args;
args.append(*createRowSerializer(ctx, expr->queryRecord(), serializerAtom));
args.append(*LINK(expr));
OwnedHqlExpr call = bindFunctionCall(rowset2DatasetXAtom, args);
buildExprAssign(ctx, target, call);
}
void HqlCppTranslator::buildSerializedDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt)
{
CHqlBoundTarget target;
OwnedITypeInfo serializedType = getSerializedForm(expr->queryType());
createTempFor(ctx, serializedType, target, typemod_none, FormatBlockedDataset);
buildAssignSerializedDataset(ctx, target, expr);
tgt.setFromTarget(target);
}
void HqlCppTranslator::buildAssignLinkedDataset(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
{
OwnedITypeInfo serializedType = getSerializedForm(target.queryType());
assertex(recordTypesMatch(serializedType, expr->queryType()));
IHqlExpression * record = ::queryRecord(target.queryType());
HqlExprArray args;
args.append(*createRowSerializer(ctx, record, deserializerAtom));
args.append(*LINK(expr));
OwnedHqlExpr call = bindFunctionCall(isGrouped(expr) ? groupedDataset2RowsetXAtom : dataset2RowsetXAtom, args, target.queryType());
buildExprAssign(ctx, target, call);
}
void HqlCppTranslator::buildLinkedDataset(BuildCtx & ctx, ITypeInfo * type, IHqlExpression * expr, CHqlBoundExpr & tgt)
{
CHqlBoundTarget target;
createTempFor(ctx, type, target, typemod_none, FormatLinkedDataset);
if (hasLinkedRow(expr->queryType()))
buildDatasetAssign(ctx, target, expr);
else
buildAssignLinkedDataset(ctx, target, expr);
tgt.setFromTarget(target);
}
void HqlCppTranslator::ensureDatasetFormat(BuildCtx & ctx, ITypeInfo * type, CHqlBoundExpr & tgt, ExpressionFormat format)
{
switch (format)
{
case FormatBlockedDataset:
if (isArrayRowset(tgt.queryType()))
{
OwnedHqlExpr deserializedExpr = tgt.getTranslatedExpr();
LinkedHqlExpr savedCount = tgt.count;
buildSerializedDataset(ctx, deserializedExpr, tgt);
if (savedCount && !isFixedWidthDataset(deserializedExpr))
tgt.count.set(savedCount);
return;
}
break;
case FormatLinkedDataset:
if (!hasLinkCountedModifier(tgt.queryType()))
{
OwnedHqlExpr serializedExpr = tgt.getTranslatedExpr();
buildLinkedDataset(ctx, type, serializedExpr, tgt);
return;
}
break;
case FormatArrayDataset:
if (!isArrayRowset(tgt.queryType()))
{
OwnedHqlExpr serializedExpr = tgt.getTranslatedExpr();
buildLinkedDataset(ctx, type, serializedExpr, tgt);
return;
}
break;
}
}
void HqlCppTranslator::buildDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, ExpressionFormat format)
{
doBuildDataset(ctx, expr, tgt, format);
ensureDatasetFormat(ctx, expr->queryType(), tgt, format);
}
void HqlCppTranslator::doBuildDataset(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, ExpressionFormat format)
{
if (expr->isPure() && ctx.getMatchExpr(expr, tgt))
return;
/*
OwnedHqlExpr transformed = normalizeAnyDatasetAliases(expr);
if (transformed && (transformed != expr))
{
doBuildDataset(ctx, transformed, tgt, format);
ctx.associateExpr(expr, tgt);
return;
}
*/
node_operator op = expr->getOperator();
switch (op)
{
case no_dataset_alias:
if (!expr->hasProperty(_normalized_Atom))
{
OwnedHqlExpr uniqueChild = normalizeDatasetAlias(expr);
doBuildDataset(ctx, uniqueChild, tgt, format);
}
else
doBuildDataset(ctx, expr->queryChild(0), tgt, format);
return;
case no_alias:
doBuildExprAlias(ctx, expr, &tgt);
return;
case no_owned_ds:
buildTempExpr(ctx, expr, tgt);
return;
case no_null:
{
tgt.count.setown(getSizetConstant(0));
tgt.length.setown(getSizetConstant(0));
IHqlExpression * record = expr->queryRecord();
ITypeInfo * type = makeTableType(makeRowType(record->getType()), NULL, NULL, NULL);
if ((format == FormatLinkedDataset) || (format == FormatArrayDataset))
type = makeAttributeModifier(type, getLinkCountedAttr());
tgt.expr.setown(createValue(no_nullptr, makeReferenceModifier(type)));
return;
}
case no_translated:
expandTranslated(expr, tgt);
return;
case no_select:
{
if (isMultiLevelDatasetSelector(expr, false))
break;
Owned selected = buildReference(ctx, expr);
selected->get(ctx, tgt);
return;
}
case no_libraryinput:
if (!buildExprInCorrectContext(ctx, expr, tgt, false))
throwUnexpected();
return;
case no_call:
case no_externalcall:
if (!hasStreamedModifier(expr->queryType()))
{
buildTempExpr(ctx, expr, tgt);
return;
}
break;
case no_newaggregate:
if (canAssignInline(&ctx, expr))
{
Owned tempRow = declareTempAnonRow(ctx, ctx, expr);
Owned rowBuilder = createRowBuilder(ctx, tempRow);
Owned createdRef = createReferenceSelector(rowBuilder);
BuildCtx subctx(ctx);
subctx.addGroup();
doBuildRowAssignAggregate(subctx, createdRef, expr);
finalizeTempRow(ctx, tempRow, rowBuilder);
convertBoundRowToDataset(ctx, tgt, tempRow, format);
return;
}
break;
case no_id2blob:
doBuildExprIdToBlob(ctx, expr, tgt);
return;
case no_rows:
{
if (!buildExprInCorrectContext(ctx, expr, tgt, false))
throwError(HQLERR_RowsUsedOutsideContext);
return;
}
case no_limit:
if (expr->hasProperty(skipAtom) || expr->hasProperty(onFailAtom))
break;
doBuildDatasetLimit(ctx, expr, tgt, format);
return;
case no_compound_childread:
case no_compound_childnormalize:
case no_compound_childaggregate:
case no_compound_selectnew:
case no_compound_inline:
case no_distributed:
case no_preservemeta:
case no_sorted:
case no_nofold:
case no_nohoist:
case no_section:
case no_sectioninput:
buildDataset(ctx, expr->queryChild(0), tgt, format);
return;
case no_forcegraph:
#ifdef _DEBUG
throwUnexpected();
#endif
buildDataset(ctx, expr->queryChild(0), tgt, format);
return;
case no_getgraphresult:
doBuildExprGetGraphResult(ctx, expr, tgt, format);
return;
case no_getresult:
case no_workunit_dataset:
if (!isGrouped(expr))
{
doBuildExprGetResult(ctx, expr, tgt);
return;
}
break;
case no_skip:
{
buildStmt(ctx, expr);
OwnedHqlExpr null = createNullExpr(expr);
buildDataset(ctx, null, tgt, format);
return;
}
case no_serialize:
if (isDummySerializeDeserialize(expr))
doBuildDataset(ctx, expr->queryChild(0)->queryChild(0), tgt, format);
else
buildSerializedDataset(ctx, expr->queryChild(0), tgt);
return;
case no_deserialize:
if (isDummySerializeDeserialize(expr))
doBuildDataset(ctx, expr->queryChild(0)->queryChild(0), tgt, format);
else
buildLinkedDataset(ctx, expr->queryType(), expr->queryChild(0), tgt);
return;
case no_datasetfromrow:
{
IHqlExpression * row = expr->queryChild(0);
if (isAlwaysActiveRow(row) && (format == FormatNatural))
{
Owned selector = buildActiveRow(ctx, row);
BuildCtx groupctx(ctx);
groupctx.addGroup();
BoundRow * bound = bindSelectorAsRootRow(groupctx, selector, row);
convertBoundRowToDataset(groupctx, tgt, bound, format);
tgt.count.setown(getSizetConstant(1));
ctx.associateExpr(expr, tgt);
return;
}
break;
}
case no_inlinetable:
case no_inlinedictionary:
if (doBuildDatasetInlineTable(ctx, expr, tgt, format))
return;
break;
case no_compound:
{
buildStmt(ctx, expr->queryChild(0));
buildDataset(ctx, expr->queryChild(1), tgt, format);
return;
}
}
bool singleRow = hasSingleRow(expr);
bool useTempRow = singleRow && canAssignInline(&ctx, expr) && (format != FormatLinkedDataset) && (format != FormatArrayDataset);
//Conditional row assignment if variable length causes offset to be recalculated outside of the if()
//if (useTempRow && (op == no_if) && isVariableSizeRecord(expr->queryRecord()))
// useTempRow = false;
if (useTempRow)
{
Owned tempRow = declareTempAnonRow(ctx, ctx, expr);
Owned rowBuilder = createRowBuilder(ctx, tempRow);
Owned builder = createSingleRowTempDatasetBuilder(expr->queryRecord(), rowBuilder);
builder->buildDeclare(ctx);
buildDatasetAssign(ctx, builder, expr);
//builder->buildFinish(ctx, tempTarget);
finalizeTempRow(ctx, tempRow, rowBuilder);
convertBoundRowToDataset(ctx, tgt, tempRow, format);
}
else
{
if (!canAssignInline(&ctx, expr))
{
CHqlBoundTarget tempTarget;
createTempFor(ctx, expr->queryType(), tempTarget, typemod_none, format);
buildDatasetAssign(ctx, tempTarget, expr);
tgt.setFromTarget(tempTarget);
//buildTempExpr(ctx, expr, tgt); // can't use this because it causes recursion on no_selectnth
}
else
{
Owned builder;
IHqlExpression * record = expr->queryRecord();
OwnedHqlExpr serializedRecord = getSerializedForm(record);
if (format == FormatNatural)
{
if (record != serializedRecord)
ensureContextAvailable(ctx);
if (!ctx.queryMatchExpr(codeContextMarkerExpr))
{
if (record != serializedRecord)
throwError(HQLERR_LinkedDatasetNoContext);
format = FormatBlockedDataset;
}
else if ((record == serializedRecord) && !options.tempDatasetsUseLinkedRows)
{
format = FormatBlockedDataset;
}
else
{
format = FormatLinkedDataset;
}
}
else if (record != serializedRecord)
format = FormatLinkedDataset; // Have to serialize it later - otherwise it won't be compatible
if (format == FormatLinkedDataset || format == FormatArrayDataset)
{
IHqlExpression * choosenLimit = NULL;
if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
{
choosenLimit = expr->queryChild(1);
expr = expr->queryChild(0);
}
//MORE: Extract limit and choosen and pass as parameters
builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
}
else if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
{
//Build a limited builder - it is likely to be just as efficient, and often much more e.g., choosen(a+b, n)
builder.setown(createChoosenDatasetBuilder(serializedRecord, expr->queryChild(1)));
expr = expr->queryChild(0);
}
else
builder.setown(createBlockedDatasetBuilder(serializedRecord));
builder->buildDeclare(ctx);
buildDatasetAssign(ctx, builder, expr);
builder->buildFinish(ctx, tgt);
}
}
if (singleRow)
tgt.count.setown(getSizetConstant(1));
else if (op == no_inlinetable)
{
IHqlExpression * transforms = expr->queryChild(0);
if (!transformListContainsSkip(transforms))
tgt.count.setown(getSizetConstant(transforms->numChildren()));
}
ctx.associateExpr(expr, tgt);
}
//---------------------------------------------------------------------------
// Dataset assignment - to temp
void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
{
node_operator op = expr->getOperator();
switch (op)
{
case no_call:
case no_externalcall:
if (!hasStreamedModifier(expr->queryType()))
{
doBuildCall(ctx, &target, expr, NULL);
return;
}
break;
case no_getgraphresult:
doBuildAssignGetGraphResult(ctx, target, expr);
return;
case no_workunit_dataset:
case no_getresult:
buildExprAssign(ctx, target, expr);
return;
case no_null:
{
CHqlBoundExpr bound;
buildDataset(ctx, expr, bound, isArrayRowset(expr->queryType()) ? FormatLinkedDataset : FormatBlockedDataset);
OwnedHqlExpr translated = bound.getTranslatedExpr();
if (target.count) ctx.addAssign(target.count, bound.count);
if (target.length) ctx.addAssign(target.length, bound.length);
ctx.addAssign(target.expr, bound.expr);
return;
}
case no_inlinedictionary:
case no_inlinetable:
{
//This will typically generate a loop. If few items then it is more efficient to expand the assigns/clones out.
if (expr->queryChild(0)->numChildren() > INLINE_TABLE_EXPAND_LIMIT)
{
CHqlBoundExpr bound;
if (doBuildDatasetInlineTable(ctx, expr, bound, FormatNatural))
{
OwnedHqlExpr translated = bound.getTranslatedExpr();
buildDatasetAssign(ctx, target, translated);
return;
}
}
break;
}
case no_alias:
{
CHqlBoundExpr bound;
buildDataset(ctx, expr, bound, FormatNatural);
OwnedHqlExpr translated = bound.getTranslatedExpr();
buildDatasetAssign(ctx, target, translated);
return;
}
case no_owned_ds:
{
ITypeInfo * targetType = target.queryType();
if (hasLinkCountedModifier(targetType) && hasWrapperModifier(targetType))
{
CHqlBoundExpr bound;
buildDataset(ctx, expr->queryChild(0), bound, FormatLinkedDataset);
OwnedHqlExpr compound = createValue(no_complex, bound.expr->getType(), LINK(bound.count), LINK(bound.expr));
ctx.addAssign(target.expr, compound);
return;
}
break;
}
case no_compound:
{
buildStmt(ctx, expr->queryChild(0));
buildDatasetAssign(ctx, target, expr->queryChild(1));
return;
}
case no_compound_childread:
case no_compound_childnormalize:
case no_compound_childaggregate:
case no_compound_selectnew:
case no_compound_inline:
case no_distributed:
case no_preservemeta:
case no_sorted:
case no_nofold:
case no_nohoist:
case no_section:
case no_sectioninput:
buildDatasetAssign(ctx, target, expr->queryChild(0));
return;
case no_serialize:
if (isDummySerializeDeserialize(expr))
buildDatasetAssign(ctx, target, expr->queryChild(0)->queryChild(0));
else
buildAssignSerializedDataset(ctx, target, expr->queryChild(0));
return;
case no_deserialize:
if (isDummySerializeDeserialize(expr))
buildDatasetAssign(ctx, target, expr->queryChild(0)->queryChild(0));
else
buildAssignLinkedDataset(ctx, target, expr->queryChild(0));
return;
case no_select:
{
bool isNew;
IHqlExpression * ds = querySelectorDataset(expr, isNew);
if (!isNew || ds->isDatarow())
{
Owned selected = buildReference(ctx, expr);
selected->assignTo(ctx, target);
return;
}
break;
}
case no_typetransfer:
{
IHqlExpression * child = expr->queryChild(0);
if (expr->isDataset() && child->isDataset())
{
//Special case no-op type transfers and assignment from a call returning unknown-type dataset
if (!recordTypesMatch(expr, child) &&
!(child->getOperator() == no_externalcall && recordTypesMatch(child, queryNullRecord())))
{
CHqlBoundExpr bound;
buildDataset(ctx, child, bound, FormatNatural);
ITypeInfo * newType = cloneModifiers(bound.expr->queryType(), expr->queryType());
bound.expr.setown(createValue(no_typetransfer, newType, LINK(bound.expr)));
OwnedHqlExpr translated = bound.getTranslatedExpr();
buildDatasetAssign(ctx, target, translated);
}
else
buildDatasetAssign(ctx, target, child);
return;
}
break;
}
}
if (!canAssignInline(&ctx, expr) && (op != no_translated))
{
buildAssignChildDataset(ctx, target, expr);
return;
}
ITypeInfo * to = target.queryType();
ITypeInfo * exprType = expr->queryType();
bool targetOutOfLine = isArrayRowset(to);
switch (op)
{
case no_limit:
assertex(!expr->hasProperty(skipAtom) && !expr->hasProperty(onFailAtom));
//Do the limit check as a post test.
//It means we may read more records than we need to, but the code is inline, and the code is generally much better.
if (target.count)
{
buildDatasetAssign(ctx, target, expr->queryChild(0));
CHqlBoundExpr bound;
bound.setFromTarget(target);
doBuildCheckDatasetLimit(ctx, expr, bound);
return;
}
break;
case no_translated:
{
bool sourceOutOfLine = isArrayRowset(exprType);
if (sourceOutOfLine != targetOutOfLine)
{
OwnedITypeInfo serializedSourceType = getSerializedForm(exprType);
OwnedITypeInfo serializedTargetType = getSerializedForm(to);
if (queryUnqualifiedType(serializedSourceType) == queryUnqualifiedType(serializedTargetType))
{
if (targetOutOfLine)
{
buildAssignLinkedDataset(ctx, target, expr);
}
else
{
buildAssignSerializedDataset(ctx, target, expr);
}
return;
}
}
break;
}
}
if (recordTypesMatch(to, exprType))
{
switch (op)
{
case no_rows:
{
CHqlBoundExpr bound;
buildDataset(ctx, expr, bound, FormatLinkedDataset);
OwnedHqlExpr translated = bound.getTranslatedExpr();
buildDatasetAssign(ctx, target, translated);
return;
}
case no_select:
{
bool isNew;
IHqlExpression * ds = querySelectorDataset(expr, isNew);
if (isNew && !ds->isDatarow())
break;
}
//fall through
case no_translated:
case no_null:
case no_id2blob:
{
_ATOM func = NULL;
if (!isArrayRowset(to))
{
if (!isArrayRowset(exprType))
func = dataset2DatasetXAtom;
}
else if (hasLinkCountedModifier(to))
{
if (hasLinkCountedModifier(exprType))
{
CHqlBoundExpr bound;
buildDataset(ctx, expr, bound, FormatLinkedDataset);
assertex(bound.count && bound.expr);
if (hasWrapperModifier(to))
{
//assigns to count and rows members
StringBuffer s;
generateExprCpp(s, target.expr);
s.append(".set(");
generateExprCpp(s, bound.count);
s.append(",");
generateExprCpp(s, bound.expr);
s.append(");");
ctx.addQuoted(s);
}
else
{
ctx.addAssign(target.count, bound.count);
HqlExprArray args;
args.append(*LINK(bound.expr));
OwnedHqlExpr call = bindTranslatedFunctionCall(linkRowsetAtom, args);
ctx.addAssign(target.expr, call);
}
return;
}
}
if (func)
{
HqlExprArray args;
args.append(*LINK(expr));
OwnedHqlExpr call = bindFunctionCall(func, args);
buildExprAssign(ctx, target, call);
return;
}
}
}
}
IHqlExpression * record = ::queryRecord(to);
Owned builder;
if (targetOutOfLine)
{
if (target.queryType()->getTypeCode() == type_dictionary)
{
builder.setown(createLinkedDictionaryBuilder(record));
}
else
{
IHqlExpression * choosenLimit = NULL;
if ((op == no_choosen) && !isChooseNAllLimit(expr->queryChild(1)) && !queryRealChild(expr, 2))
{
choosenLimit = expr->queryChild(1);
expr = expr->queryChild(0);
}
builder.setown(createLinkedDatasetBuilder(record, choosenLimit));
}
}
else
builder.setown(createBlockedDatasetBuilder(record));
builder->buildDeclare(ctx);
buildDatasetAssign(ctx, builder, expr);
builder->buildFinish(ctx, target);
}
//---------------------------------------------------------------------------
void HqlCppTranslator::doBuildCheckDatasetLimit(BuildCtx & ctx, IHqlExpression * expr, const CHqlBoundExpr & bound)
{
IHqlExpression * record = expr->queryRecord();
IHqlExpression * limit = expr->queryChild(1);
OwnedHqlExpr test;
if (!bound.count && bound.length && isFixedRecordSize(record))
{
OwnedHqlExpr size = bound.length->queryValue() ? LINK(bound.length) : createTranslated(bound.length);
OwnedHqlExpr maxSize = createValue(no_mul, LINK(sizetType), ensureExprType(limit, sizetType), getSizetConstant(getFixedRecordSize(record)));
test.setown(createBoolExpr(no_gt, ensureExprType(size, sizetType), LINK(maxSize)));
}
else
{
OwnedHqlExpr count = getBoundCount(bound);
OwnedHqlExpr translatedCount = count->queryValue() ? LINK(count) : createTranslated(count);
test.setown(createBoolExpr(no_gt, ensureExprType(translatedCount, sizetType), ensureExprType(limit, sizetType)));
}
OwnedHqlExpr folded = foldHqlExpression(test);
LinkedHqlExpr fail = queryRealChild(expr, 2);
if (folded->queryValue())
{
if (!folded->queryValue()->getBoolValue())
return;
StringBuffer failMessageText;
if (fail)
{
OwnedHqlExpr failMessage = getFailMessage(fail, true);
if (failMessage && failMessage->queryValue())
failMessage->queryValue()->getStringValue(failMessageText);
}
if (failMessageText.length())
WARNING1(HQLWRN_LimitAlwaysExceededX, failMessageText.str());
else
WARNING(HQLWRN_LimitAlwaysExceeded);
}
if (!fail)
fail.setown(createFailAction("Limit exceeded", limit, NULL, queryCurrentActivityId(ctx)));
BuildCtx subctx(ctx);
buildFilter(subctx, folded);
buildStmt(subctx, fail);
}
void HqlCppTranslator::doBuildDatasetLimit(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, ExpressionFormat format)
{
buildDataset(ctx, expr->queryChild(0), tgt, format);
doBuildCheckDatasetLimit(ctx, expr, tgt);
}
bool HqlCppTranslator::doBuildDatasetInlineTable(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, ExpressionFormat format)
{
if (!options.generateStaticInlineTables)
return false;
IHqlExpression * transforms = expr->queryChild(0);
IHqlExpression * record = expr->queryRecord();
if (transforms->numChildren() == 0)
{
OwnedHqlExpr null = createDataset(no_null, LINK(record));
buildDataset(ctx, null, tgt, format);
return true;
}
BuildCtx declareCtx(*code, literalAtom);
//Remove unique id when checking for constant datasets already generated
OwnedHqlExpr exprNoUnique = removeProperty(expr, _uid_Atom);
if (declareCtx.getMatchExpr(exprNoUnique, tgt))
return true;
HqlExprArray rows;
unsigned maxRows = transforms->numChildren();
unsigned row;
for (row = 0; row < maxRows; row++)
{
OwnedHqlExpr constRow = createConstantRowExpr(transforms->queryChild(row));
if (!constRow || !canGenerateStringInline(constRow->queryType()->getSize()))
return false;
rows.append(*constRow.getClear());
}
HqlExprArray boundRows;
ForEachItemIn(i, rows)
{
CHqlBoundExpr bound;
buildConstRow(record, &rows.item(i), bound);
boundRows.append(*bound.expr.getClear());
}
Owned rowType = makeConstantModifier(makeReferenceModifier(makeRowType(LINK(queryRecordType(expr->queryType())))));
OwnedHqlExpr values = createValue(no_list, makeSetType(LINK(rowType)), boundRows);
Owned tableType = makeConstantModifier(makeArrayType(LINK(rowType), maxRows));
OwnedITypeInfo rowsType = makeOutOfLineModifier(makeTableType(LINK(rowType), NULL, NULL, NULL));
OwnedHqlExpr table = declareCtx.getTempDeclare(tableType, values);
if (options.spanMultipleCpp)
{
BuildCtx protoctx(*code, mainprototypesAtom);
protoctx.addDeclareExternal(table);
}
tgt.count.setown(getSizetConstant(maxRows));
tgt.expr.setown(createValue(no_typetransfer, LINK(rowsType), LINK(table)));
declareCtx.associateExpr(exprNoUnique, tgt);
return true;
}
//---------------------------------------------------------------------------
// Dataset creation via builder
void HqlCppTranslator::buildDatasetAssignTempTable(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
{
OwnedHqlExpr values = normalizeListCasts(expr->queryChild(0));
if (values->getOperator() == no_null)
return;
IHqlExpression * record = expr->queryChild(1);
OwnedHqlExpr rowsExpr;
if (values->queryType()->getTypeCode() == type_set)
{
if ((values->getOperator() == no_list) && !values->isConstant())
{
ForEachChild(i, values)
{
BuildCtx loopctx(ctx);
BoundRow * targetRow = target->buildCreateRow(loopctx);
OwnedHqlExpr targetField = createSelectExpr(LINK(targetRow->querySelector()), LINK(record->queryChild(0)));
buildAssign(loopctx, targetField, values->queryChild(i));
target->finishRow(loopctx, targetRow);
}
}
else
{
Owned cursor = createSetSelector(ctx, values);
BuildCtx loopctx(ctx);
CHqlBoundExpr boundCurElement;
cursor->buildIterateLoop(loopctx, boundCurElement, false);
BoundRow * targetRow = target->buildCreateRow(loopctx);
OwnedHqlExpr targetField = createSelectExpr(LINK(targetRow->querySelector()), LINK(record->queryChild(0)));
OwnedHqlExpr value = boundCurElement.getTranslatedExpr();
buildAssign(loopctx, targetField, value);
target->finishRow(loopctx, targetRow);
}
}
else
{
BuildCtx subctx(ctx);
BoundRow * targetRow = target->buildCreateRow(subctx);
Owned targetRef = buildActiveRow(subctx, targetRow->querySelector());
buildRowAssign(subctx, targetRef, values);
target->finishRow(subctx, targetRow);
}
}
void HqlCppTranslator::buildDatasetAssignInlineTable(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
{
IHqlExpression * transforms = expr->queryChild(0);
if (transforms->numChildren() == 0)
return;
unsigned maxRows = transforms->numChildren();
unsigned row;
const bool copyConstantRows = true;//getFieldCount(expr->queryRecord()) > 2;
for (row = 0; row < maxRows; row++)
{
IHqlExpression * transform = transforms->queryChild(row);
OwnedHqlExpr rowValue = createRow(no_createrow, LINK(transform));
BuildCtx subctx(ctx);
CHqlBoundExpr bound;
//Work in progress. Check if there are several fields - otherwise not worth it.s
if (doBuildRowConstantTransform(transform, bound))
{
BoundRow * row = bindConstantRow(subctx, rowValue, bound);
if (target->buildLinkRow(subctx, row))
continue;
}
BoundRow * targetRow = target->buildCreateRow(subctx);
Owned targetRef = buildActiveRow(subctx, targetRow->querySelector());
buildRowAssign(subctx, targetRef, rowValue);
target->finishRow(subctx, targetRow);
}
}
class InlineDatasetSkipCallback : public CInterface, implements IHqlCodeCallback
{
public:
IMPLEMENT_IINTERFACE
virtual void buildCode(HqlCppTranslator & translator, BuildCtx & ctx)
{
ctx.addContinue();
}
};
void HqlCppTranslator::buildDatasetAssignProject(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
{
BuildCtx iterctx(ctx);
IHqlExpression * ds = expr->queryChild(0);
IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
OwnedHqlExpr counterVar;
if (counter)
{
counterVar.setown(iterctx.getTempDeclare(unsignedType, queryZero()));
}
bool containsSkip = transformContainsSkip(expr->queryChild(1));
BoundRow * sourceCursor = buildDatasetIterate(iterctx, ds, containsSkip);
if (counter)
{
iterctx.associateExpr(counter, counterVar);
OwnedHqlExpr inc = createValue(no_postinc, LINK(unsignedType), LINK(counterVar));
iterctx.addExpr(inc);
}
if (sourceCursor)
{
BoundRow * targetRow = target->buildCreateRow(iterctx);
HqlExprAssociation * skipAssociation = NULL;
if (containsSkip)
{
OwnedHqlExpr callback = createUnknown(no_unknown, makeVoidType(), NULL, new InlineDatasetSkipCallback);
skipAssociation = ctx.associateExpr(skipActionMarker, callback);
}
Owned targetRef = buildActiveRow(iterctx, targetRow->querySelector());
switch (expr->getOperator())
{
case no_hqlproject:
doBuildRowAssignProject(iterctx, targetRef, expr);
break;
case no_newusertable:
doBuildRowAssignUserTable(iterctx, targetRef, expr);
break;
}
ctx.removeAssociation(skipAssociation);
target->finishRow(iterctx, targetRow);
}
}
void HqlCppTranslator::buildDatasetAssignJoin(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
{
IHqlExpression * left = expr->queryChild(0);
IHqlExpression * right = expr->queryChild(1);
IHqlExpression * cond = expr->queryChild(2);
IHqlExpression * selSeq = querySelSeq(expr);
bool leftOuter = expr->hasProperty(leftonlyAtom) || expr->hasProperty(leftouterAtom);
CHqlBoundExpr nullRhs;
if (leftOuter)
buildDefaultRow(ctx, right, nullRhs);
BuildCtx leftIterCtx(ctx);
BoundRow * leftCursor = buildDatasetIterate(leftIterCtx, left, false);
bindTableCursor(leftIterCtx, left, leftCursor->queryBound(), no_left, selSeq);
OwnedHqlExpr matchedAnyVar;
if (leftOuter)
matchedAnyVar.setown(leftIterCtx.getTempDeclare(queryBoolType(), queryBoolExpr(false)));
BuildCtx rightIterCtx(leftIterCtx);
BoundRow * rightCursor = buildDatasetIterate(rightIterCtx, right, false);
bindTableCursor(rightIterCtx, right, rightCursor->queryBound(), no_right, selSeq);
OwnedHqlExpr cseCond = options.spotCSE ? spotScalarCSE(cond) : LINK(cond);
buildFilter(rightIterCtx, cseCond);
if (!expr->hasProperty(leftonlyAtom))
{
BoundRow * targetRow = target->buildCreateRow(rightIterCtx);
Owned targetRef = buildActiveRow(rightIterCtx, targetRow->querySelector());
OwnedHqlExpr rowValue = createRow(no_createrow, LINK(expr->queryChild(3)));
buildRowAssign(rightIterCtx, targetRef, rowValue);
target->finishRow(rightIterCtx, targetRow);
}
if (matchedAnyVar)
{
buildAssignToTemp(rightIterCtx, matchedAnyVar, queryBoolExpr(true));
OwnedHqlExpr test = getInverse(matchedAnyVar);
leftIterCtx.addFilter(test);
OwnedHqlExpr defaultRowPtr = getPointer(nullRhs.expr);
bindTableCursor(leftIterCtx, right, defaultRowPtr, no_right, selSeq);
BoundRow * targetRow = target->buildCreateRow(leftIterCtx);
Owned targetRef = buildActiveRow(leftIterCtx, targetRow->querySelector());
OwnedHqlExpr rowValue = createRow(no_createrow, LINK(expr->queryChild(3)));
buildRowAssign(leftIterCtx, targetRef, rowValue);
target->finishRow(leftIterCtx, targetRow);
}
}
void HqlCppTranslator::buildDatasetAssignAggregate(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * expr)
{
BuildCtx subctx(ctx);
subctx.addGroup();
BoundRow * targetRow = target->buildCreateRow(subctx);
Owned targetRef = buildActiveRow(subctx, targetRow->querySelector());
doBuildRowAssignAggregate(subctx, targetRef, expr);
target->finishRow(subctx, targetRow);
}
void HqlCppTranslator::buildDatasetAssign(BuildCtx & ctx, IHqlCppDatasetBuilder * target, IHqlExpression * _expr)
{
OwnedHqlExpr expr = forceInlineAssignDataset(ctx, _expr);
bool isRowAssign = false;
BuildCtx subctx(ctx);
switch (expr->getOperator())
{
case no_addfiles:
buildDatasetAssign(subctx, target, expr->queryChild(0));
buildDatasetAssign(subctx, target, expr->queryChild(1));
return;
case no_temptable:
buildDatasetAssignTempTable(subctx, target, expr);
//MORE: Create rows and assign each one in turn. Could possibly be done with a different dataset selector
return;
case no_inlinedictionary:
case no_inlinetable:
buildDatasetAssignInlineTable(subctx, target, expr);
return;
case no_xmlproject:
buildDatasetAssignXmlProject(subctx, target, expr);
return;
case no_datasetfromrow:
{
isRowAssign = true;
expr.set(expr->queryChild(0));
break;
}
case no_if:
{
CHqlBoundExpr bound;
buildExpr(subctx, expr->queryChild(0), bound);
IHqlStmt * filter = subctx.addFilter(bound.expr);
buildDatasetAssign(subctx, target, expr->queryChild(1));
IHqlExpression * elseExpr = expr->queryChild(2);
if (elseExpr && elseExpr->getOperator() != no_null)
{
subctx.selectElse(filter);
buildDatasetAssign(subctx, target, elseExpr);
}
}
return;
case no_null:
return;
case no_activetable:
case no_temprow:
case no_createrow:
case no_projectrow:
case no_typetransfer:
isRowAssign = true;
break;
case no_newaggregate:
buildDatasetAssignAggregate(subctx, target, expr);
return;
case no_hqlproject:
case no_newusertable:
buildDatasetAssignProject(subctx, target, expr);
return;
case no_compound_childread:
case no_compound_childnormalize:
case no_compound_childaggregate:
case no_compound_selectnew:
case no_compound_inline:
case no_distributed:
case no_preservemeta:
case no_sorted:
case no_nofold:
case no_nohoist:
case no_section:
case no_sectioninput:
buildDatasetAssign(subctx, target, expr->queryChild(0));
return;
case no_alias_scope:
// expandAliasScope(subctx, expr);
buildDatasetAssign(subctx, target, expr->queryChild(0));
return;
case no_filter:
{
//We want to evaluate invariant conditions outside of the loop, rather than inside the dataset assignment
//Currently the test is whether the expression is independent of any tables. Better would be to
//see if the test was dependent on any of the datasets introduced by expr->queryChild(0).
HqlExprArray conds;
unwindFilterConditions(conds, expr);
IHqlExpression * ds = expr->queryChild(0);
#if 0
HqlExprCopyArray selectors;
loop
{
selectors.append(*ds->queryNormalizedSelector());
IHqlExpression * root = queryRoot(expr);
if (!root || root->getOperator() != no_select)
break;
bool isNew;
ds = querySelectorDataset(root, isNew);
if (!isNew)
break;
}
#endif
unsigned max = conds.ordinality();
unsigned i = 0;
bool optimized = false;
while (i < max)
{
IHqlExpression & cur = conds.item(i);
#if 0
bool overlap = false;
ForEachItemIn(j, selectors)
{
if (containsSelector(&cur, &selectors.item(j)))
{
overlap = true;
break;
}
}
#else
bool overlap = containsSelector(&cur, ds->queryNormalizedSelector());
#endif
if (!overlap)
{
buildFilter(subctx, &cur);
conds.remove(i);
optimized = true;
max--;
}
else
i++;
}
if (max == 0)
{
buildDatasetAssign(subctx, target, ds);
return;
}
if (optimized)
{
conds.add(*LINK(ds), 0);
expr.setown(expr->clone(conds));
}
break;
}
case no_join:
buildDatasetAssignJoin(subctx, target, expr);
return;
}
ITypeInfo * type = expr->queryType();
if (type)
{
switch (type->getTypeCode())
{
case type_record: // e.g. dataset.recordfield
case type_row:
isRowAssign = true;
break;
}
}
if (isRowAssign)
{
bool done = false;
subctx.addGroup();
//Some code primarily here to improve the generated code for productions inside parse statements for text parsing.
//see if we can replace a memcpy of the child record with a link...
if (!target->isRestricted())
{
switch (expr->getOperator())
{
case no_matchattr:
case no_left:
case no_right:
{
//Only try for really simple cases...
Owned sourceRef = buildNewRow(subctx, expr);
Owned sourceRow = sourceRef->getRow(subctx);
if (!target->buildLinkRow(subctx, sourceRow))
{
BoundRow * targetRow = target->buildCreateRow(subctx);
Owned targetRef = buildActiveRow(subctx, targetRow->querySelector());
buildRowAssign(subctx, targetRef, sourceRef);
target->finishRow(subctx, targetRow);
}
done = true;
break;
}
}
}
if (!done)
{
BoundRow * match = static_cast(ctx.queryAssociation(expr, AssocRow, NULL));
if (match && target->buildLinkRow(subctx, match))
done = true;
}
if (!done)
{
BoundRow * targetRow = target->buildCreateRow(subctx);
Owned targetRef = buildActiveRow(subctx, targetRow->querySelector());
buildRowAssign(subctx, targetRef, expr);
target->finishRow(subctx, targetRow);
}
}
else
{
if (!target->buildAppendRows(subctx, expr))
{
BoundRow * sourceRow = buildDatasetIterate(subctx, expr, false);
if (sourceRow)
{
if (!target->buildLinkRow(subctx, sourceRow))
{
BoundRow * targetRow = target->buildCreateRow(subctx);
Owned targetRef = buildActiveRow(subctx, targetRow->querySelector());
Owned sourceRef = buildActiveRow(subctx, sourceRow->querySelector());
buildRowAssign(subctx, targetRef, sourceRef);
target->finishRow(subctx, targetRow);
}
}
}
}
}
//---------------------------------------------------------------------------
// Dataset iteration
BoundRow * HqlCppTranslator::buildDatasetIterateSelectN(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
OwnedHqlExpr counter = ctx.getTempDeclare(sizetType, NULL);
buildAssignToTemp(ctx, counter, expr->queryChild(1));
BoundRow * cursor = buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
if (cursor)
{
OwnedHqlExpr dec = createValue(no_predec, counter->getType(), LINK(counter));
OwnedHqlExpr test = createBoolExpr(no_eq, LINK(dec), getSizetConstant(0));
ctx.addFilter(test);
}
return cursor;
}
BoundRow * HqlCppTranslator::buildDatasetIterateChoosen(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
OwnedHqlExpr counter = ctx.getTempDeclare(sizetType, queryZero());
CHqlBoundExpr boundLow, boundHigh;
OwnedHqlExpr foldedHigh = foldHqlExpression(expr->queryChild(1));
if (!isChooseNAllLimit(foldedHigh))
buildSimpleExpr(ctx, foldedHigh, boundHigh);
if (queryRealChild(expr, 2))
{
OwnedHqlExpr foldedLow = foldHqlExpression(expr->queryChild(2));
OwnedHqlExpr low = adjustValue(foldedLow, -1);
buildSimpleExpr(ctx, low, boundLow);
if (boundHigh.expr)
boundHigh.expr.setown(createValue(no_add, LINK(boundHigh.queryType()), LINK(boundHigh.expr), LINK(boundLow.expr)));
}
BoundRow * cursor = buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
if (cursor)
{
OwnedHqlExpr inc = createValue(no_postinc, counter->getType(), LINK(counter));
ctx.addExpr(inc);
OwnedHqlExpr cond;
if (boundLow.expr)
extendConditionOwn(cond, no_and, createBoolExpr(no_gt, LINK(counter), LINK(boundLow.expr)));
if (boundHigh.expr)
extendConditionOwn(cond, no_and, createBoolExpr(no_le, LINK(counter), LINK(boundHigh.expr)));
if (cond)
ctx.addFilter(cond);
}
return cursor;
}
BoundRow * HqlCppTranslator::buildDatasetIterateLimit(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
OwnedHqlExpr counter = ctx.getTempDeclare(sizetType, queryZero());
CHqlBoundExpr boundHigh;
OwnedHqlExpr foldedHigh = foldHqlExpression(expr->queryChild(1));
buildSimpleExpr(ctx, foldedHigh, boundHigh);
BoundRow * cursor = buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
if (cursor)
{
OwnedHqlExpr inc = createValue(no_preinc, counter->getType(), LINK(counter));
OwnedHqlExpr cond = createBoolExpr(no_gt, LINK(inc), LINK(boundHigh.expr));
BuildCtx subctx(ctx);
subctx.addFilter(cond);
LinkedHqlExpr fail = expr->queryChild(2);
if (!fail || fail->isAttribute())
fail.setown(createFailAction("Limit exceeded", foldedHigh, NULL, queryCurrentActivityId(ctx)));
buildStmt(subctx, fail);
}
return cursor;
}
BoundRow * HqlCppTranslator::buildDatasetIterateProject(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
IHqlExpression * dataset = expr->queryChild(0);
OwnedHqlExpr counterVar;
IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
if (counter)
{
counterVar.setown(ctx.getTempDeclare(unsignedType, queryZero()));
}
bool containsSkip = transformContainsSkip(expr->queryChild(1));
if (containsSkip)
needToBreak = true;
buildDatasetIterate(ctx, dataset, needToBreak);
Owned tempRow = declareTempAnonRow(ctx, ctx, expr);
if (counter)
{
ctx.associateExpr(counter, counterVar);
OwnedHqlExpr inc = createValue(no_postinc, LINK(unsignedType), LINK(counterVar));
ctx.addExpr(inc);
}
Owned rowBuilder = createRowBuilder(ctx, tempRow);
OwnedHqlExpr leftSelect = createSelector(no_left, dataset, querySelSeq(expr));
OwnedHqlExpr transform = replaceSelector(expr->queryChild(1), leftSelect, dataset->queryNormalizedSelector());
HqlExprAssociation * skipAssociation = NULL;
if (containsSkip)
{
OwnedHqlExpr callback = createUnknown(no_unknown, makeVoidType(), NULL, new InlineDatasetSkipCallback);
skipAssociation = ctx.associateExpr(skipActionMarker, callback);
}
doTransform(ctx, transform, rowBuilder);
ctx.removeAssociation(skipAssociation); //remove it in case keeping hold of it causes issues.
finalizeTempRow(ctx, tempRow, rowBuilder);
return bindTableCursor(ctx, expr, tempRow->queryBound());
}
BoundRow * HqlCppTranslator::buildDatasetIterateUserTable(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
IHqlExpression * dataset = expr->queryChild(0);
buildDatasetIterate(ctx, dataset, needToBreak);
Owned tempRow = declareTempAnonRow(ctx, ctx, expr);
Owned rowBuilder = createRowBuilder(ctx, tempRow);
doTransform(ctx, expr->queryChild(2), rowBuilder);
finalizeTempRow(ctx, tempRow, rowBuilder);
return bindTableCursor(ctx, expr, tempRow->queryBound());
}
BoundRow * HqlCppTranslator::buildDatasetIterateSpecialTempTable(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
IHqlExpression * values = expr->queryChild(0);
bool requiresTempRow = false;
ITypeInfo * type = values->queryType()->queryChildType();
switch (type->getTypeCode())
{
case type_alien:
case type_bitfield:
requiresTempRow = true;
break;
default:
if (type->getSize() == UNKNOWN_LENGTH)
requiresTempRow = true;
break;
}
Owned cursor = createSetSelector(ctx, values);
CHqlBoundExpr boundCurElement;
cursor->buildIterateLoop(ctx, boundCurElement, false);
OwnedHqlExpr address = getPointer(boundCurElement.expr);
if (requiresTempRow)
{
Owned tempRow = declareTempAnonRow(ctx, ctx, expr);
Owned rowBuilder = createRowBuilder(ctx, tempRow);
IHqlExpression * record = expr->queryRecord();
OwnedHqlExpr target = createSelectExpr(LINK(rowBuilder->querySelector()), LINK(record->queryChild(0)));
OwnedHqlExpr curValue = boundCurElement.getTranslatedExpr();
buildAssign(ctx, target, curValue);
finalizeTempRow(ctx, tempRow, rowBuilder);
return bindTableCursor(ctx, expr, tempRow->queryBound());
}
else
{
assertex(!boundCurElement.length && !boundCurElement.count);
address.setown(createValue(no_implicitcast, makeRowReferenceType(expr), LINK(address)));
return bindTableCursor(ctx, expr, address);
}
}
BoundRow * HqlCppTranslator::buildDatasetIterateStreamedCall(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
CHqlBoundExpr bound;
doBuildExprCall(ctx, expr, bound);
ITypeInfo * exprType = expr->queryType();
Owned wrappedType = makeWrapperModifier(LINK(exprType));
OwnedHqlExpr temp = ctx.getTempDeclare(wrappedType, bound.expr);
ctx.addLoop(NULL, NULL, false);
Owned wrappedRowType = makeWrapperModifier(LINK(queryRowType(exprType)));
OwnedHqlExpr tempRow = ctx.getTempDeclare(wrappedRowType, NULL);
StringBuffer s;
generateExprCpp(s, tempRow).append(".setown(");
generateExprCpp(s, temp).append("->nextRow());");
ctx.addQuoted(s);
s.clear().append("if (!");generateExprCpp(s, tempRow).append(".getbytes()) break;");
ctx.addQuoted(s);
return bindTableCursor(ctx, expr, tempRow);
}
BoundRow * HqlCppTranslator::buildDatasetIterate(BuildCtx & ctx, IHqlExpression * expr, bool needToBreak)
{
if (!canProcessInline(&ctx, expr))
{
Owned cursor = createDatasetSelector(ctx, expr);
return cursor->buildIterateLoop(ctx, needToBreak);
}
switch (expr->getOperator())
{
case no_dataset_alias:
if (!expr->hasProperty(_normalized_Atom))
{
OwnedHqlExpr uniqueChild = normalizeDatasetAlias(expr);
BoundRow * childCursor = buildDatasetIterate(ctx, uniqueChild, needToBreak);
return rebindTableCursor(ctx, expr, childCursor, no_none, NULL);
}
else
{
throwUnexpected();
//The following would only be triggered for a splitter (not yet generated), and that would require
//disambiguation when that was built.
BoundRow * childCursor = buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
return rebindTableCursor(ctx, expr, childCursor, no_none, NULL);
}
case no_null:
buildFilter(ctx, queryBoolExpr(false));
return NULL;
case no_filter:
{
IHqlExpression * dataset = expr->queryChild(0);
#ifdef _OPTIMZE_INLINE_FILTERS_
//Good code, but messes up accidental cse in some createSegmentMonitor calls.
HqlExprAttr invariant;
OwnedHqlExpr cond = extractFilterConditions(invariant, expr, dataset);
if (invariant)
buildFilter(ctx, invariant);
//MORE: if (canAssignInline(ctx, ds) && !canIterateInline(ctx, ds)) break;
BoundRow * cursor = buildDatasetIterate(ctx, dataset, needToBreak);
if (cond)
buildFilter(ctx, cond);
#else
//MORE: if (canAssignInline(ctx, ds) && !canIterateInline(ctx, ds)) break;
BoundRow * cursor = buildDatasetIterate(ctx, dataset, needToBreak);
unsigned max = expr->numChildren();
for (unsigned i=1; i < max; i++)
buildFilter(ctx, expr->queryChild(i));
#endif
return cursor;
}
case no_id2blob:
case no_select:
case no_translated:
{
Owned cursor = createDatasetSelector(ctx, expr);
return cursor->buildIterateLoop(ctx, needToBreak);
}
case no_choosen:
return buildDatasetIterateChoosen(ctx, expr, needToBreak);
case no_limit:
return buildDatasetIterateLimit(ctx, expr, needToBreak);
case no_index:
case no_selectnth:
return buildDatasetIterateSelectN(ctx, expr, needToBreak);
case no_hqlproject:
return buildDatasetIterateProject(ctx, expr, needToBreak);
case no_newusertable:
return buildDatasetIterateUserTable(ctx, expr, needToBreak);
case no_compound_childread:
case no_compound_childnormalize:
case no_compound_childaggregate:
case no_compound_selectnew:
case no_compound_inline:
case no_distributed:
case no_preservemeta:
case no_sorted:
case no_nofold:
case no_nohoist:
return buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
case no_sectioninput:
case no_section:
{
BoundRow * row = buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
#ifdef _DEBUG
StringBuffer s;
if (expr->getOperator() == no_section)
s.append("//---- section ");
else
s.append("//---- end section ");
getStringValue(s, expr->queryChild(1), "").append(" ");
getStringValue(s, expr->queryChild(2)).append("----");
ctx.addQuoted(s);
#endif
return row;
}
case no_alias_scope:
// expandAliasScope(ctx, expr);
return buildDatasetIterate(ctx, expr->queryChild(0), needToBreak);
case no_temptable:
{
IHqlExpression * values = expr->queryChild(0);
if (values->queryType()->getTypeCode() == type_set)
{
if (values->getOperator() == no_alias)
values = values->queryChild(0);
bool special = false;
switch (values->getOperator())
{
case no_getresult:
case no_null:
special = true;
break;
}
if (special)
return buildDatasetIterateSpecialTempTable(ctx, expr, needToBreak);
}
break;
}
case no_call:
case no_externalcall:
if (hasStreamedModifier(expr->queryType()))
return buildDatasetIterateStreamedCall(ctx, expr, needToBreak);
break;
#if 0
//Following should improve the code, but I'm not sure how to correctly convert a referenceSelector to a boundExpr (since it may be with an existing row)
case no_datasetfromrow:
if (!needToBreak)
{
BoundRow * row = buildNewRow(ctx, expr->queryChild(0));
bindTableCursor(ctx, expr, tempRow->queryBound(), no_none, NULL);
return row;
}
break;
#endif
}
Owned cursor = createDatasetSelector(ctx, expr);
return cursor->buildIterateLoop(ctx, needToBreak);
}
//---------------------------------------------------------------------------
// Row Assignment
void HqlCppTranslator::buildCompoundAssign(BuildCtx & ctx, IHqlExpression * left, IReferenceSelector * leftSelector, IHqlExpression * rightScope, IHqlExpression * rightSelector)
{
switch (left->getOperator())
{
case no_ifblock:
{
BuildCtx subctx(ctx);
OwnedHqlExpr test = replaceSelector(left->queryChild(0), querySelfReference(), leftSelector->queryExpr());
buildFilter(subctx, test);
buildCompoundAssign(subctx, left->queryChild(1), leftSelector, rightScope, rightSelector);
//This calculates the size of the previous block. It means that subsequent uses of the
//offsets are cached - even if they are inside another ifblock().
CHqlBoundExpr bound;
IHqlExpression * mapexpr = createSelectExpr(LINK(leftSelector->queryExpr()), LINK(left));
OwnedHqlExpr size = createValue(no_sizeof, makeIntType(4,false), mapexpr);
buildCachedExpr(ctx, size, bound);
}
break;
case no_field:
{
Owned selectedLeft = leftSelector->select(ctx, left);
OwnedHqlExpr selectedRight;
IHqlExpression * leftRecord = ::queryRecord(leftSelector->queryType());
if (!rightScope || (leftRecord==rightScope))
{
selectedRight.setown(createSelectExpr(LINK(rightSelector), LINK(left)));
}
else
{
IHqlSimpleScope * scope = rightScope->querySimpleScope();
IHqlExpression * resolved = scope->lookupSymbol(left->queryName());
assertex(resolved);
selectedRight.setown(createSelectExpr(LINK(rightSelector), resolved));
}
if (left->queryType()->getTypeCode() == type_row)
buildCompoundAssign(ctx, left->queryRecord(), selectedLeft, selectedRight->queryRecord(), selectedRight);
else
selectedLeft->set(ctx, selectedRight);
break;
}
break;
case no_record:
{
ForEachChild(i, left)
buildCompoundAssign(ctx, left->queryChild(i), leftSelector, rightScope, rightSelector);
break;
}
case no_attr:
case no_attr_expr:
case no_attr_link:
break;
default:
UNIMPLEMENTED;
}
}
void HqlCppTranslator::buildCompoundAssign(BuildCtx & ctx, IHqlExpression * left, IHqlExpression * right)
{
//MORE: May need to resolve lhs and rhs in scope to find out the record type.
// buildCompoundAssign(ctx, column->queryRecord(), selector->queryExpr(), value->queryRecord(), value);
UNIMPLEMENTED;
}
void HqlCppTranslator::doBuildRowAssignAggregateClear(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
IHqlExpression * transform = expr->queryChild(2);
unsigned numAggregates = transform->numChildren();
unsigned idx;
OwnedHqlExpr self = getSelf(expr);
for (idx = 0; idx < numAggregates; idx++)
{
IHqlExpression * cur = transform->queryChild(idx);
Owned curTarget = createSelfSelect(ctx, target, cur->queryChild(0), self);
IHqlExpression * src = cur->queryChild(1);
switch (src->getOperator())
{
case no_countgroup:
case no_maxgroup:
case no_mingroup:
case no_sumgroup:
case no_existsgroup:
curTarget->buildClear(ctx, 0);
break;
default:
if (src->isConstant())
curTarget->set(ctx, src);
else
curTarget->buildClear(ctx, 0);
break;
}
}
}
void HqlCppTranslator::doBuildRowAssignAggregateNext(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr, bool isSingleExists, IHqlExpression * alreadyDoneExpr)
{
IHqlExpression * transform = expr->queryChild(2);
unsigned numAggregates = transform->numChildren();
OwnedHqlExpr self = getSelf(expr);
OwnedHqlExpr notAlreadyDone = alreadyDoneExpr ? getInverse(alreadyDoneExpr) : NULL;
bool isVariableOffset = false;
for (unsigned idx = 0; idx < numAggregates; idx++)
{
IHqlExpression * cur = transform->queryChild(idx);
if (cur->isAttribute())
continue;
IHqlExpression * targetSelect = cur->queryChild(0);
Owned curTarget = createSelfSelect(ctx, target, targetSelect, self);
IHqlExpression * src = cur->queryChild(1);
IHqlExpression * arg = src->queryChild(0);
IHqlExpression * cond = src->queryChild(1);
BuildCtx condctx(ctx);
node_operator srcOp = src->getOperator();
switch (srcOp)
{
case no_countgroup:
{
assertex(!(arg && isVariableOffset));
if (arg)
buildFilter(condctx, arg);
OwnedHqlExpr one = getSizetConstant(1);
if (isVariableOffset)
{
IHqlStmt * ifStmt = condctx.addFilter(notAlreadyDone);
curTarget->set(condctx, one);
condctx.selectElse(ifStmt);
}
buildIncrementAssign(condctx, curTarget, one);
}
break;
case no_sumgroup:
{
assertex(!(cond && isVariableOffset));
if (cond)
buildFilter(condctx, cond);
if (isVariableOffset)
{
IHqlStmt * ifStmt = condctx.addFilter(notAlreadyDone);
curTarget->set(condctx, arg);
condctx.selectElse(ifStmt);
}
buildIncrementAssign(condctx, curTarget, arg);
}
break;
case no_maxgroup:
case no_mingroup:
{
node_operator op = (srcOp == no_maxgroup) ? no_gt : no_lt;
assertex(!cond);
OwnedHqlExpr castArg = ensureExprType(arg, targetSelect->queryType()); // cast to correct type, assume it can fit in the target type.
OwnedHqlExpr temp = buildSimplifyExpr(condctx, castArg);
OwnedHqlExpr compare = createBoolExpr (op, LINK(temp), LINK(curTarget->queryExpr()));
if (notAlreadyDone)
compare.setown(createBoolExpr(no_or, LINK(notAlreadyDone), compare.getClear()));
buildFilter(condctx, compare);
curTarget->set(condctx, temp);
}
break;
case no_existsgroup:
assertex(!(arg && isVariableOffset));
if (arg)
buildFilter(condctx, arg);
curTarget->set(condctx, queryBoolExpr(true));
if (isSingleExists)
condctx.addBreak();
break;
default:
if (!src->isConstant() || isVariableOffset)
{
condctx.addFilter(notAlreadyDone);
curTarget->set(condctx, src);
}
break;
}
if (targetSelect->queryType()->getSize() == UNKNOWN_LENGTH)
isVariableOffset = true;
}
if (alreadyDoneExpr)
buildAssignToTemp(ctx, alreadyDoneExpr, queryBoolExpr(true));
}
void HqlCppTranslator::doBuildRowAssignAggregate(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
IHqlExpression * dataset = expr->queryChild(0);
IHqlExpression * transform = expr->queryChild(2);
unsigned numAggregates = transform->numChildren();
if (isKeyedCountAggregate(expr))
throwError(HQLERR_KeyedCountNonKeyable);
bool needGuard = false;
bool isSingleExists = true;
unsigned idx;
for (idx = 0; idx < numAggregates; idx++)
{
IHqlExpression * cur = transform->queryChild(idx);
if (cur->isAttribute())
continue;
IHqlExpression * tgt = cur->queryChild(0);
IHqlExpression * src = cur->queryChild(1);
switch (src->getOperator())
{
case no_countgroup:
case no_sumgroup:
isSingleExists = false;
break;
case no_existsgroup:
break;
case no_mingroup:
isSingleExists = false;
needGuard = true;
break;
case no_maxgroup:
isSingleExists = false;
if (!isNullValueMinimumValue(src->queryType()))
needGuard = true;
break;
default:
isSingleExists = false;
if (!src->isConstant())
needGuard = true;
break;
}
if ((tgt->queryType()->getSize() == UNKNOWN_LENGTH) && (idx+1 != numAggregates))
needGuard = true;
}
OwnedHqlExpr guard;
if (needGuard)
{
Owned boolType = makeBoolType();
guard.setown(ctx.getTempDeclare(boolType, queryBoolExpr(false)));
}
doBuildRowAssignAggregateClear(ctx, target, expr);
BuildCtx condctx(ctx);
BoundRow * cursor = buildDatasetIterate(condctx, dataset, isSingleExists);
doBuildRowAssignAggregateNext(condctx, target, expr, isSingleExists, guard);
}
void HqlCppTranslator::doBuildRowAssignProject(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
IHqlExpression * dataset = expr->queryChild(0);
IHqlExpression * counter = queryPropertyChild(expr, _countProject_Atom, 0);
if (counter && !ctx.queryMatchExpr(counter))
throwError(HQLERR_CounterNotFound);
BuildCtx subctx(ctx);
assertex(target->isRoot());
IHqlExpression * selSeq = querySelSeq(expr);
OwnedHqlExpr leftSelect = createSelector(no_left, dataset, selSeq);
OwnedHqlExpr activeDataset = ensureActiveRow(dataset->queryNormalizedSelector());
OwnedHqlExpr transform = queryNewReplaceSelector(expr->queryChild(1), leftSelect, activeDataset);
Owned selfCursor;
if (!transform)
{
subctx.addGroup();
selfCursor.set(bindSelectorAsSelf(subctx, target, expr));
//Mapping may potentially be ambiguous, so do things correctly (see hqlsource for details)
BoundRow * prevCursor = resolveSelectorDataset(subctx, dataset->queryNormalizedSelector());
transform.set(expr->queryChild(1));
bindTableCursor(subctx, dataset, prevCursor->queryBound(), no_left, selSeq);
}
else
{
//Not introducing any new left rows => no problem assigning to target selector
selfCursor.setown(target->getRow(subctx));
}
doTransform(subctx, transform, selfCursor);
}
void HqlCppTranslator::doBuildRowAssignCreateRow(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
IHqlExpression * transform = expr->queryChild(0);
if (transform->isConstant())
{
#ifdef USE_CONSTANT_ROW_FOR_ASSIGN
//Generally not worthwhile - unless maybe it has a large number of fields....
CHqlBoundExpr bound;
if (doBuildRowConstantTransform(transform, bound))
{
OwnedHqlExpr raw = bound.getTranslatedExpr();
buildRowAssign(ctx, target, raw);
return;
}
#endif
}
Owned selfCursor = target->getRow(ctx);
doTransform(ctx, transform, selfCursor);
}
void HqlCppTranslator::doBuildRowAssignNullRow(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
target->buildClear(ctx, 0);
}
void HqlCppTranslator::doBuildRowAssignProjectRow(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
IHqlExpression * srcRow = expr->queryChild(0);
IHqlExpression * transform = expr->queryChild(1);
Owned source = buildNewRow(ctx, srcRow);
BuildCtx subctx(ctx);
OwnedHqlExpr leftSelect = createSelector(no_left, srcRow, querySelSeq(expr));
OwnedHqlExpr newTransform = replaceSelector(transform, leftSelect, srcRow);
Owned selfCursor = target->getRow(subctx);
doTransform(subctx, newTransform, selfCursor);
}
void HqlCppTranslator::doBuildRowAssignSerializeRow(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
IHqlExpression * srcRow = expr->queryChild(0);
Owned source = buildNewRow(ctx, srcRow);
BuildCtx subctx(ctx);
Owned leftCursor = source->getRow(subctx);
BoundRow * selfCursor = bindSelectorAsSelf(subctx, target, expr);
IHqlExpression * unserializedRecord = srcRow->queryRecord();
//If builder isn't provided then the target must be a fixed size record that doesn't require serialization
//Therefore this should never be called.
assertex(selfCursor->queryBuilder());
{
HqlExprArray args;
args.append(*createRowSerializer(ctx, unserializedRecord, serializerAtom));
args.append(*LINK(srcRow));
Owned type = makeTransformType(expr->queryRecord()->getType());
OwnedHqlExpr call = bindFunctionCall(rtlSerializeToBuilderAtom, args, type);
doTransform(subctx, call, selfCursor);
//MORE: This doesn't associated the returned size with the target if assigned to a child field.
//very unusual code, so not too concerned.
}
subctx.removeAssociation(selfCursor);
}
void HqlCppTranslator::doBuildRowAssignUserTable(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
Owned selfCursor = target->getRow(ctx);
doTransform(ctx, expr->queryChild(2), selfCursor);
}
void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * target, IHqlExpression * expr)
{
switch (expr->getOperator())
{
case no_temprow:
throwUnexpected();
case no_projectrow:
doBuildRowAssignProjectRow(ctx, target, expr);
return;
case no_createrow:
doBuildRowAssignCreateRow(ctx, target, expr);
return;
case no_null:
doBuildRowAssignNullRow(ctx, target, expr);
return;
case no_serialize:
if (isDummySerializeDeserialize(expr))
buildRowAssign(ctx, target, expr->queryChild(0)->queryChild(0));
else
doBuildRowAssignSerializeRow(ctx, target, expr);
return;
case no_if:
{
OwnedHqlExpr foldedCond = foldHqlExpression(expr->queryChild(0));
if (foldedCond->queryValue())
{
unsigned branch = (foldedCond->queryValue()->getBoolValue()) ? 1 : 2;
buildRowAssign(ctx, target, expr->queryChild(branch));
return;
}
//Assigning a variable size record can mean that references to self need recalculating outside of the condition,
//producing poor code.
if (!isVariableSizeRecord(expr->queryRecord()))
{
BuildCtx condctx(ctx);
IHqlStmt * cond = buildFilterViaExpr(condctx, foldedCond);
buildRowAssign(condctx, target, expr->queryChild(1));
condctx.selectElse(cond);
buildRowAssign(condctx, target, expr->queryChild(2));
return;
}
}
break;
/*
case no_externalcall:
//MORE: Should assign directly to the target, but may not be very easy....
if (target->isBinary() && queryRecord(source->queryType()) == expr->queryRecord())
{
CHqlBoundExpr address;
target->buildAddress(ctx, address);
row.setown(createValue(no_typetransfer, makeReferenceModifier(LINK(selector->queryType())), LINK(address.expr)));
doBuildCall(ctx, &target, expr, NULL);
}
break;
*/
case no_comma:
case no_compound:
buildStmt(ctx, expr->queryChild(0));
buildRowAssign(ctx, target, expr->queryChild(1));
return;
}
Owned src = buildNewRow(ctx, expr);
buildRowAssign(ctx, target, src);
}
void HqlCppTranslator::buildRowAssign(BuildCtx & ctx, IReferenceSelector * target, IReferenceSelector * source)
{
BuildCtx subctx(ctx);
IHqlExpression * sourceRecord = ::queryRecord(source->queryType());
IHqlExpression * targetRecord = ::queryRecord(target->queryType());
//if record structures are identical, then we must just be able to block copy the information across.
bool useMemcpy = (sourceRecord == targetRecord) && source->isBinary() && !source->isConditional() &&
!recordRequiresSerialization(sourceRecord);
if (useMemcpy)
{
if (source->queryRootRow()->isConditional())
{
IHqlStmt * ifStmt = subctx.addFilter(source->queryRootRow()->queryBound());
target->setRow(subctx, source);
subctx.selectElse(ifStmt);
target->buildClear(subctx, 0);
}
else
target->setRow(subctx, source);
}
else
buildCompoundAssign(subctx, targetRecord, target, sourceRecord, source->queryExpr());
}
//---------------------------------------------------------------------------
// Dataset selection
IReferenceSelector * HqlCppTranslator::doBuildRowSelectTop(BuildCtx & ctx, IHqlExpression * expr)
{
//create a temporary
Owned rowType = makeReferenceModifier(expr->getType());
OwnedHqlExpr rowExpr = ctx.getTempDeclare(rowType, NULL);
Owned row = createBoundRow(expr, rowExpr);
ctx.associate(*row); // associate here because it is compared inside the loop
CHqlBoundExpr boundCleared;
#ifdef CREATE_DEAULT_ROW_IF_NULL
buildDefaultRow(ctx, expr, boundCleared);
#else
buildNullRow(ctx, expr, boundCleared);
#endif
OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
//Declare row for final level, iterate the appropriate number of times, and then assign and break.
BuildCtx initctx(ctx);
initctx.addGroup(); // add a group to allow a temporary to be declared later.
initctx.addAssign(rowExpr, defaultRowPtr);
HqlExprAssociation * savedMarker = ctx.associateExpr(queryConditionalRowMarker(), rowExpr);
BuildCtx iterctx(ctx);
IHqlExpression * sort = expr->queryChild(0);
IHqlExpression * dataset = sort->queryChild(0);
IHqlExpression * sortList = sort->queryChild(1);
BoundRow * chooseCursor = buildDatasetIterate(iterctx, dataset, false);
//if (!best) { best=row; } else { if (next < best) best = row; } Must short-circuit the test of best
OwnedHqlExpr testBest = createBoolExpr(no_eq, LINK(rowExpr), LINK(defaultRowPtr));
IHqlStmt * ifStmt = iterctx.addFilter(testBest);
{
OwnedHqlExpr source = ensureIteratedRowIsLive(initctx, ctx, iterctx, chooseCursor, dataset, expr);
OwnedHqlExpr castLeftRow = createValue(no_implicitcast, LINK(rowType), LINK(source));//chooseCursor->queryBound()));
iterctx.addAssign(rowExpr, castLeftRow);
}
iterctx.selectElse(ifStmt);
CHqlBoundExpr bound;
buildOrderedCompare(iterctx, dataset, sortList, bound, dataset, expr);
OwnedHqlExpr compare = createBoolExpr(no_lt, LINK(bound.expr), getZero());
iterctx.addFilter(compare);
{
OwnedHqlExpr source = ensureIteratedRowIsLive(initctx, ctx, iterctx, chooseCursor, dataset, expr);
OwnedHqlExpr castLeftRow = createValue(no_implicitcast, LINK(rowType), LINK(source));//chooseCursor->queryBound()));
iterctx.addAssign(rowExpr, castLeftRow);
}
ctx.removeAssociation(savedMarker);
//Set conditional later on, so test in main loop is explicit
#ifndef CREATE_DEAULT_ROW_IF_NULL
row->setConditional(true);
#endif
return createReferenceSelector(row);
}
BoundRow * HqlCppTranslator::buildOptimizeSelectFirstRow(BuildCtx & ctx, IHqlExpression * expr)
{
BoundRow * parentRow = NULL;
node_operator op = expr->getOperator();
switch (op)
{
case no_compound_childaggregate:
return buildOptimizeSelectFirstRow(ctx, expr->queryChild(0));
case no_hqlproject:
case no_newusertable:
{
parentRow = buildOptimizeSelectFirstRow(ctx, expr->queryChild(0));
if (!parentRow)
return NULL;
}
//fall through
case no_newaggregate:
{
Owned tempRow = declareTempAnonRow(ctx, ctx, expr);
Owned rowBuilder = createRowBuilder(ctx, tempRow);
Owned createdRef = createReferenceSelector(rowBuilder);
BuildCtx subctx(ctx);
subctx.addGroup();
if (parentRow)
{
if (op == no_hqlproject)
bindTableCursor(ctx, expr->queryChild(0), tempRow->queryBound(), no_left, querySelSeq(expr));
else
bindTableCursor(ctx, expr->queryChild(0), tempRow->queryBound(), no_none, NULL);
}
doBuildRowAssignAggregate(subctx, createdRef, expr);
finalizeTempRow(ctx, tempRow, rowBuilder);
return tempRow;
}
//
default:
return NULL;
}
}
void HqlCppTranslator::convertBoundDatasetToFirstRow(IHqlExpression * expr, CHqlBoundExpr & bound)
{
Owned type = makeReferenceModifier(expr->getType());
ITypeInfo * boundType = bound.queryType();
if (isArrayRowset(boundType))
{
Linked rowType = queryUnqualifiedType(queryRowType(boundType));
rowType.setown(makeReferenceModifier(LINK(rowType)));
if (hasLinkedRow(boundType))
type.setown(setLinkCountedAttr(type, true));
bound.expr.setown(createValue(no_deref, LINK(rowType), LINK(bound.expr)));
}
else if (bound.queryType()->isReference())
bound.expr.setown(createValue(no_typetransfer, LINK(type), bound.expr.getClear()));
else
bound.expr.setown(createValue(no_implicitcast, LINK(type), bound.expr.getClear()));
}
void HqlCppTranslator::convertBoundRowToDataset(BuildCtx & ctx, CHqlBoundExpr & bound, const BoundRow * row, ExpressionFormat preferredFormat)
{
IHqlExpression * boundRow = row->queryBound();
IHqlExpression * record = row->queryDataset()->queryRecord();
Owned type = makeTableType(makeRowType(LINK(record->queryType())), NULL, NULL, NULL);
Owned refType = makeReferenceModifier(LINK(type));
if (hasLinkCountedModifier(boundRow->queryType()) && (preferredFormat != FormatBlockedDataset))
{
OwnedHqlExpr curActivityId = getCurrentActivityId(ctx);
StringBuffer allocatorName;
ensureRowAllocator(allocatorName, ctx, record, curActivityId);
OwnedHqlExpr src = getPointer(boundRow);
//We can't just take the address of the link counted row, have to create a temporary dataset
//could be fixed once the link counting is tracked on rows and datasets
CHqlBoundTarget target;
createTempFor(ctx, type, target, typemod_none, FormatLinkedDataset);
StringBuffer s;
generateExprCpp(s, target.expr).append(".setRow(").append(allocatorName).append(",");
generateExprCpp(s, src).append(");");
ctx.addQuoted(s);
bound.setFromTarget(target);
}
else if (boundRow->queryType()->isReference())
bound.expr.setown(createValue(no_typetransfer, LINK(refType), LINK(boundRow)));
else
bound.expr.setown(createValue(no_implicitcast, LINK(refType), LINK(boundRow)));
bound.count.setown(getSizetConstant(1));
}
IHqlExpression * HqlCppTranslator::ensureIteratedRowIsLive(BuildCtx & initctx, BuildCtx & searchctx, BuildCtx & iterctx, BoundRow * row, IHqlExpression * dataset, IHqlExpression * rowExpr)
{
//The problem is that we are iterating through the rows in a dataset, and we need to access the "best" row outside of the loop
//However subsequent iterations of the loop might invalidate the current row, so we need to ensure the best row is retained.
//There should really be a better way, but it isn't too bad with link counted rows.
bool needToPreserve = false;
IHqlExpression * ds = dataset;
while (ds && !needToPreserve)
{
if (ds->isDatarow())
{
if (initctx.queryAssociation(ds, AssocRow, NULL))
break;
}
else
{
if (initctx.queryMatchExpr(ds))
break;
}
switch (ds->getOperator())
{
case no_filter:
case no_grouped:
case no_stepped:
case no_sorted:
case no_distributed:
case no_preservemeta:
case no_choosen:
case no_selectnth: // can occur as the lhs of no_select
case no_compound_childnormalize:
case no_compound_selectnew:
case no_compound_childread:
case no_dataset_alias:
ds = ds->queryChild(0);
break;
case no_select:
if (ds->isDataset())
{
if (ds->hasProperty(newAtom))
{
ds = ds->queryChild(0);
//don't walk complexds[1].childDataset since the [1] will be generated as a temporary
// if (!ds->isDataset() && (ds->getOperator() != no_select))
// ds = NULL;
}
else
ds = NULL;
}
else
{
//ds.x.y, always walk to to ds.x
ds = ds->queryChild(0);
}
break;
case no_rows:
case no_call:
case no_externalcall:
case no_getresult:
case no_getgraphresult:
case no_alias:
ds = NULL;
break;
default:
needToPreserve = true;
break;
}
}
OwnedHqlExpr source = getPointer(row->queryBound());
if (!needToPreserve || initctx.hasAssociation(*row, false))
return source.getClear();
//If link counted, then declare a member that is a link counted row to ensure this row remains linked.
ITypeInfo * rowType = rowExpr->queryType();
if (hasLinkCountedModifier(row->queryBound()))
{
CHqlBoundTarget saveTarget;
createTempFor(initctx, rowType, saveTarget, typemod_wrapper, FormatLinkedDataset);
StringBuffer s;
generateExprCpp(s, saveTarget.expr);
s.append(".set(");
generateExprCpp(s, source).append(");");
iterctx.addQuoted(s);
return getPointer(saveTarget.expr);
}
BuildCtx childctx(iterctx);
childctx.addGroup();
Owned tempRow = declareTempRow(childctx, childctx, rowExpr);
Owned rowBuilder = createRowBuilder(childctx, tempRow);
OwnedHqlExpr sourceSelector = ensureActiveRow(row->querySelector());
buildAssign(childctx, rowBuilder->querySelector(), sourceSelector);
finalizeTempRow(childctx, tempRow, rowBuilder);
return getPointer(tempRow->queryBound());
}
IReferenceSelector * HqlCppTranslator::buildDatasetIndexViaIterator(BuildCtx & ctx, IHqlExpression * expr)
{
OwnedHqlExpr dataset = normalizeAnyDatasetAliases(querySkipDatasetMeta(expr->queryChild(0)));
IHqlExpression * index = expr->queryChild(1);
IHqlExpression * childDataset = dataset;
switch (dataset->getOperator())
{
case no_hqlproject:
//optimize selectnth(project(rows, t), n) to projectrow(selectnth(rows, n), t)
IHqlExpression * transform = dataset->queryChild(1);
if (!containsSkip(transform) && !expr->hasProperty(_countProject_Atom))
childDataset = dataset->queryChild(0);
break;
}
//create a temporary
//Following works because rows are created as temporaries in the class, so still in scope outside the iterate loop.
//Not a strictly correct assumption - e.g., if ever done in the main process() code.
Owned rowType = makeReferenceModifier(expr->getType());
OwnedHqlExpr rowExpr = ctx.getTempDeclare(rowType, NULL);
Owned row = createBoundRow(expr, rowExpr);
CHqlBoundExpr boundCleared;
#ifdef CREATE_DEAULT_ROW_IF_NULL
buildDefaultRow(ctx, expr, boundCleared);
#else
buildNullRow(ctx, expr, boundCleared);
row->setConditional(true);
#endif
OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
//Declare row for final level, iterate the appropriate number of times, and then assign and break.
BuildCtx initctx(ctx);
initctx.addGroup(); // add a group to allow a temporary to be declared later.
initctx.addAssign(rowExpr, defaultRowPtr);
HqlExprAssociation * savedMarker = ctx.associateExpr(queryConditionalRowMarker(), rowExpr);
BuildCtx iterctx(ctx);
bool done = false;
if (childDataset->getOperator() == no_rows) //hasOutOfLineModifier(dataset->queryType()))
{
CHqlBoundExpr boundDs;
buildDataset(ctx, childDataset, boundDs, FormatNatural);
if (boundDs.count && isArrayRowset(boundDs.expr->queryType()))
{
OwnedHqlExpr castIndex = ensureExprType(index, unsignedType);
OwnedHqlExpr adjustedIndex = adjustValue(castIndex, -1);
CHqlBoundExpr boundIndex;
buildExpr(ctx, adjustedIndex, boundIndex);
OwnedHqlExpr count = getBoundCount(boundDs); // could be serialized, so can't assume bound.count is set
OwnedHqlExpr test = createValue(no_gt, makeBoolType(), LINK(count), LINK(boundIndex.expr));
iterctx.addFilter(test);
if (dataset != childDataset)
{
Owned datasetRowType = makeRowReferenceType(boundDs);
OwnedHqlExpr selectedRow = createValue(no_index, LINK(datasetRowType), LINK(boundDs.expr), LINK(boundIndex.expr));
OwnedHqlExpr projected = createRow(no_projectrow, createTranslated(selectedRow), createComma(LINK(dataset->queryChild(1)), LINK(querySelSeq(dataset))));
Owned newRow = buildNewRow(iterctx, projected);
OwnedHqlExpr newPtr = getPointer(newRow->queryRootRow()->queryBound());
iterctx.addAssign(rowExpr, newPtr);
}
else
{
OwnedHqlExpr selectedRow = createValue(no_index, LINK(rowType), LINK(boundDs.expr), LINK(boundIndex.expr));
iterctx.addAssign(rowExpr, selectedRow);
}
done = true;
}
}
if (!done)
{
BoundRow * chooseCursor;
//If choosing the first element, then no need to maintain a counter...
IValue * indexValue = index->queryValue();
if (indexValue && (indexValue->getIntValue() == 1))
chooseCursor = buildDatasetIterate(iterctx, dataset, true);
else
chooseCursor = buildDatasetIterate(iterctx, expr, true);
if (chooseCursor)
{
OwnedHqlExpr source = getPointer(chooseCursor->queryBound());
//MORE: Need casts because cursor may be (probably are) constant, but temporary isn't
//should somehow fnd out by looking at the cursors.
OwnedHqlExpr castLeftRow = createValue(no_implicitcast, LINK(rowType), LINK(source));
iterctx.addAssign(rowExpr, castLeftRow);
iterctx.addBreak();
}
}
ctx.removeAssociation(savedMarker);
ctx.associate(*row);
return createReferenceSelector(row);
}
IReferenceSelector * HqlCppTranslator::buildDatasetIndex(BuildCtx & ctx, IHqlExpression * expr)
{
HqlExprAssociation * match = ctx.queryAssociation(expr, AssocRow, NULL);
if (match)
return createReferenceSelector(static_cast(match));
#if 0
//Causes some queries (ncf10) to run out of memory, so disable for the moment.
OwnedHqlExpr optimized = optimizeHqlExpression(expr, getOptimizeFlags()|HOOcompoundproject);
if (optimized != expr)
return buildNewRow(ctx, optimized);
#endif
OwnedHqlExpr dataset = normalizeAnyDatasetAliases(expr->queryChild(0));
//Special cases:
//i) selecting row [1] from something that only has a single row
//ii) selecting row [n] from something that can be iterated.
//iii) row[1] from something sorted that can be iterated.
BoundRow * row = NULL;
if (isTrivialSelectN(expr))
{
BoundRow * row = NULL;
#if 0
//This could be a good idea - but again it can mess up cse since dataset never gets bound.
//Could enable if I implement cse on datasets within transforms.
// if (canIterateInline(&ctx, dataset))
// row = buildOptimizeSelectFirstRow(ctx, dataset);
#endif
if (!row)
{
CHqlBoundExpr bound;
buildDataset(ctx, dataset, bound, FormatNatural);
convertBoundDatasetToFirstRow(expr, bound);
row = bindRow(ctx, expr, bound.expr);
}
return createReferenceSelector(row);
}
else if (canIterateInline(&ctx, dataset))
{
//MORE? Following doesn't work for implicit normalize which iterates multiple levels
bool specialCase = false;
dataset.set(querySkipDatasetMeta(dataset));
switch (dataset->getOperator())
{
case no_select:
specialCase = !isMultiLevelDatasetSelector(expr, false);
break;
case no_if:
case no_inlinedictionary:
case no_inlinetable:
case no_join:
//Always creates a temporary, so don't use an iterator
specialCase = true;
break;
default:
specialCase = alwaysEvaluatesToBound(dataset);
break;
}
if (!specialCase)
return buildDatasetIndexViaIterator(ctx, expr);
}
else if (isSelectSortedTop(expr) && canIterateInline(&ctx, dataset->queryChild(0)))
{
return doBuildRowSelectTop(ctx, expr);
}
//MORE: Is this a good idea???
else if (!canProcessInline(&ctx, expr))
{
CHqlBoundExpr bound;
OwnedHqlExpr dsExpr = expr->isDatarow() ? createDatasetFromRow(LINK(expr)) : LINK(expr);
buildDataset(ctx, dsExpr, bound, FormatNatural);
convertBoundDatasetToFirstRow(expr, bound);
row = bindRow(ctx, expr, bound.expr);
}
if (!row)
{
Owned cursor = createDatasetSelector(ctx, dataset);
row = cursor->buildSelect(ctx, expr);
if (!row)
{
CHqlBoundExpr boundCleared;
buildDefaultRow(ctx, dataset, boundCleared);
OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
row = bindRow(ctx, expr, defaultRowPtr);
}
}
return createReferenceSelector(row);
}
IReferenceSelector * HqlCppTranslator::buildDatasetSelectMap(BuildCtx & ctx, IHqlExpression * expr)
{
HqlExprAssociation * match = ctx.queryAssociation(expr, AssocRow, NULL);
if (match)
return createReferenceSelector(static_cast(match));
OwnedHqlExpr dataset = normalizeAnyDatasetAliases(expr->queryChild(0));
// Create a row to pass to the lookup function
BoundRow * row = NULL;
if (!canProcessInline(&ctx, expr))
{
CHqlBoundExpr bound;
OwnedHqlExpr dsExpr = expr->isDatarow() ? createDatasetFromRow(LINK(expr)) : LINK(expr);
buildDataset(ctx, dsExpr, bound, FormatNatural);
convertBoundDatasetToFirstRow(expr, bound);
row = bindRow(ctx, expr, bound.expr);
}
if (!row)
{
Owned cursor = createDatasetSelector(ctx, dataset);
row = cursor->buildSelect(ctx, expr);
if (!row)
{
CHqlBoundExpr boundCleared;
buildDefaultRow(ctx, dataset, boundCleared);
OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
row = bindRow(ctx, expr, defaultRowPtr);
}
}
return createReferenceSelector(row);
}
//---------------------------------------------------------------------------
IHqlExpression * HqlCppTranslator::buildGetLocalResult(BuildCtx & ctx, IHqlExpression * expr, bool preferLinkedRows)
{
IHqlExpression * graphId = expr->queryChild(1);
IHqlExpression * resultNum = expr->queryChild(2);
Linked exprType = queryUnqualifiedType(expr->queryType());
if (preferLinkedRows && !hasLinkCountedModifier(exprType))
exprType.setown(makeAttributeModifier(LINK(exprType), getLinkCountedAttr()));
if (expr->hasProperty(externalAtom))
{
IHqlExpression * resultInstance = queryPropertyChild(expr, externalAtom, 0);
HqlExprAssociation * matchedResults = ctx.queryMatchExpr(resultInstance);
if (!matchedResults)
{
//Very unusual - a result is required from a child query, but that child query is actually in
//the parent/grandparent. We need to evaluate in the parent instead.
CHqlBoundExpr match;
if (!buildExprInCorrectContext(ctx, expr, match, false))
throwUnexpected();
return match.getTranslatedExpr();
}
HqlExprArray args;
args.append(*LINK(matchedResults->queryExpr()));
args.append(*LINK(resultNum));
if (preferLinkedRows)
return bindFunctionCall(getChildQueryLinkedResultAtom, args, exprType);
return bindFunctionCall(getChildQueryResultAtom, args, exprType);
}
assertex(activeActivities.ordinality());
queryAddResultDependancy(activeActivities.tos(), graphId, resultNum);
SubGraphInfo * activeSubgraph = queryActiveSubGraph(ctx);
assertex(activeSubgraph && graphId == activeSubgraph->graphTag);
unique_id_t id = activeSubgraph->graphId;
EvalContext * instance = queryEvalContext(ctx);
OwnedHqlExpr retInstanceExpr;
if (instance && !insideOnCreate(ctx))
retInstanceExpr.setown(instance->createGraphLookup(id, false));
else
retInstanceExpr.setown(doCreateGraphLookup(ctx, ctx, id, "this", true));
HqlExprArray args;
args.append(*LINK(retInstanceExpr));
args.append(*LINK(resultNum));
if (preferLinkedRows)
return bindFunctionCall(getLocalLinkedResultAtom, args, exprType);
return bindFunctionCall(getLocalResultAtom, args, exprType);
}
void HqlCppTranslator::doBuildAssignGetGraphResult(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
{
if (expr->hasProperty(_streaming_Atom))
throwError(HQLERR_LoopTooComplexForParallel);
bool isTargetLinkCounted = hasLinkCountedModifier(target.queryType());
if (expr->hasProperty(externalAtom))
{
OwnedHqlExpr call = buildGetLocalResult(ctx, expr, isTargetLinkCounted);
buildExprAssign(ctx, target, call);
return;
}
if (!isCurrentActiveGraph(ctx, expr->queryChild(1)))
{
CHqlBoundExpr match;
if (!buildExprInCorrectContext(ctx, expr, match, false))
throwError(HQLERR_GraphContextNotFound);
assign(ctx, target, match);
return;
}
OwnedHqlExpr call = buildGetLocalResult(ctx, expr, isTargetLinkCounted);
buildExprAssign(ctx, target, call);
}
void HqlCppTranslator::doBuildExprGetGraphResult(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & tgt, ExpressionFormat format)
{
if (!expr->hasProperty(externalAtom))
{
doBuildAliasValue(ctx, expr, tgt);
return;
if (!isCurrentActiveGraph(ctx, expr->queryChild(1)))
{
if (!buildExprInCorrectContext(ctx, expr, tgt, false))
throwError(HQLERR_GraphContextNotFound);
return;
}
}
bool useLinkCounted = recordRequiresSerialization(expr->queryRecord()) || options.tempDatasetsUseLinkedRows;
OwnedHqlExpr call = buildGetLocalResult(ctx, expr, useLinkCounted);
switch (expr->queryType()->getTypeCode())
{
case type_row:
throwUnexpected();
case type_dictionary:
case type_table:
case type_groupedtable:
buildTempExpr(ctx, call, tgt);
break;
default:
buildExpr(ctx, call, tgt);
break;
}
}
ABoundActivity * HqlCppTranslator::doBuildActivityGetGraphResult(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * graphId = expr->queryChild(1);
IHqlExpression * resultNum = expr->queryChild(2);
ThorActivityKind activityKind = (expr->hasProperty(_streaming_Atom) ? TAKlocalstreamread : TAKlocalresultread);
bool useImplementationClass = options.minimizeActivityClasses && (resultNum->getOperator() == no_constant);
Owned instance = new ActivityInstance(*this, ctx, activityKind, expr, "LocalResultRead");
if (useImplementationClass)
instance->setImplementationClass(newLocalResultReadArgAtom);
if (expr->hasProperty(_loop_Atom))
{
if (isCurrentActiveGraph(ctx, graphId))
instance->graphLabel.set("Begin Loop");
else
instance->graphLabel.set("Outer Loop Input");
}
buildActivityFramework(instance);
buildInstancePrefix(instance);
if (!useImplementationClass)
doBuildUnsignedFunction(instance->classctx, "querySequence", resultNum);
else
instance->addConstructorParameter(resultNum);
addGraphIdAttribute(instance, ctx, graphId);
buildInstanceSuffix(instance);
queryAddResultDependancy(*instance->queryBoundActivity(), graphId, resultNum);
return instance->getBoundActivity();
}
ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
{
IHqlExpression * dataset = expr->queryChild(0);
IHqlExpression * graphId = expr->queryChild(1);
IHqlExpression * resultNum = expr->queryChild(2);
bool isSpill = expr->hasProperty(_spill_Atom);
ABoundActivity * parentActivity = activeActivities.ordinality() ? &activeActivities.tos() : NULL;
Owned boundDataset = buildCachedActivity(ctx, dataset);
bool useImplementationClass = options.minimizeActivityClasses;
Owned instance;
if (expr->getOperator() == no_spillgraphresult)
{
instance.setown(new ActivityInstance(*this, ctx, TAKlocalresultspill, expr, "LocalResultSpill"));
}
else
{
instance.setown(new ActivityInstance(*this, ctx, TAKlocalresultwrite, expr, "LocalResultWrite"));
}
if (useImplementationClass)
instance->setImplementationClass(newLocalResultSpillArgAtom);
if (expr->hasProperty(_loop_Atom))
instance->graphLabel.set("End Loop");
buildActivityFramework(instance, isRoot && !isSpill);
buildInstancePrefix(instance);
if (!useImplementationClass)
{
doBuildUnsignedFunction(instance->classctx, "querySequence", resultNum);
doBuildBoolFunction(instance->classctx, "usedOutsideGraph", !isSpill);
}
else
{
instance->addConstructorParameter(resultNum);
instance->addConstructorParameter(queryBoolExpr(!isSpill));
}
if (parentActivity && !insideRemoteGraph(ctx) && !isSpill)
{
const char * relationship;
if (expr->hasProperty(_loop_Atom))
relationship = "Body";
else if (insideRemoteGraph(ctx))
relationship = "Remote";
else
relationship = "Child";
addDependency(ctx, instance->queryBoundActivity(), parentActivity, childAtom, relationship);
}
instance->addAttributeBool("_isSpill", isSpill);
if (targetRoxie())
addGraphIdAttribute(instance, ctx, graphId);
buildInstanceSuffix(instance);
buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
associateRemoteResult(*instance, graphId, resultNum);
return instance->getBoundActivity();
}
ABoundActivity * HqlCppTranslator::doBuildActivityReturnResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot)
{
IHqlExpression * dataset = expr->queryChild(0);
ABoundActivity * parentActivity = activeActivities.ordinality() ? &activeActivities.tos() : NULL;
Owned boundDataset = buildCachedActivity(ctx, dataset);
ThorActivityKind kind;
const char * helper;
if (dataset->isDataset())
{
kind = TAKdatasetresult;
helper = "DatasetResult";
}
else
{
kind = TAKrowresult;
helper = "RowResult";
}
Owned instance = new ActivityInstance(*this, ctx, kind, expr, helper);
buildActivityFramework(instance, isRoot);
buildInstancePrefix(instance);
if (parentActivity && !insideRemoteGraph(ctx))
addDependency(ctx, instance->queryBoundActivity(), parentActivity, childAtom, "Child");
buildInstanceSuffix(instance);
buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
return instance->getBoundActivity();
}
void HqlCppTranslator::doBuildAssignLoopCounter(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr)
{
if (!isCurrentActiveGraph(ctx, expr->queryChild(0)))
{
CHqlBoundExpr match;
if (!buildExprInCorrectContext(ctx, expr, match, false))
throwError(HQLERR_GraphContextNotFound);
assign(ctx, target, match);
return;
}
HqlExprArray args;
OwnedHqlExpr call = bindFunctionCall(getGraphLoopCounterAtom, args);
buildExprAssign(ctx, target, call);
}
//---------------------------------------------------------------------------
ABoundActivity * HqlCppTranslator::doBuildActivityGetGraphLoopResult(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * graphId = expr->queryChild(1);
IHqlExpression * resultNum = expr->queryChild(2);
Owned instance = new ActivityInstance(*this, ctx, TAKgraphloopresultread, expr, "GraphLoopResultRead");
buildActivityFramework(instance);
buildInstancePrefix(instance);
doBuildUnsignedFunction(instance->startctx, "querySequence", resultNum);
addGraphIdAttribute(instance, ctx, graphId);
buildInstanceSuffix(instance);
return instance->getBoundActivity();
}
ABoundActivity * HqlCppTranslator::doBuildActivitySetGraphLoopResult(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * dataset = expr->queryChild(0);
IHqlExpression * graphId = expr->queryChild(1);
bool isSpill = expr->hasProperty(_spill_Atom);
ABoundActivity * parentActivity = activeActivities.ordinality() ? &activeActivities.tos() : NULL;
Owned boundDataset = buildCachedActivity(ctx, dataset);
bool useImplementationClass = options.minimizeActivityClasses;
Owned instance = new ActivityInstance(*this, ctx, TAKgraphloopresultwrite, expr, "GraphLoopResultWrite");
if (useImplementationClass)
instance->setImplementationClass(newGraphLoopResultWriteArgAtom);
buildActivityFramework(instance, true);
buildInstancePrefix(instance);
if (parentActivity && !insideRemoteGraph(ctx) && !isSpill)
addDependency(ctx, instance->queryBoundActivity(), parentActivity, childAtom, "Body");
if (targetRoxie())
addGraphIdAttribute(instance, ctx, graphId);
buildInstanceSuffix(instance);
buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
return instance->getBoundActivity();
}
//---------------------------------------------------------------------------
static IHqlExpression * queryResultExpr(IHqlExpression * expr)
{
loop
{
switch (expr->getOperator())
{
case no_compound:
expr = expr->queryChild(1);
break;
case no_subgraph:
expr = expr->queryChild(0);
break;
case no_returnresult:
return expr;
default:
throwUnexpectedOp(expr->getOperator());
}
}
}
ABoundActivity * HqlCppTranslator::doBuildActivityForceLocal(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * child = expr->queryChild(0);
if (targetHThor() || (targetThor() && !insideChildQuery(ctx)))
{
WARNING(HQLWRN_LocalHasNoEffect);
return buildCachedActivity(ctx, child);
}
OwnedHqlExpr result = createValue(no_returnresult, makeVoidType(), LINK(child));
OwnedHqlExpr remote = resourceThorGraph(*this, result, RoxieCluster, 1, NULL);
unique_id_t localId = doBuildThorSubGraph(ctx, remote, SubGraphRemote);
Owned instance = new ActivityInstance(*this, ctx, TAKlocalgraph, expr, "Null");
buildActivityFramework(instance);
buildInstancePrefix(instance);
instance->addAttributeInt("_subgraph", localId);
ActivityAssociation * match = static_cast(ctx.queryAssociation(queryResultExpr(remote), AssocActivity, NULL));
assertex(match);
addDependency(ctx, match->activity, instance->queryBoundActivity(), childAtom);
buildInstanceSuffix(instance);
return instance->getBoundActivity();
}
void HqlCppTranslator::doBuildStmtApply(BuildCtx & ctx, IHqlExpression * expr)
{
IHqlExpression * dataset = expr->queryChild(0);
IHqlExpression * start = expr->queryProperty(beforeAtom);
IHqlExpression * end = expr->queryProperty(afterAtom);
if (start)
buildStmt(ctx, start->queryChild(0));
BuildCtx condctx(ctx);
BoundRow * cursor = buildDatasetIterate(condctx, dataset, false);
unsigned max = expr->numChildren();
for (unsigned i=1; i < max; i++)
{
IHqlExpression * cur = expr->queryChild(i);
if (!cur->isAttribute())
buildStmt(condctx, cur);
}
if (end)
buildStmt(ctx, end->queryChild(0));
}