HPCC-15767 Split volatile into two flags

No differences from previous regression runs

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hql/hqlatoms.cpp

@@ -284,6 +284,7 @@ IAtom * _nlpParse_Atom;
 IAtom * noBoundCheckAtom;
 IAtom * noCaseAtom;
 IAtom * noConstAtom;
+IAtom * _noDuplicate_Atom;
 IAtom * nofoldAtom;
 IAtom * _noHoist_Atom;
 IAtom * noLocalAtom;
@@ -735,6 +736,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(noBoundCheck);
     MAKEATOM(noCase);
     MAKEATOM(noConst);
+    MAKESYSATOM(noDuplicate);
     MAKEATOM(nofold);
     MAKESYSATOM(noHoist);
     MAKEATOM(noLocal);

ecl/hql/hqlatoms.hpp

@@ -288,6 +288,7 @@ extern HQL_API IAtom * _nlpParse_Atom;
 extern HQL_API IAtom * noBoundCheckAtom;
 extern HQL_API IAtom * noCaseAtom;
 extern HQL_API IAtom * noConstAtom;
+extern HQL_API IAtom * _noDuplicate_Atom;
 extern HQL_API IAtom * nofoldAtom;
 extern HQL_API IAtom * _noHoist_Atom;
 extern HQL_API IAtom * noLocalAtom;

ecl/hql/hqlexpr.cpp

@@ -3423,7 +3423,7 @@ void CHqlExpression::initFlagsBeforeOperands()
         break;
     case no_random:
         infoFlags2 &= ~(HEF2constant);
-        infoFlags |= HEFvolatile;
+        infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
         break;
     case no_wait:
         infoFlags2 |= HEF2globalAction;
@@ -3602,7 +3602,7 @@ void CHqlExpression::updateFlagsAfterOperands()
     switch (op)
     {
     case no_pure:
-        infoFlags &= ~(HEFvolatile|HEFaction|HEFthrowds|HEFthrowscalar|HEFcontainsSkip);
+        infoFlags &= ~(HEFnoduplicate|HEFaction|HEFthrowds|HEFthrowscalar|HEFcontainsSkip);
         break;
     case no_record:
         {
@@ -3754,7 +3754,7 @@ void CHqlExpression::updateFlagsAfterOperands()
             infoFlags2 |= HEF2constant;
             IAtom * name = queryName();
             if (name == _volatileId_Atom)
-                infoFlags |= HEFvolatile;
+                infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
             break;
         }
     case no_newxmlparse:
@@ -3791,13 +3791,13 @@ void CHqlExpression::updateFlagsAfterOperands()
             if (name == onFailAtom)
                 infoFlags &= ~(HEFonFailDependent|HEFcontainsSkip); // ONFAIL(SKIP) - skip shouldn't extend any further
             else if (name == _volatileId_Atom)
-                infoFlags |= HEFvolatile;
+                infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
             infoFlags &= ~(HEFthrowscalar|HEFthrowds|HEFoldthrows);
             break;
         }
     case no_clustersize:
         //wrong, but improves the generated code
-        infoFlags |= HEFvolatile;
+        infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
         break;
     case no_type:
         {
@@ -3857,7 +3857,7 @@ void CHqlExpression::updateFlagsAfterOperands()
                 if (bodycode->getOperator() == no_embedbody)
                 {
                     if (bodycode->queryAttribute(actionAtom))
-                        infoFlags |= HEFvolatile;
+                        infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
                 }
             }
             else
@@ -3868,13 +3868,13 @@ void CHqlExpression::updateFlagsAfterOperands()
     case no_getresult:
         {
             if (false && matchesConstantValue(queryAttributeChild(this, sequenceAtom, 0), ResultSequenceOnce))
-                infoFlags |= HEFvolatile;
+                infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
             break;
         }
     case no_embedbody:
         {
             if (queryAttribute(actionAtom))
-                infoFlags |= HEFvolatile;
+                infoFlags |= (HEFnoduplicate|HEFcontextDependentException);
             break;
         }
     }
@@ -4134,6 +4134,17 @@ void CHqlExpression::onAppendOperand(IHqlExpression & child, unsigned whichOpera
     unsigned childFlags = child.getInfoFlags();
     unsigned childFlags2 = child.getInfoFlags2();
     node_operator childOp = child.getOperator();
+
+    const unsigned contextFlags = HEFcontextDependentException|HEFthrowscalar|HEFthrowds;
+    if (childFlags & contextFlags)
+    {
+        if (isDataset() || isAction())
+        {
+            if (!child.isDataset() && !child.isAction())
+                childFlags &= ~contextFlags;
+        }
+    }
+
     switch (op)
     {
     case no_keyindex:
@@ -4198,7 +4209,7 @@ void CHqlExpression::onAppendOperand(IHqlExpression & child, unsigned whichOpera
     {
     case no_transform:
     case no_newtransform:
-        childFlags &= ~(HEFtransformDependent|HEFcontainsSkip|HEFthrowscalar|HEFthrowds);
+        childFlags &= ~(HEFcontextDependentException|HEFtransformDependent|HEFcontainsSkip|HEFthrowscalar|HEFthrowds);
         break;
     }
 
@@ -9944,7 +9955,7 @@ IHqlExpression * CHqlScopeParameter::lookupSymbol(IIdAtom * searchName, unsigned
 //==============================================================================================================
 
 CHqlDelayedScope::CHqlDelayedScope(HqlExprArray &_ownedOperands)
- : CHqlExpressionWithTables(no_delayedscope)
+ : CHqlExpressionWithTables(no_delayedscope), type(nullptr)
 {
     setOperands(_ownedOperands); // after type is initialized
     type = queryChild(0)->queryType();
@@ -10329,16 +10340,17 @@ CHqlExternalCall::CHqlExternalCall(IHqlExpression * _funcdef, ITypeInfo * _type,
         impureFlags |= HEFcontextDependentException;
     if (body->hasAttribute(costlyAtom))
         impureFlags |= HEFcostly;
+    if (body->hasAttribute(_noDuplicate_Atom))
+        impureFlags |= HEFnoduplicate;
 
     if (isVolatileFuncdef(funcdef))
-        impureFlags |= HEFvolatile;
+        impureFlags |= (HEFnoduplicate|HEFcontextDependentException);
     //Once aren't really pure, but are as far as the code generator is concerned.  Split into more flags if it becomes an issue.
     if (!body->hasAttribute(pureAtom) && !body->hasAttribute(onceAtom))
     {
-        infoFlags |= (HEFvolatile);
+        infoFlags |= (HEFnoduplicate);
     }
 
-#if 0
     if (body->hasAttribute(ctxmethodAtom))
     {
         StringBuffer entrypoint;
@@ -10350,10 +10362,9 @@ CHqlExternalCall::CHqlExternalCall(IHqlExpression * _funcdef, ITypeInfo * _type,
         }
         if (streq(entrypoint.str(), "getPlatform"))
         {
-            //impureFlags |= HEFvolatile;
+            //impureFlags |= (HEFvolatilevalue|HEFcontextDependentException);
         }
     }
-#endif
 
     infoFlags |= impureFlags;
     
@@ -12548,11 +12559,12 @@ IHqlExpression * createExternalFuncdefFromInternal(IHqlExpression * funcdef)
     HqlExprArray attrs;
     unwindChildren(attrs, body, 1);
 
-    if (body->isPure())
-        attrs.append(*createAttribute(pureAtom));
-    if (body->getInfoFlags() & HEFaction)
-        attrs.append(*createAttribute(actionAtom));
-    if (body->getInfoFlags() & HEFcontextDependentException)
+    //This should mirror the code in CHqlExternalCall::CHqlExternalCall
+    unsigned impureFlags = body->getInfoFlags();
+    if (impureFlags & (HEFthrowds|HEFthrowscalar))
+        attrs.append(*createAttribute(failAtom));
+
+    if (impureFlags & HEFcontextDependentException)
         attrs.append(*createAttribute(contextSensitiveAtom));
     if (functionBodyUsesContext(body))
         attrs.append(*LINK(cachedContextAttribute));
@@ -12561,6 +12573,21 @@ IHqlExpression * createExternalFuncdefFromInternal(IHqlExpression * funcdef)
     if (child && child->getOperator()==no_embedbody)
         unwindAttribute(attrs, child, inlineAtom);
 
+    if (impureFlags & HEFcostly)
+        attrs.append(*createAttribute(costlyAtom));
+
+    if (impureFlags & HEFnoduplicate)
+        attrs.append(*createAttribute(_noDuplicate_Atom));
+
+    if (impureFlags & HEFaction)
+        attrs.append(*createAttribute(actionAtom));
+
+    if (impureFlags & HEFcontainsNlpText)
+        attrs.append(*createAttribute(userMatchFunctionAtom));
+
+    if (!(impureFlags & HEFimpure))// && attrs.empty())
+        attrs.append(*createAttribute(pureAtom));
+
     ITypeInfo * returnType = funcdef->queryType()->queryChildType();
     OwnedHqlExpr externalExpr = createExternalReference(funcdef->queryId(), LINK(returnType), attrs);
     return replaceChild(funcdef, 0, externalExpr);
@@ -15512,7 +15539,7 @@ bool isContextDependent(IHqlExpression * expr, bool ignoreFailures, bool ignoreG
 
 bool isPureCanSkip(IHqlExpression * expr)
 {
-    return (expr->getInfoFlags() & (HEFvolatile|HEFaction|HEFthrowscalar|HEFthrowds)) == 0; 
+    return (expr->getInfoFlags() & (HEFnoduplicate|HEFaction|HEFthrowscalar|HEFthrowds)) == 0;
 }
 
 bool hasSideEffects(IHqlExpression * expr)

ecl/hql/hqlexpr.hpp

@@ -114,7 +114,7 @@ enum
     HEFcontainsDatasetAliasLocally= 0x00000040,
 
 //impure properties (see head of hqlexpr.cpp for detailed discussion)
-    HEFvolatile                 = 0x00000080,           // value changes each time it is called - e.g., RANDOM()
+    HEFnoduplicate              = 0x00000080,           // value changes each time it is called - e.g., RANDOM()
     HEFcontextDependentException= 0x00000100,           // depends on the context, but not known how
     HEFcostly                   = 0x00000200,           // an expensive operation
     HEFaction                   = 0x00000400,           // an action, or something that can have a side-effect.  Not convinced this is needed
@@ -158,7 +158,7 @@ enum
 
     HEFintersectionFlags        = (0),
     HEFunionFlags               = (HEFunbound|HEFfunctionOfGroupAggregate|
-                                   HEFvolatile|HEFcontextDependentException|HEFcostly|HEFaction|HEFthrowscalar|HEFthrowds|HEFoldthrows|
+                                   HEFnoduplicate|HEFcontextDependentException|HEFcostly|HEFaction|HEFthrowscalar|HEFthrowds|HEFoldthrows|
                                    HEFonFailDependent|HEFcontainsActiveDataset|HEFcontainsActiveNonSelector|HEFcontainsDataset|
                                    HEFtranslated|HEFgraphDependent|HEFcontainsNlpText|HEFcontainsXmlText|HEFtransformDependent|
                                    HEFcontainsSkip|HEFcontainsCounter|HEFassertkeyed|HEFcontainsAlias|HEFcontainsAliasLocally|
@@ -166,7 +166,7 @@ enum
 
     HEFcontextDependentNoThrow  = (HEFcontextDependent & ~(HEFthrowscalar|HEFthrowds|HEFoldthrows)),
     HEFcontextDependentDataset  = (HEFcontextDependent & ~(HEFthrowscalar)),
-    HEFimpure                   = (HEFvolatile|HEFaction|HEFthrowds|HEFthrowscalar|HEFcontainsSkip),
+    HEFimpure                   = (HEFnoduplicate|HEFaction|HEFthrowds|HEFthrowscalar|HEFcontainsSkip),
 };
 
 //NB: increase the member variable if it grows 
@@ -1670,11 +1670,11 @@ extern HQL_API IHqlExpression * queryNewSelectAttrExpr();
 
 //The following functions deal with the different aspects of impure functions - volatile,costly,throw,skip,...
 
-inline bool isVolatile(IHqlExpression * expr)           { return (expr->getInfoFlags() & HEFvolatile) != 0; }
+inline bool isVolatile(IHqlExpression * expr)           { return (expr->getInfoFlags() & HEFnoduplicate) != 0; }
 //Is it ok to duplicate the evaluation of this expression in another context?
-inline bool canDuplicateExpr(IHqlExpression * expr)      { return (expr->getInfoFlags() & (HEFvolatile|HEFcostly)) == 0; }
+inline bool canDuplicateExpr(IHqlExpression * expr)      { return (expr->getInfoFlags() & (HEFnoduplicate|HEFcostly)) == 0; }
 //Is it legal to evaluate this expression in a different context - e.g, in a parent instead of child query
-inline bool canChangeContext(IHqlExpression * expr)     { return (expr->getInfoFlags() & (HEFvolatile|HEFcontextDependent|HEFthrow|HEFcontainsSkip|HEFcostly)) == 0; }
+inline bool canChangeContext(IHqlExpression * expr)     { return (expr->getInfoFlags() & (HEFcontextDependent|HEFthrow|HEFcontainsSkip|HEFcostly)) == 0; }
 //Is it ok to convert a conditional expression to an unconditional expression?
 inline bool canRemoveGuard(IHqlExpression * expr)       { return (expr->getInfoFlags() & (HEFthrow|HEFcontainsSkip|HEFcostly)) == 0; }
 //Is it legal to reuse the value created in another context for this expression?

ecl/hql/hqlmeta.cpp

@@ -714,6 +714,11 @@ bool isKnownDistribution(IHqlExpression * distribution)
     return distribution && (distribution != queryUnknownAttribute());
 }
 
+bool isKnownNonVolatileDistribution(IHqlExpression * distribution)
+{
+    return isKnownDistribution(distribution) && !isVolatile(distribution);
+}
+
 bool isSortedDistribution(IHqlExpression * distribution)
 {
     return distribution && (distribution->queryName() == sortedAtom);
@@ -830,7 +835,6 @@ extern HQL_API IHqlExpression * mapJoinDistribution(TableProjectMapper & mapper,
     return NULL;
 }
 
-
 extern HQL_API IHqlExpression * mapDistribution(IHqlExpression * distribution, TableProjectMapper & mapper)
 {
     if (!distribution) 
@@ -1314,7 +1318,7 @@ static bool includesFieldsOutsideGrouping(IHqlExpression * distribution, const H
 bool isPartitionedForGroup(IHqlExpression * table, IHqlExpression *grouping, bool isGroupAll)
 {
     IHqlExpression * distribution = queryDistribution(table);
-    if (!isKnownDistribution(distribution) || !distribution->isPure())
+    if (!isKnownNonVolatileDistribution(distribution))
         return false;
 
     OwnedHqlExpr normalizedGrouping = normalizeSortlist(grouping, table);
@@ -1790,8 +1794,7 @@ bool isDistributedCoLocally(IHqlExpression * dataset1, IHqlExpression * dataset2
     IHqlExpression * distribute2 = queryDistribution(dataset2);
     //Check the distribution functions are equivalent - by walking through in parallel, and don't contain any
     //references to fields not in the join conditions
-    if (isKnownDistribution(distribute1) && distribute1->isPure() &&
-        isKnownDistribution(distribute2) && distribute2->isPure())
+    if (isKnownNonVolatileDistribution(distribute1) && isKnownNonVolatileDistribution(distribute2))
     {
         //If sorted they are only going to be codistributed if they came from the same sort
         //We could only determine that by making the sort orders unique - by appending a uid

ecl/hql/hqlthql.cpp

@@ -711,6 +711,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
 #ifdef SHOWCONTEXTDETAIL
             s.append('[');
             unsigned flags = expr->getInfoFlags();
+            if (flags & HEFnoduplicate) s.append('V');
             if (flags & HEFgraphDependent) s.append('G');
             if (flags & HEFcontainsSkip) s.append('S');
             if (flags & HEFcontainsCounter) s.append('C');