HPCC-8335 Option on a field to control the default value

The default value of a field will be used:

1. When a dictionary lookup does not find a match
2. When a dataset select indexes a non-existent row
3. When assigning SELF := [] in a transform
4. In the default row passed to transforms by a OUTER join
5. When reading from XML and a column is not present

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlexpr.cpp

@@ -2870,8 +2870,7 @@ IHqlExpression * ensureExprType(IHqlExpression * expr, ITypeInfo * type, node_op
     }
 
     node_operator op = expr->getOperator();
-    if (op == no_null)
-        return createNullExpr(type);
+    assertex (op != no_null);
 
     IValue * value = expr->queryValue();
     if (value && type->assignableFrom(exprType))    // this last condition is unnecessary, but changes some persist crcs if removed
@@ -12856,6 +12855,11 @@ extern HQL_API IHqlExpression * createNullExpr(ITypeInfo * type)
 
 extern HQL_API IHqlExpression * createNullExpr(IHqlExpression * expr)
 {
+    if (expr->getOperator()==no_select)
+        return createNullExpr(expr->queryChild(1));
+    IHqlExpression * defaultValue = queryPropertyChild(expr, defaultAtom, 0);
+    if (defaultValue)
+        return LINK(defaultValue);
     return createNullExpr(expr->queryType());
 }
 
@@ -12886,23 +12890,22 @@ extern HQL_API IHqlExpression * createPureVirtual(ITypeInfo * type)
         return createValue(no_purevirtual, makeIntType(8, true));
 }
 
-extern HQL_API bool isNullExpr(IHqlExpression * expr, ITypeInfo * type)
+extern HQL_API bool isNullExpr(IHqlExpression * expr, IHqlExpression * field)
 {
     ITypeInfo * exprType = expr->queryType();
-    if (exprType->getTypeCode() != type->getTypeCode())
+    ITypeInfo * fieldType = field->queryType();
+    if (exprType->getTypeCode() != fieldType->getTypeCode())
         return false;
 
     IValue * value = expr->queryValue();
-    switch (type->getTypeCode())
+    switch (fieldType->getTypeCode())
     {
     case type_boolean:
-        return (value && value->getBoolValue() == false);
     case type_int:
     case type_swapint:
     case type_date:
     case type_enumerated:
     case type_bitfield:
-        return (value && value->getIntValue() == 0);
     case type_data:
     case type_string:
     case type_varstring:
@@ -12917,8 +12920,8 @@ extern HQL_API bool isNullExpr(IHqlExpression * expr, ITypeInfo * type)
         {
             if (!value)
                 return false;
-            OwnedHqlExpr null = createNullExpr(type);
-            OwnedHqlExpr castValue = ensureExprType(expr, type);
+            OwnedHqlExpr null = createNullExpr(field);
+            OwnedHqlExpr castValue = ensureExprType(expr, fieldType);
             return null->queryBody() == castValue->queryBody();
         }
     case type_row:

ecl/hql/hqlexpr.hpp

@@ -1332,7 +1332,7 @@ extern HQL_API IHqlExpression * createPureVirtual(ITypeInfo * type);
 extern HQL_API IHqlExpression * cloneOrLink(IHqlExpression * expr, HqlExprArray & children);
 extern HQL_API IHqlExpression * createConstantOne();
 extern HQL_API IHqlExpression * createLocalAttribute();
-extern HQL_API bool isNullExpr(IHqlExpression * expr, ITypeInfo * type);
+extern HQL_API bool isNullExpr(IHqlExpression * expr, IHqlExpression * field);
 inline bool isNull(IHqlExpression * expr)       { return expr->getOperator() == no_null; }
 inline bool isNullAction(IHqlExpression * expr) { return isNull(expr) && expr->isAction(); }
 inline bool isFail(IHqlExpression * expr)       { return expr->getOperator() == no_fail; }

ecl/hql/hqlgram.y

@@ -161,6 +161,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   DATASET
   __DEBUG__
   DEDUP
+  DEFAULT
   DEFINE
   DENORMALIZE
   DEPRECATED
@@ -4403,6 +4404,10 @@ fieldAttr
                         {
                             $$.setExpr(createAttribute(xmlDefaultAtom, $3.getExpr()), $1);
                         }
+    | DEFAULT '(' constExpression ')'
+                        {
+                            $$.setExpr(createAttribute(defaultAtom, $3.getExpr()), $1);
+                        }
     | STRING_CONST
                         {
                             OwnedHqlExpr expr = $1.getExpr();

ecl/hql/hqlgram2.cpp

@@ -1490,7 +1490,9 @@ void HqlGram::doAddAssignment(IHqlExpression * transform, IHqlExpression * _fiel
 void HqlGram::appendTransformAssign(IHqlExpression * transform, IHqlExpression * to, IHqlExpression * _from, const attribute& errpos)
 {
     LinkedHqlExpr from = _from;
-    if (from->isConstant())
+    if (isNull(from))
+        from.setown(createNullExpr(to));
+    else if (from->isConstant())
         from.setown(ensureExprType(from, to->queryType()));
     if (to->isDataset() && !recordTypesMatch(from, to))
     {
@@ -1959,7 +1961,7 @@ void HqlGram::doCheckAssignedNormalizeTransform(HqlExprArray * assigns, IHqlExpr
                         if (cur->hasProperty(virtualAtom))
                         {
                             reportWarning(ERR_TRANS_NOVALUE4FIELD, errpos.pos, "Transform does not supply a value for field \"%s\"", fldName.str());
-                            OwnedHqlExpr null = createNullExpr(cur->queryType());
+                            OwnedHqlExpr null = createNullExpr(cur);
                             if (assigns)
                                 assigns->append(*createAssign(LINK(targetSelected), LINK(null)));
                             else
@@ -2268,6 +2270,35 @@ void HqlGram::addField(const attribute &errpos, _ATOM name, ITypeInfo *_type, IH
             value = newValue;
         }
     }
+    if (attrs)
+    {
+        HqlExprArray allAttrs;
+        attrs->unwindList(allAttrs, no_comma);
+        IHqlExpression * defaultAttr = queryProperty(defaultAtom, allAttrs);
+        if (defaultAttr)
+        {
+            IHqlExpression * defaultValue = defaultAttr->queryChild(0);
+            if (defaultValue)
+            {
+                ITypeInfo * defvalueType = defaultValue->queryType();
+                if (defvalueType != expectedType)
+                {
+                    if (!expectedType->assignableFrom(defvalueType->queryPromotedType()))
+                        canNotAssignTypeError(fieldType,defvalueType,errpos);
+                    if (castLosesInformation(expectedType, defvalueType))
+                        reportWarning(ERR_TYPE_INCOMPATIBLE, errpos.pos, "%s", "Default value too large");
+                    if (expectedType->getTypeCode() != type_row)
+                    {
+                        IHqlExpression * newValue = ensureExprType(defaultValue, expectedType);
+                        allAttrs.zap(*defaultAttr);
+                        allAttrs.append(*createAttribute(defaultAtom, newValue));
+                        attrs->Release();
+                        attrs = createComma(allAttrs);
+                    }
+                }
+            }
+        }
+    }
 
     switch (fieldType->getTypeCode())
     {
@@ -6646,7 +6677,7 @@ IHqlExpression * HqlGram::checkOutputRecord(IHqlExpression *record, const attrib
                     allConstant = false;                                        // no point reporting this as well.
 
                     HqlExprArray args;
-                    args.append(*createNullExpr(field->queryType()));
+                    args.append(*createNullExpr(field));
                     newField.setown(field->clone(args));
                 }
 

ecl/hql/hqllex.l

@@ -656,6 +656,7 @@ CRON                { RETURNSYM(CRON); }
 CSV                 { RETURNSYM(CSV); }
 DATASET             { RETURNSYM(DATASET); }
 DEDUP               { RETURNSYM(DEDUP); }
+DEFAULT             { RETURNSYM(DEFAULT); }
 DEFINE              { RETURNSYM(DEFINE); }
 DENORMALIZE         { RETURNSYM(DENORMALIZE); }
 DEPRECATED          { RETURNSYM(DEPRECATED); }

ecl/hql/hqlutil.cpp

@@ -5316,7 +5316,7 @@ void TempTableTransformer::createTempTableAssign(HqlExprArray & assigns, IHqlExp
                     castValue.setown(ensureExprType(src, type));
                 }
                 else
-                    castValue.setown(createNullExpr(type));
+                    castValue.setown(createNullExpr(expr));
             }
             assigns.append(*createAssign(LINK(target), LINK(castValue)));
             mapper.setMapping(target, castValue);

ecl/hqlcpp/hqlckey.cpp

@@ -362,8 +362,7 @@ void KeyedJoinInfo::buildClearRecord(BuildCtx & ctx, RecordSelectIterator & rawI
         OwnedHqlExpr rawSelect = rawIter.get();
         OwnedHqlExpr keySelect = keyIter.get();
 
-        ITypeInfo * keyFieldType = keySelect->queryType();
-        OwnedHqlExpr null = createNullExpr(keyFieldType);
+        OwnedHqlExpr null = createNullExpr(keySelect);
         OwnedHqlExpr keyNull = (rawIter.isInsideIfBlock() || (rawIter.isInsideNested() && isInPayload())) ? LINK(null) : getHozedKeyValue(null);
         OwnedHqlExpr folded = foldHqlExpression(keyNull);
         translator.buildAssign(ctx, rawSelect, folded);

ecl/hqlcpp/hqltcppc.cpp

@@ -2619,6 +2619,8 @@ void CXmlColumnInfo::buildColumnAssign(HqlCppTranslator & translator, BuildCtx &
     {
         _ATOM func = NULL;
         IHqlExpression * defaultValue = queryPropertyChild(column, xmlDefaultAtom, 0);
+        if (!defaultValue)
+            defaultValue = queryPropertyChild(column, defaultAtom, 0);
 
         switch (type->getTypeCode())
         {
@@ -2763,6 +2765,8 @@ IHqlExpression * CXmlColumnInfo::getXmlSetExpr(HqlCppTranslator & translator, Bu
     builder->buildDeclare(ctx);
 
     LinkedHqlExpr defaultValue = queryPropertyChild(column, xmlDefaultAtom, 0);
+    if (!defaultValue)
+        defaultValue.set(queryPropertyChild(column, defaultAtom, 0));
     bool defaultIsAllValue = defaultValue && (defaultValue->getOperator() == no_all);
     if (checkForAll)
     {
@@ -2802,6 +2806,8 @@ IHqlExpression * CXmlColumnInfo::getCallExpr(HqlCppTranslator & translator, Buil
     Linked<ITypeInfo> type = queryPhysicalType();
     _ATOM func = NULL;
     IHqlExpression * defaultValue = queryPropertyChild(column, xmlDefaultAtom, 0);
+    if (!defaultValue)
+        defaultValue = queryPropertyChild(column, defaultAtom, 0);
 
     switch (type->getTypeCode())
     {

ecl/hqlcpp/hqltcppc2.cpp

@@ -751,7 +751,7 @@ void CChildLinkedDatasetColumnInfo::setColumn(HqlCppTranslator & translator, Bui
     boundTarget.expr.setown(convertAddressToValue(addressData, queryType()));
 
     if (value->getOperator() == no_null)
-        value.setown(createNullExpr(resultType));
+        value.setown(createNullExpr(column));
     
     translator.buildDatasetAssign(ctx, boundTarget, value);
 }

ecl/hqlcpp/hqlttcpp.cpp

@@ -4542,7 +4542,7 @@ IHqlExpression * queryNullDsSelect(__int64 & selectIndex, IHqlExpression * value
     IValue * index = ds->queryChild(1)->queryValue();
     if (!index)
         return NULL;
-    if (!isNullExpr(other, value->queryType()))
+    if (!isNullExpr(other, value))
         return NULL;
     selectIndex = index->getIntValue();
     return ds->queryChild(0);

rtl/eclrtl/rtlds.cpp

@@ -453,7 +453,7 @@ inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer *
     for (unsigned i=0; i < count; i++)
     {
         byte *row = rows[i];
-        if (row)   // When serializing a dictionary, there may be nulls in the rowset. These can be skipped (we rehash on deserialize)
+        if (row)
         {
             serializer->serialize(out, rows[i]);
             if (isGrouped)
@@ -462,6 +462,10 @@ inline void doSerializeRowset(IRowSerializerTarget & out, IOutputRowSerializer *
                 out.put(1, &eogPending);
             }
         }
+        else
+        {
+            assert(isGrouped); // should not be seeing NULLs otherwise - should not use this function for DICTIONARY
+        }
     }
 }
 

testing/ecl/default.ecl

@@ -0,0 +1,44 @@
+// Test that default field values are used as many places as I can think of
+
+zero := nofold(0);
+
+// 1. when outputting an out-of-range row
+
+ds := DATASET([{ 'A' }], {string1 f {DEFAULT('B')}});
+
+//ds[0];      // should be constant folded
+//ds[zero];   // should NOT be
+
+dsn := DATASET([], {string1 f {DEFAULT('B')}});
+//dsn[0];     // should be constant folded
+//dsn[zero];  // can also be
+
+// 2. when looking up in a dictionary
+
+dict := DICTIONARY([{ 1 => 'A' }], { unsigned1 v => string1 f {DEFAULT('B')}});
+
+//dict[0];      // Might one day be constant folded
+//dict[zero];   // should NOT be
+
+dictn := DICTIONARY([], { unsigned1 v => string1 f {DEFAULT('B')}});   // Bit useless !
+
+//dictn[0];      // Might be constant folded
+//dictn[zero];   // Might be constant folded
+
+// 3. When assigning self := []
+
+ds t1 := TRANSFORM
+  SELF := [];
+END;
+
+PROJECT(ds, t1);
+
+// 4. In the default rows for outer joins
+
+ds t2(ds L, dsn R) := TRANSFORM
+  SELF.f := R.f;
+END;
+
+j := JOIN(ds, dsn, left.f=right.f, t2(LEFT, RIGHT),  LEFT OUTER);
+
+j[1].f;