HPCC-9145 Fix some memory leaks

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/eclcc/eclcc.cpp

@@ -1266,7 +1266,6 @@ void EclCC::generateOutput(EclCompileInstance & instance)
         else
         {
             // Output option settings
-            instance.wu->getDebugValues();
             Owned<IStringIterator> debugValues = &instance.wu->getDebugValues();
             ForEach (*debugValues)
             {

ecl/hql/hqlattr.cpp

@@ -1255,7 +1255,7 @@ ITypeInfo * getSerializedForm(ITypeInfo * type, _ATOM variation)
             IHqlExpression * serializedRecord = querySerializedForm(record, variation);
             if (record == serializedRecord)
                 return LINK(type);
-            return cloneModifiers(type, serializedRecord->getType());
+            return cloneModifiers(type, serializedRecord->queryType());
         }
     case type_row:
     case type_transform:

ecl/hql/hqlgram.hpp

@@ -965,7 +965,7 @@ protected:
     void transferOptions(attribute & filenameAttr, attribute & optionsAttr);
     IHqlExpression * extractTransformFromExtra(SharedHqlExpr & extra);
     void expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHqlSimpleScope * scope, ITypeInfo * & lastFieldType, const attribute & errpos);
-    void mergeDictionaryPayload(SharedHqlExpr & record, SharedHqlExpr & payload, const attribute & errpos);
+    void mergeDictionaryPayload(OwnedHqlExpr & record, IHqlExpression * payload, const attribute & errpos);
     void modifyIndexPayloadRecord(SharedHqlExpr & record, SharedHqlExpr & payload, SharedHqlExpr & extra, const attribute & errpos);
 
     bool haveAssignedToChildren(IHqlExpression * select);

ecl/hql/hqlgram.y

@@ -5115,7 +5115,7 @@ compareExpr
                             parser->normalizeExpression($1);
                             parser->normalizeExpression($4);
                             parser->normalizeExpression($4, type_dictionary, false);
-                            IHqlExpression *dict = $4.getExpr();
+                            OwnedHqlExpr dict = $4.getExpr();
                             OwnedHqlExpr row = createValue(no_rowvalue, makeNullType(), $1.getExpr());
                             OwnedHqlExpr indict = createINDictExpr(parser->errorHandler, $4.pos, row, dict);
                             $$.setExpr(getInverse(indict));
@@ -5126,8 +5126,8 @@ compareExpr
                             parser->normalizeExpression($1);
                             parser->normalizeExpression($4);
                             parser->normalizeExpression($4, type_dictionary, false);
-                            IHqlExpression *dict = $4.getExpr();
-                            IHqlExpression *row = $1.getExpr();
+                            OwnedHqlExpr dict = $4.getExpr();
+                            OwnedHqlExpr row = $1.getExpr();
                             OwnedHqlExpr indict = createINDictRow(parser->errorHandler, $4.pos, row, dict);
                             $$.setExpr(getInverse(indict));
                             $$.setPosition($3);
@@ -5137,7 +5137,7 @@ compareExpr
                             parser->normalizeExpression($1);
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($3, type_dictionary, false);
-                            IHqlExpression *dict = $3.getExpr();
+                            OwnedHqlExpr dict = $3.getExpr();
                             OwnedHqlExpr row = createValue(no_rowvalue, makeNullType(), $1.getExpr());
                             $$.setExpr(createINDictExpr(parser->errorHandler, $3.pos, row, dict));
                             $$.setPosition($2);
@@ -5147,8 +5147,8 @@ compareExpr
                             parser->normalizeExpression($1);
                             parser->normalizeExpression($3);
                             parser->normalizeExpression($3, type_dictionary, false);
-                            IHqlExpression *dict = $3.getExpr();
-                            IHqlExpression *row = $1.getExpr();
+                            OwnedHqlExpr dict = $3.getExpr();
+                            OwnedHqlExpr row = $1.getExpr();
                             $$.setExpr(createINDictRow(parser->errorHandler, $3.pos, row, dict));
                             $$.setPosition($2);
                         }
@@ -6859,8 +6859,9 @@ dataRow
                         {
                             HqlExprArray args;
                             $3.unwindCommaList(args);
+                            OwnedHqlExpr dict = $1.getExpr();
                             OwnedHqlExpr row = createValue(no_rowvalue, makeNullType(), args);
-                            $$.setExpr(createSelectMapRow(parser->errorHandler, $3.pos, $1.getExpr(), row.getClear()));
+                            $$.setExpr(createSelectMapRow(parser->errorHandler, $3.pos, dict, row));
                         }
     | dataSet '[' NOBOUNDCHECK expression ']'
                         {   

ecl/hql/hqlgram2.cpp

@@ -7841,7 +7841,7 @@ void HqlGram::expandPayload(HqlExprArray & fields, IHqlExpression * payload, IHq
     }
 }
 
-void HqlGram::mergeDictionaryPayload(SharedHqlExpr & record, SharedHqlExpr & payload, const attribute & errpos)
+void HqlGram::mergeDictionaryPayload(OwnedHqlExpr & record, IHqlExpression * payload, const attribute & errpos)
 {
     checkRecordIsValid(errpos, record.get());
     IHqlSimpleScope * scope = record->querySimpleScope();
@@ -7948,7 +7948,7 @@ void HqlGram::extractIndexRecordAndExtra(SharedHqlExpr & record, SharedHqlExpr &
     if (payload)
     {
         extra.setown(createComma(extra.getClear(), LINK(payload)));
-        record.setown(removeProperty(record.getClear(), _payload_Atom));
+        record.setown(removeProperty(record, _payload_Atom));
     }
 }
 

ecl/hql/hqlir.cpp

@@ -1906,7 +1906,7 @@ id_t ExpressionIRPlayer::processExpr(IHqlExpression * expr)
     expr->setTransformExtraUnlinked(expr);
 
     id_t nextId = doProcessExpr(expr);
-    expr->setTransformExtra(new ExpressionId(nextId));
+    expr->setTransformExtraOwned(new ExpressionId(nextId));
     return nextId;
 }
 

ecl/hql/hqlutil.cpp

@@ -5486,7 +5486,7 @@ IHqlExpression * createSelectMapRow(IErrorReceiver * errors, ECLlocation & locat
     OwnedHqlExpr record = getDictionarySearchRecord(dict->queryRecord());
     TempTableTransformer transformer(errors, location, true);
     OwnedHqlExpr newTransform = transformer.createTempTableTransform(values, record);
-    return createRow(no_selectmap, dict, createRow(no_createrow, newTransform.getClear()));
+    return createRow(no_selectmap, LINK(dict), createRow(no_createrow, newTransform.getClear()));
 }
 
 IHqlExpression *createINDictExpr(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *expr, IHqlExpression *dict)
@@ -5494,7 +5494,7 @@ IHqlExpression *createINDictExpr(IErrorReceiver * errors, ECLlocation & location
     OwnedHqlExpr record = getDictionarySearchRecord(dict->queryRecord());
     TempTableTransformer transformer(errors, location, true);
     OwnedHqlExpr newTransform = transformer.createTempTableTransform(expr, record);
-    return createBoolExpr(no_indict, createRow(no_createrow, newTransform.getClear()), dict);
+    return createBoolExpr(no_indict, createRow(no_createrow, newTransform.getClear()), LINK(dict));
 }
 
 IHqlExpression *createINDictRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression *row, IHqlExpression *dict)
@@ -5502,7 +5502,7 @@ IHqlExpression *createINDictRow(IErrorReceiver * errors, ECLlocation & location,
     OwnedHqlExpr record = getDictionarySearchRecord(dict->queryRecord());
     Owned<ITypeInfo> rowType = makeRowType(record->getType());
     OwnedHqlExpr castRow = ensureExprType(row, rowType);
-    return createBoolExpr(no_indict, castRow.getClear(), dict);
+    return createBoolExpr(no_indict, castRow.getClear(), LINK(dict));
 }
 
 IHqlExpression * convertTempRowToCreateRow(IErrorReceiver * errors, ECLlocation & location, IHqlExpression * expr)

ecl/hql/hqlvalid.cpp

@@ -191,7 +191,7 @@ IHqlExpression * queryAmbiguousRollupCondition(IHqlExpression * expr, bool stric
     ForEachItemIn(i, selects)
     {
         IHqlExpression * select = &selects.item(i);
-        IHqlExpression * value = getExtractSelect(transform, selector, select);
+        OwnedHqlExpr value = getExtractSelect(transform, selector, select);
         if (!value)
             return select;
 
@@ -227,7 +227,7 @@ IHqlExpression * queryAmbiguousRollupCondition(IHqlExpression * expr, const HqlE
     ForEachItemIn(i, selects)
     {
         IHqlExpression * select = &selects.item(i);
-        IHqlExpression * value = getExtractSelect(transform, selector, select);
+        OwnedHqlExpr value = getExtractSelect(transform, selector, select);
         if (!value)
             return select;
 
@@ -259,7 +259,7 @@ void filterAmbiguousRollupCondition(HqlExprArray & ambiguousSelects, HqlExprArra
     ForEachItemInRev(i, selects)
     {
         IHqlExpression * select = &selects.item(i);
-        IHqlExpression * value = getExtractSelect(transform, selector, select);
+        OwnedHqlExpr value = getExtractSelect(transform, selector, select);
         if (value)
         {
             if (value->getOperator() == no_constant)

ecl/hqlcpp/hqlcpp.cpp

@@ -7622,7 +7622,7 @@ void HqlCppTranslator::doBuildExprNot(BuildCtx & ctx, IHqlExpression * expr, CHq
     assertex(expr->queryChild(0)->isBoolean());
     CHqlBoundExpr bound;
     buildExpr(ctx, expr->queryChild(0), bound);
-    tgt.expr.set(getInverse(bound.expr));
+    tgt.expr.setown(getInverse(bound.expr));
 }
 
 

ecl/hqlcpp/hqlcppds.cpp

@@ -4888,14 +4888,14 @@ IReferenceSelector * HqlCppTranslator::buildDatasetSelectMap(BuildCtx & ctx, IHq
 
     //MORE: This should really be a createDictionarySelector call.
     Owned<IHqlCppDatasetCursor> cursor = createDatasetSelector(ctx, dictionary);
-    BoundRow * row = cursor->buildSelectMap(ctx, expr);
+    Owned<BoundRow> row = cursor->buildSelectMap(ctx, expr);
 
     if (!row)
     {
         CHqlBoundExpr boundCleared;
         buildDefaultRow(ctx, dictionary, boundCleared);
         OwnedHqlExpr defaultRowPtr = getPointer(boundCleared.expr);
-        row = bindRow(ctx, expr, defaultRowPtr);
+        row.setown(bindRow(ctx, expr, defaultRowPtr));
     }
 
     return createReferenceSelector(row);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -7612,8 +7612,8 @@ void HqlCppTranslator::doBuildStmtEnsureResult(BuildCtx & ctx, IHqlExpression *
         if (filename)
         {
             args.append(*LINK(filename));
-            IHqlExpression * fileExists = createValue(no_not, makeBoolType(), bindFunctionCall(fileExistsAtom, args));
-            checkExists.setown(createBoolExpr(no_or, checkExists.getClear(), fileExists));
+            OwnedHqlExpr fileExists = createValue(no_not, makeBoolType(), bindFunctionCall(fileExistsAtom, args));
+            checkExists.setown(createBoolExpr(no_or, checkExists.getClear(), fileExists.getClear()));
         }
     }
 

ecl/hqlcpp/hqliproj.cpp

@@ -457,7 +457,7 @@ void UsedFieldSet::calcFinalRecord(bool canPack, bool ignoreIfEmpty)
         finalRecord.setown(getPackedRecord(finalRecord));
 }
 
-void UsedFieldSet::gatherExpandSelectsUsed(HqlExprArray * selfSelects, SelectUsedArray * parentSelects, IHqlExpression * selector, IHqlExpression * source)
+void UsedFieldSet::gatherExpandSelectsUsed(HqlExprArray * selfSelects, HqlExprArray * parentSelects, IHqlExpression * selector, IHqlExpression * source)
 {
     assertex(selfSelects ? selector != NULL : true);
     for (unsigned i1 = maxGathered; i1 < fields.ordinality(); i1++)
@@ -507,7 +507,7 @@ inline bool isSelector(IHqlExpression * expr)
     return (expr->getOperator() == no_select) && !isNewSelector(expr);
 }
 
-void UsedFieldSet::gatherTransformValuesUsed(HqlExprArray * selfSelects, SelectUsedArray * parentSelects, HqlExprArray * values, IHqlExpression * selector, IHqlExpression * transform)
+void UsedFieldSet::gatherTransformValuesUsed(HqlExprArray * selfSelects, HqlExprArray * parentSelects, HqlExprArray * values, IHqlExpression * selector, IHqlExpression * transform)
 {
     for (unsigned i = maxGathered; i < fields.ordinality(); i++)
     {
@@ -2213,6 +2213,15 @@ void ImplicitProjectTransformer::processSelects(ComplexImplicitProjectInfo * ext
     }
 }
 
+void ImplicitProjectTransformer::processSelects(ComplexImplicitProjectInfo * extra, HqlExprArray const & selectsUsed, IHqlExpression * ds, IHqlExpression * leftSelect, IHqlExpression * rightSelect)
+{
+    ForEachItemIn(i2, selectsUsed)
+    {
+        IHqlExpression * curSelect = &selectsUsed.item(i2);
+        processSelect(extra, curSelect, ds, leftSelect, rightSelect);
+    }
+}
+
 
 void ImplicitProjectTransformer::processTransform(ComplexImplicitProjectInfo * extra, IHqlExpression * transform, IHqlExpression * dsSelect, IHqlExpression * leftSelect, IHqlExpression * rightSelect)
 {
@@ -2301,7 +2310,7 @@ void ImplicitProjectTransformer::calculateFieldsUsed(IHqlExpression * expr)
                 extra->addAllOutputs();
 
             //MORE: querySelectsUsedForField() could be optimized by creating a map first, but it is only ~1% of time, so not really worth it.
-            SelectUsedArray parentSelects;
+            HqlExprArray parentSelects;
             HqlExprArray values;
             extra->outputFields.gatherTransformValuesUsed(NULL, &parentSelects, &values, NULL, transform);
             processSelects(extra, parentSelects, dsSelect, leftSelect, rightSelect);
@@ -2476,7 +2485,7 @@ void ImplicitProjectTransformer::calculateFieldsUsed(IHqlExpression * expr)
                 //NB: outputfields can extend...
                 while (!extra->outputFields.allGathered())
                 {
-                    SelectUsedArray parentSelects;
+                    HqlExprArray parentSelects;
                     HqlExprArray values;
                     HqlExprArray selfSelects;
                     extra->outputFields.gatherTransformValuesUsed(&selfSelects, &parentSelects, &values, dsSelect, transform);
@@ -2544,7 +2553,7 @@ void ImplicitProjectTransformer::calculateFieldsUsed(IHqlExpression * expr)
 
             while (!extra->outputFields.allGathered())
             {
-                SelectUsedArray parentSelects;
+                HqlExprArray parentSelects;
                 HqlExprArray values;
                 HqlExprArray selfSelects;
                 extra->outputFields.gatherTransformValuesUsed(&selfSelects, &parentSelects, &values, left, transform);

ecl/hqlcpp/hqliproj.ipp

@@ -130,7 +130,7 @@ public:
     IHqlExpression * createFilteredTransform(IHqlExpression * transform, const UsedFieldSet * exceptions) const;
     void calcFinalRecord(bool canPack, bool ignoreIfEmpty);
     NestedField * findNested(IHqlExpression * field) const;
-    void gatherTransformValuesUsed(HqlExprArray * selfSelects, SelectUsedArray * parentSelects, HqlExprArray * values, IHqlExpression * selector, IHqlExpression * transform);
+    void gatherTransformValuesUsed(HqlExprArray * selfSelects, HqlExprArray * parentSelects, HqlExprArray * values, IHqlExpression * selector, IHqlExpression * transform);
     void getText(StringBuffer & s) const;
     void intersectFields(const UsedFieldSet & source);
     bool isEmpty() const;
@@ -161,7 +161,7 @@ protected:
     void createFilteredAssigns(HqlExprArray & assigns, IHqlExpression * transform, IHqlExpression * newSelf, const UsedFieldSet * exceptions) const;
     IHqlExpression * createRowTransform(IHqlExpression * row, const UsedFieldSet * exceptions) const;
     void kill();
-    void gatherExpandSelectsUsed(HqlExprArray * selfSelects, SelectUsedArray * parentSelects, IHqlExpression * selector, IHqlExpression * source);
+    void gatherExpandSelectsUsed(HqlExprArray * selfSelects, HqlExprArray * parentSelects, IHqlExpression * selector, IHqlExpression * source);
     unsigned getOriginalPosition(IHqlExpression * field) const;
 
 protected:
@@ -380,6 +380,7 @@ protected:
 
     void processSelect(ComplexImplicitProjectInfo * extra, IHqlExpression * curSelect, IHqlExpression * ds, IHqlExpression * leftSelect, IHqlExpression * rightSelect);
     void processSelects(ComplexImplicitProjectInfo * extra, SelectUsedArray const & selectsUsed, IHqlExpression * ds, IHqlExpression * leftSelect, IHqlExpression * rightSelect);
+    void processSelects(ComplexImplicitProjectInfo * extra, HqlExprArray const & selectsUsed, IHqlExpression * ds, IHqlExpression * leftSelect, IHqlExpression * rightSelect);
     void processTransform(ComplexImplicitProjectInfo * extra, IHqlExpression * transform, IHqlExpression * ds, IHqlExpression * leftSelect, IHqlExpression * rightSelect);
 
 protected:

ecl/regress/regress.sh

@@ -37,6 +37,7 @@ userflags=
 eclcc=
 diff=
 query=
+valgrind=
 np=`grep -c processor /proc/cpuinfo`
 export ECLCC_ECLINCLUDE_PATH=
 
@@ -56,7 +57,7 @@ if [[ $1 = '' ]]; then
     exit -1
 fi
 if [[ $* != '' ]]; then
-    while getopts "t:c:I:e:d:f:q:l:x:" opt; do
+    while getopts "t:c:I:e:d:f:q:l:x:v" opt; do
         case $opt in
             t)
                 target_dir=$OPTARG
@@ -85,6 +86,11 @@ if [[ $* != '' ]]; then
             x)
                 userflags="$userflags $OPTARG"
                 ;;
+            v)
+               	valgrind="valgrind --leak-check=full --suppressions=suppressions.txt "
+               	eclcc="$valgrind $eclcc"
+               	compare_dir=
+               	;;
             :)
                 echo $syntax
                 exit -1

testing/ecl/dict1.ecl

@@ -13,7 +13,6 @@ d := nofold(DATASET(
    {1, [{'Richard' => 100}]},
    {1, [{'Richard' => 100}]}
   ], { unsigned top, DICTIONARY({STRING8 name => UNSIGNED iq}) nest}));
-//d := nofold(DATASET([{1, [{'Richard' , 50}]}, {1, [{'Richard' , 100}]}], { unsigned top, DATASET({STRING8 name , UNSIGNED iq}) nest}));
 
 d t(d L, d R) := TRANSFORM
   SELF.nest := L.nest + R.nest;