Merge pull request #4106 from ghalliday/issue8586

HPCC-8586 Remove code that tries to percolate maxlength when projecting

Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hqlerrors.hpp

@@ -63,7 +63,6 @@
 #define WRN_SORT_INVARIANT          1038
 #define WRN_POSSIBLE_FILTER         1039
 #define WRN_BAD_LOOPFLAG            1040
-#define WRN_RECORDDEFAULTMAXLENGTH  1041
 #define WRN_COND_ASSIGN_NO_PREV     1042
 #define WRN_NOT_INTERFACE           1043
 #define WRN_PACKED_MAY_CHANGE       1044

ecl/hql/hqlgram2.cpp

@@ -7160,7 +7160,6 @@ static void unwindExtraFields(HqlExprArray & fields, IHqlExpression * expr)
 
 IHqlExpression * HqlGram::createRecordUnion(IHqlExpression * left, IHqlExpression * right, const attribute & errpos)
 {
-    //MORE: maxlength should be handled better - but should really be specified on the fields.
     RecordFieldDifference compare(left, *this, errpos);
     OwnedHqlExpr extra = ::createRecordExcept(right, compare);
 
@@ -7199,7 +7198,6 @@ IHqlExpression * HqlGram::createIndexFromRecord(IHqlExpression * record, IHqlExp
 void HqlGram::inheritRecordMaxLength(IHqlExpression * dataset, SharedHqlExpr & record)
 {
     IHqlExpression * maxLength = queryRecordProperty(dataset->queryRecord(), maxLengthAtom);
-//  if (maxLength && isVariableSizeRecord(record) && !queryRecordProperty(record, maxLengthAtom))
     if (maxLength && !queryRecordProperty(record, maxLengthAtom))
     {
         HqlExprArray fields;
@@ -9597,7 +9595,6 @@ IHqlExpression * HqlGram::createIffDataset(IHqlExpression * record, IHqlExpressi
 
 IHqlExpression * HqlGram::createIff(attribute & condAttr, attribute & leftAttr, attribute & rightAttr)
 {
-    //MORE: PromoteIfType should ideally add a maxlength to the type if possible.
     ITypeInfo * type = checkPromoteIfType(leftAttr, rightAttr);
     OwnedHqlExpr left = leftAttr.getExpr();
     OwnedHqlExpr right = rightAttr.getExpr();
@@ -10883,7 +10880,6 @@ IHqlExpression* HqlGram::createDefJoinTransform(IHqlExpression* left,IHqlExpress
         // create result record
         CHqlRecord * newRec = (CHqlRecord *)createRecord();
 
-        //MORE: maxlength should be handled better
         expandDefJoinAttrs(newRec, leftRecord);
         expandDefJoinAttrs(newRec, rightRecord);
         //Clone left - including ifblocks etc.

ecl/hql/hqlutil.cpp

@@ -3179,30 +3179,6 @@ void SubStringHelper::init(IHqlExpression * _src, IHqlExpression * range)
 }
 
 
-bool hasMaxLength(IHqlExpression * record)
-{
-    ForEachChild(i, record)
-    {
-        IHqlExpression * cur = record->queryChild(i);
-        switch (cur->getOperator())
-        {
-        case no_record:
-            //inherited maxlength?
-            if (hasMaxLength(cur))
-                return true;
-            break;
-        case no_attr:
-        case no_attr_expr:
-        case no_attr_link:
-            if (cur->queryName() == maxLengthAtom)
-                return true;
-            break;
-        }
-    }
-    return false;
-}
-
-
 void unwindFields(HqlExprArray & fields, IHqlExpression * record)
 {
     ForEachChild(i, record)
@@ -4066,7 +4042,6 @@ extern HQL_API IHqlExpression * convertScalarAggregateToDataset(IHqlExpression *
         return NULL;
     }
 
-    //more: InheritMaxlength
     OwnedHqlExpr field;
     if ((newop == no_mingroup || newop == no_maxgroup) && (arg->getOperator() == no_select))
         field.set(arg->queryChild(1));                  // inherit maxlength etc...
@@ -6136,7 +6111,6 @@ WarningProcessor::WarningProcessor()
 { 
     firstLocalOnWarning = 0; 
     activeSymbol = NULL; 
-//  addGlobalOnWarning(WRN_RECORDDEFAULTMAXLENGTH, errorAtom);
 }
 
 

ecl/hql/hqlutil.hpp

@@ -100,7 +100,6 @@ extern HQL_API unsigned countTotalFields(IHqlExpression * record, bool includeVi
 extern HQL_API bool transformContainsSkip(IHqlExpression * transform);
 extern HQL_API bool transformListContainsSkip(IHqlExpression * transforms);
 extern HQL_API bool recordContainsNestedRecord(IHqlExpression * record);
-extern HQL_API bool hasMaxLength(IHqlExpression * record);
 
 extern HQL_API IHqlExpression * queryInvalidCsvRecordField(IHqlExpression * expr);
 extern HQL_API bool isValidCsvRecord(IHqlExpression * expr);

ecl/hqlcpp/hqlcerrors.hpp

@@ -120,7 +120,6 @@
 #define HQLERR_KeyedJoinTooComplex              4097
 #define HQLERR_KeyAccessNeedCast                4098
 #define HQLERR_KeyAccessNoKeyField              4099
-#define HQLERR_MaxLengthNotSetOnRecord          4100
 #define HQLERR_NotSupportedInsideNoThor         4102
 #define HQLERR_RegexNoTransformSupport          4103
 #define HQLERR_AccessMatchAttrInChildQuery      4104
@@ -401,7 +400,6 @@
 #define HQLERR_KeyedJoinTooComplex_Text         "Key condition (%s) is too complex, it cannot be done with the key."
 #define HQLERR_KeyAccessNeedCast_Text           "Key condition (%s) requires casts on comparison of field '%s'"
 #define HQLERR_KeyAccessNoKeyField_Text         "Key condition (%s) does not have any comparisons against key fields"
-#define HQLERR_MaxLengthNotSetOnRecord_Text     "No explicit maxlength provided for record %s"
 #define HQLERR_MinusOnString_Text               "unary - cannot be performed on a string"
 #define HQLERR_NotSupportedInsideNoThor_Text    "%s is not supported inside NOTHOR()"
 #define HQLERR_RegexNoTransformSupport_Text     "Regular expression parsing doesn't support productions - need to use tomita"

ecl/hqlcpp/hqlckey.cpp

@@ -828,7 +828,7 @@ void KeyedJoinInfo::optimizeTransfer(SharedHqlExpr & targetDataset, SharedHqlExp
             OwnedHqlExpr newExtraFilter = hasExtra ? optimizeTransfer(fields, values, *extraFilter, oldLeft) : NULL;
             if (newFilter && (newExtraFilter || !hasExtra) && fields.ordinality() < getFieldCount(dataset->queryRecord()))
             {
-                OwnedHqlExpr extractedRecord = translator.createRecordInheritMaxLength(fields, dataset);
+                OwnedHqlExpr extractedRecord = createRecord(fields);
                 OwnedHqlExpr serializedRecord = getSerializedForm(extractedRecord, diskAtom);
                 targetDataset.setown(createDataset(no_anon, LINK(serializedRecord), NULL));
 
@@ -1003,7 +1003,7 @@ void KeyedJoinInfo::optimizeExtractJoinFields()
         if (extractedRecord || (fieldsAccessed.ordinality() != 0))
         {
             if (!extractedRecord)
-                extractedRecord.setown(translator.createRecordInheritMaxLength(fieldsAccessed, rawRhs));
+                extractedRecord.setown(createRecord(fieldsAccessed));
             extractJoinFieldsRecord.setown(getSerializedForm(extractedRecord, diskAtom));
             OwnedHqlExpr self = getSelf(extractJoinFieldsRecord);
             OwnedHqlExpr memorySelf = getSelf(extractedRecord);

ecl/hqlcpp/hqlcpp.ipp

@@ -1004,7 +1004,6 @@ public:
     unsigned getMaxRecordSize(IHqlExpression * record);
     IHqlExpression * getRecordSize(IHqlExpression * dataset);
     unsigned getCsvMaxLength(IHqlExpression * csvAttr);
-    IHqlExpression * createRecordInheritMaxLength(HqlExprArray & fields, IHqlExpression * donor);
     void ensureRowSerializer(StringBuffer & serializerName, BuildCtx & ctx, IHqlExpression * record, _ATOM format, _ATOM kind);
     void ensureRowPrefetcher(StringBuffer & prefetcherName, BuildCtx & ctx, IHqlExpression * record);
     IHqlExpression * createSerializer(BuildCtx & ctx, IHqlExpression * record, _ATOM format, _ATOM kind);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -329,8 +329,6 @@ ColumnToOffsetMap * RecordOffsetMap::queryMapping(IHqlExpression * record, unsig
         match = new ColumnToOffsetMap(record, 1, maxRecordSize, false);
         match->init(*this);
         addOwn(*match);
-        if (maxRecordSize == 0)
-            match->checkValidMaxSize();
     }
     return match;
 }
@@ -2207,7 +2205,12 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
                 size32_t minSize = getMinRecordSize(record);
                 size32_t expectedSize = getExpectedRecordSize(record);
                 StringBuffer temp;
-                temp.append(minSize).append("..").append(maxSize).append("(").append(expectedSize).append(")");
+                temp.append(minSize).append("..");
+                if (maxRecordSizeUsesDefault(record))
+                    temp.append("?");
+                else
+                    temp.append(maxSize);
+                temp.append("(").append(expectedSize).append(")");
                 addAttribute("recordSize", temp.str());
             }
             else
@@ -11362,7 +11365,7 @@ void HqlCppTranslator::generateSerializeKey(BuildCtx & nestedctx, node_operator
             BuildCtx classctx(nestedctx);
             beginNestedClass(classctx, memberName, "ISortKeySerializer");
 
-            IHqlExpression * keyRecord = createRecordInheritMaxLength(keyFields, record);
+            IHqlExpression * keyRecord = createRecord(keyFields);
             Owned<IHqlExpression> keyDataset = createDataset(no_anon, keyRecord);
 
             DatasetReference keyActiveRef(keyDataset, no_activetable, NULL);
@@ -13581,7 +13584,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityDedup(BuildCtx & ctx, IHqlExpr
             selects.append(*createSelectExpr(getActiveTableSelector(), LINK(field)));
         }
 
-        OwnedHqlExpr keyDataset = createDataset(no_anon, createRecordInheritMaxLength(fields, dataset));
+        OwnedHqlExpr keyDataset = createDataset(no_anon, createRecord(fields));
 
         //virtual IOutputMetaData * queryKeySize()
         buildMetaMember(instance->classctx, keyDataset, false, "queryKeySize");
@@ -17703,39 +17706,6 @@ bool HqlCppTranslator::recordContainsIfBlock(IHqlExpression * record)
     return queryRecordOffsetMap(record)->queryContainsIfBlock();
 }
 
-IHqlExpression * HqlCppTranslator::createRecordInheritMaxLength(HqlExprArray & fields, IHqlExpression * donor)
-{
-    IHqlExpression * record = donor->queryRecord();
-    unsigned prevLength = fields.ordinality();
-    LinkedHqlExpr max;
-
-    if (!queryProperty(maxLengthAtom, fields))
-    {
-        max.set(donor->queryProperty(maxLengthAtom));
-        if (!max && hasMaxLength(record))
-        {
-            //maxlength inherited somewhere...
-            OwnedHqlExpr serializedDonorRecord = getSerializedForm(record, diskAtom);
-            ColumnToOffsetMap * map = queryRecordOffsetMap(serializedDonorRecord);
-            max.setown(createAttribute(maxLengthAtom, getSizetConstant(map->getMaxSize())));
-        }
-
-        if (max)
-            fields.append(*LINK(max));
-    }
-
-    OwnedHqlExpr ret = createRecord(fields);
-    fields.trunc(prevLength);
-    if (max)
-    {
-        OwnedHqlExpr serialized = getSerializedForm(ret, diskAtom);
-        if (isFixedRecordSize(serialized))
-            ret.setown(createRecord(fields));
-    }
-    return ret.getClear();
-}
-
-
 //-- Code to transform the expressions ready for generating source code.
 
 static void logECL(const LogMsgCategory & category, size32_t len, const char * ecl)

ecl/hqlcpp/hqliproj.cpp

@@ -455,26 +455,6 @@ void UsedFieldSet::calcFinalRecord(bool canPack, bool ignoreIfEmpty)
     finalRecord.setown(createRecord(recordFields));
     if (canPack)
         finalRecord.setown(getPackedRecord(finalRecord));
-
-    OwnedHqlExpr serializedRecord = getSerializedForm(finalRecord, diskAtom);
-    if (maxRecordSizeUsesDefault(serializedRecord))
-    {
-        HqlExprArray recordFields;
-        unwindChildren(recordFields, finalRecord);
-        //Lost some indication of the record size->add an attribute
-        IHqlExpression * max = originalRecord->queryProperty(maxLengthAtom);
-        if (max)
-            recordFields.append(*LINK(max));
-        else
-        {
-            bool isKnownSize, useDefaultRecordSize;
-            OwnedHqlExpr oldSerializedRecord = getSerializedForm(originalRecord, diskAtom);
-            unsigned oldRecordSize = getMaxRecordSize(oldSerializedRecord, 0, isKnownSize, useDefaultRecordSize);
-            if (!useDefaultRecordSize)
-                recordFields.append(*createAttribute(maxLengthAtom, getSizetConstant(oldRecordSize)));
-        }
-        finalRecord.setown(createRecord(recordFields));
-    }
 }
 
 void UsedFieldSet::gatherExpandSelectsUsed(HqlExprArray * selfSelects, SelectUsedArray * parentSelects, IHqlExpression * selector, IHqlExpression * source)

ecl/hqlcpp/hqlsource.cpp

@@ -7094,7 +7094,6 @@ void FetchBuilder::buildMembers(IHqlExpression * expr)
         }
     case no_xml:
         {
-            //MORE: MaxLength?
             // virtual const char * getXmlIteratorPath()
             translator.doBuildVarStringFunction(instance->classctx, "getXmlIteratorPath", queryRealChild(tableExpr->queryChild(2), 0));
             break;

ecl/hqlcpp/hqltcppc.cpp

@@ -3088,17 +3088,6 @@ void ColumnToOffsetMap::ensureMaxSizeCached()
     }
 }
 
-void ColumnToOffsetMap::checkValidMaxSize()
-{
-    ensureMaxSizeCached();
-    if (!cachedDefaultMaxSizeUsed)
-        return;
-
-    StringBuffer s;
-    getExprECL(record, s);
-    throwError1(HQLERR_MaxLengthNotSetOnRecord, s.str());
-}
-
 unsigned ColumnToOffsetMap::getMaxSize()
 {
     ensureMaxSizeCached();

ecl/hqlcpp/hqltcppc.ipp

@@ -526,7 +526,6 @@ public:
     virtual const void * getKey() const { return &record; }
 
     void init(RecordOffsetMap & map);
-    void checkValidMaxSize();
     unsigned getFixedRecordSize();
     bool isFixedWidth() const                       { return fixedSizeRecord; }
     unsigned getMaxSize();

ecl/hqlcpp/hqlttcpp.cpp

@@ -3276,9 +3276,6 @@ IHqlExpression * ThorHqlTransformer::normalizeTableToAggregate(IHqlExpression *
     HqlExprArray aggregateAssigns;
     HqlExprArray extraAssigns;
 
-    IHqlExpression * maxLength = queryRecordProperty(record, maxLengthAtom);
-    if (maxLength)
-        aggregateFields.append(*LINK(maxLength));
     bool extraSelectNeeded = false;
     OwnedHqlExpr self = getSelf(expr);
     ForEachChild(idx, transform)
@@ -3338,7 +3335,7 @@ IHqlExpression * ThorHqlTransformer::normalizeTableToAggregate(IHqlExpression *
         newGroupBy = createSortList(newGroupElement);
     }
 
-    IHqlExpression * aggregateRecord = extraSelectNeeded ? translator.createRecordInheritMaxLength(aggregateFields, record) : LINK(record);
+    IHqlExpression * aggregateRecord = extraSelectNeeded ? createRecord(aggregateFields) : LINK(record);
     OwnedHqlExpr aggregateSelf = getSelf(aggregateRecord);
     replaceAssignSelector(aggregateAssigns, aggregateSelf);
     IHqlExpression * aggregateTransform = createValue(no_newtransform, makeTransformType(aggregateRecord->getType()), aggregateAssigns);