Merge remote-tracking branch 'origin/candidate-3.10.0' into candidate-3.10.x

Conflicts:
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/ft/daftformat.cpp

@@ -1245,6 +1245,9 @@ void CRemotePartitioner::callRemote()
             unsigned compatflags = 0;                   // compatibility flags (not yet used)
             msg.append(compatflags);
             msg.append(decryptKey);
+            //Add extra data at the end to provide backward compatibility
+            srcFormat.serializeExtra(msg, 1);
+            tgtFormat.serializeExtra(msg, 1);
 
             if (!catchWriteBuffer(socket, msg))
                 throwError1(RFSERR_TimeoutWaitConnect, url.str());

dali/ft/filecopy.cpp

@@ -360,6 +360,9 @@ bool FileTransferThread::performTransfer()
         msg.append(sprayer.encryptKey);
         msg.append(sprayer.decryptKey);
 
+        sprayer.srcFormat.serializeExtra(msg, 1);
+        sprayer.tgtFormat.serializeExtra(msg, 1);
+
         if (!catchWriteBuffer(socket, msg))
             throwError1(RFSERR_TimeoutWaitConnect, url.str());
 

dali/ft/filecopy.hpp

@@ -46,12 +46,14 @@ public:
     FileFormat(FileFormatType _type = FFTunknown, unsigned _recordSize = 0) { set(_type, _recordSize); maxRecordSize = 0;}
 
     void deserialize(MemoryBuffer & in);
+    void deserializeExtra(MemoryBuffer & in, unsigned version);
     bool equals(const FileFormat & other) const     { return (type == other.type) && (recordSize == other.recordSize); }
     unsigned getUnitSize() const;
     bool isUtf() const                              { return (type >= FFTutf) && (type <= FFTutf32le); }
     bool restore(IPropertyTree * props);
     void save(IPropertyTree * props);
     void serialize(MemoryBuffer & out) const;
+    void serializeExtra(MemoryBuffer & out, unsigned version) const;
     void set(FileFormatType _type, unsigned _recordSize = 0) { type = _type, recordSize = _recordSize; }
     void set(const FileFormat & src);
 

dali/ft/ftbase.cpp

@@ -227,13 +227,28 @@ void FileFormat::deserialize(MemoryBuffer & in)
         ::deserialize(in, separate);
         ::deserialize(in, quote);
         ::deserialize(in, terminate);
-        ::deserialize(in, escape);
         ::deserialize(in, rowTag);
         break;
     }
 }
 
 
+void FileFormat::deserializeExtra(MemoryBuffer & in, unsigned version)
+{
+    switch (type)
+    {
+    case FFTcsv:
+    case FFTutf:
+    case FFTutf8: case FFTutf8n:
+    case FFTutf16: case FFTutf16be: case FFTutf16le:
+    case FFTutf32: case FFTutf32be: case FFTutf32le:
+        if (version == 1)
+            ::deserialize(in, escape);
+        break;
+    }
+}
+
+
 unsigned FileFormat::getUnitSize() const
 {
     switch (type)
@@ -386,12 +401,26 @@ void FileFormat::serialize(MemoryBuffer & out) const
         ::serialize(out, separate);
         ::serialize(out, quote);
         ::serialize(out, terminate);
-        ::serialize(out, escape);
         ::serialize(out, rowTag);
         break;
     }
 }
 
+void FileFormat::serializeExtra(MemoryBuffer & out, unsigned version) const
+{
+    switch (type)
+    {
+    case FFTcsv:
+    case FFTutf:
+    case FFTutf8: case FFTutf8n:
+    case FFTutf16: case FFTutf16be: case FFTutf16le:
+    case FFTutf32: case FFTutf32be: case FFTutf32le:
+        if (version == 1)
+            ::serialize(out, escape);
+        break;
+    }
+}
+
 void FileFormat::set(const FileFormat & src) 
 { 
     type = src.type; 

dali/ft/ftslave.cpp

@@ -83,6 +83,12 @@ bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryB
     StringAttr encryptkey;
     if (msg.remaining())
         msg.read(encryptkey);
+    if (msg.remaining())
+    {
+        srcFormat.deserializeExtra(msg, 1);
+        tgtFormat.deserializeExtra(msg, 1);
+    }
+
     StringBuffer text;
     fullPath.getRemotePath(text);
     LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());

dali/ft/fttransform.cpp

@@ -631,7 +631,11 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
         msg.read(transferBufferSize);
     if (msg.remaining()) 
         msg.read(encryptKey).read(decryptKey);
-
+    if (msg.remaining())
+    {
+        srcFormat.deserializeExtra(msg, 1);
+        tgtFormat.deserializeExtra(msg, 1);
+    }
 
     LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize);
     PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0);

ecl/hql/hqltrans.ipp

@@ -150,7 +150,7 @@ public:
 #endif
         noteMemory();
         unlockTransformMutex(); 
-        endTime();          // must come afetr unlockTransformMutex
+        endTime();          // must come after unlockTransformMutex
 #ifdef TRANSFORM_STATS
         info.tally(stats);
 #endif

ecl/hql/hqlutil.cpp

@@ -4714,6 +4714,44 @@ IHqlExpression * removeVirtualFields(IHqlExpression * record)
     return record->clone(args);
 }
 
+static HqlTransformerInfo fieldPropertyRemoverInfo("FieldPropertyRemover");
+class FieldPropertyRemover : public NewHqlTransformer
+{
+public:
+    FieldPropertyRemover(_ATOM _name) : NewHqlTransformer(fieldPropertyRemoverInfo), name(_name) {}
+
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        switch (expr->getOperator())
+        {
+        //By default fields within the following are not transformed...
+        case no_record:
+        case no_ifblock:
+        case no_select: // Ensure fields used by ifblocks get transformed
+            return completeTransform(expr);
+
+        case no_field:
+            {
+                OwnedHqlExpr transformed = transformField(expr);
+                while (transformed->hasProperty(name))
+                    transformed.setown(removeProperty(transformed, name));
+                return transformed.getClear();
+            }
+
+        default:
+            return NewHqlTransformer::createTransformed(expr);
+        }
+    }
+
+private:
+    _ATOM name;
+};
+
+IHqlExpression * removePropertyFromFields(IHqlExpression * expr, _ATOM name)
+{
+    FieldPropertyRemover remover(name);
+    return remover.transformRoot(expr);
+}
 
 #if 0
 void VirtualReplacer::createProjectAssignments(HqlExprArray & assigns, IHqlExpression * expr, IHqlExpression * tgtSelector, IHqlExpression * srcSelector, IHqlExpression * dataset)

ecl/hql/hqlutil.hpp

@@ -67,6 +67,7 @@ extern HQL_API void gatherIndexBuildSortOrder(HqlExprArray & sorts, IHqlExpressi
 extern HQL_API bool recordContainsBlobs(IHqlExpression * record);
 inline bool recordIsEmpty(IHqlExpression * record) { return queryLastField(record) == NULL; }
 extern HQL_API IHqlExpression * queryVirtualFileposField(IHqlExpression * record);
+extern HQL_API IHqlExpression * removePropertyFromFields(IHqlExpression * expr, _ATOM name);
 
 extern HQL_API IHqlExpression * flattenListOwn(IHqlExpression * list);
 extern HQL_API void flattenListOwn(HqlExprArray & out, IHqlExpression * list);

ecl/hqlcpp/hqlcpp.cpp

@@ -1704,6 +1704,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.showSeqInGraph,"showSeqInGraph",false),  // For tracking down why projects are not commoned up
         DebugOption(options.normalizeSelectorSequence,"normalizeSelectorSequence",false),  // For tracking down why projects are not commoned up
         DebugOption(options.transformCaseToChoose,"transformCaseToChoose",true),
+        DebugOption(options.removeXpathFromOutput,"removeXpathFromOutput",false),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -716,6 +716,7 @@ struct HqlCppOptions
     bool                showSeqInGraph;
     bool                normalizeSelectorSequence;
     bool                transformCaseToChoose;
+    bool                removeXpathFromOutput;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlcpputil.cpp

@@ -445,3 +445,11 @@ IHqlExpression * mapInternalFunctionParameters(IHqlExpression * expr)
 
     return LINK(expr);
 }
+
+
+bool mustInitializeField(IHqlExpression * field)
+{
+    if (hasLinkCountedModifier(field))
+        return true;
+    return false;
+}

ecl/hqlcpp/hqlcpputil.hpp

@@ -51,6 +51,8 @@ extern bool canCreateTemporary(IHqlExpression * expr);
 extern IHqlExpression * projectCreateSetDataset(IHqlExpression * createsetExpr);
 extern IHqlExpression * mapInternalFunctionParameters(IHqlExpression * expr);
 
+extern bool mustInitializeField(IHqlExpression * field);
+
 //Common types and expressions...
 extern ITypeInfo * boolType;
 extern ITypeInfo * sizetType;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10736,7 +10736,11 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
             buildReturn(namectx, name, constUnknownVarStringType);
         }
 
-        Owned<IWUResult> result = createDatasetResultSchema(seq, name, record, true, false);
+        LinkedHqlExpr cleanedRecord = record;
+        if (options.removeXpathFromOutput)
+            cleanedRecord.setown(removePropertyFromFields(cleanedRecord, xpathAtom));
+
+        Owned<IWUResult> result = createDatasetResultSchema(seq, name, cleanedRecord, true, false);
         if (result)
         {
             result->setResultRowLimit(-1);
@@ -11005,7 +11009,6 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
     }
 }
 
-
 void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear)
 {
     ForEachChild(i, record)
@@ -11029,7 +11032,7 @@ void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression *
                 {
                     generateSerializeAssigns(ctx, cur->queryRecord(), selected, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
                 }
-                else if (needToClear)
+                else if (needToClear || mustInitializeField(cur))
                 {
                     //MORE: Might want to recurse if a record
                     Owned<IHqlExpression> self = tgtDataset.mapScalar(selected, selfSelect);

ecl/hqlcpp/hqliproj.cpp

@@ -142,7 +142,13 @@ void UsedFieldSet::appendField(IHqlExpression & ownedField)
 
 void UsedFieldSet::clone(const UsedFieldSet & source)
 {
-    assertex(originalFields == source.originalFields);
+    if (originalFields != source.originalFields)
+    {
+        //In very rare circumstances it is possible to have non-identical records with identical structures
+        //A typical case is the same structure defined in two different places, using a symbol to specify a maximum
+        //length.  The locations of the symbols differ, so the records do not match exactly.
+        assertex(recordTypesMatch(queryOriginalRecord(), source.queryOriginalRecord()));
+    }
     ForEachItemIn(i, source.fields)
         appendField(OLINK(source.fields.item(i)));