Merge remote-tracking branch 'origin/candidate-3.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/ft/daftformat.cpp

@@ -1245,6 +1245,9 @@ void CRemotePartitioner::callRemote()
             unsigned compatflags = 0;                   // compatibility flags (not yet used)
             msg.append(compatflags);
             msg.append(decryptKey);
+            //Add extra data at the end to provide backward compatibility
+            srcFormat.serializeExtra(msg, 1);
+            tgtFormat.serializeExtra(msg, 1);
 
             if (!catchWriteBuffer(socket, msg))
                 throwError1(RFSERR_TimeoutWaitConnect, url.str());

dali/ft/filecopy.cpp

@@ -360,6 +360,9 @@ bool FileTransferThread::performTransfer()
         msg.append(sprayer.encryptKey);
         msg.append(sprayer.decryptKey);
 
+        sprayer.srcFormat.serializeExtra(msg, 1);
+        sprayer.tgtFormat.serializeExtra(msg, 1);
+
         if (!catchWriteBuffer(socket, msg))
             throwError1(RFSERR_TimeoutWaitConnect, url.str());
 

dali/ft/filecopy.hpp

@@ -46,12 +46,14 @@ public:
     FileFormat(FileFormatType _type = FFTunknown, unsigned _recordSize = 0) { set(_type, _recordSize); maxRecordSize = 0;}
 
     void deserialize(MemoryBuffer & in);
+    void deserializeExtra(MemoryBuffer & in, unsigned version);
     bool equals(const FileFormat & other) const     { return (type == other.type) && (recordSize == other.recordSize); }
     unsigned getUnitSize() const;
     bool isUtf() const                              { return (type >= FFTutf) && (type <= FFTutf32le); }
     bool restore(IPropertyTree * props);
     void save(IPropertyTree * props);
     void serialize(MemoryBuffer & out) const;
+    void serializeExtra(MemoryBuffer & out, unsigned version) const;
     void set(FileFormatType _type, unsigned _recordSize = 0) { type = _type, recordSize = _recordSize; }
     void set(const FileFormat & src);
 

dali/ft/ftbase.cpp

@@ -227,13 +227,28 @@ void FileFormat::deserialize(MemoryBuffer & in)
         ::deserialize(in, separate);
         ::deserialize(in, quote);
         ::deserialize(in, terminate);
-        ::deserialize(in, escape);
         ::deserialize(in, rowTag);
         break;
     }
 }
 
 
+void FileFormat::deserializeExtra(MemoryBuffer & in, unsigned version)
+{
+    switch (type)
+    {
+    case FFTcsv:
+    case FFTutf:
+    case FFTutf8: case FFTutf8n:
+    case FFTutf16: case FFTutf16be: case FFTutf16le:
+    case FFTutf32: case FFTutf32be: case FFTutf32le:
+        if (version == 1)
+            ::deserialize(in, escape);
+        break;
+    }
+}
+
+
 unsigned FileFormat::getUnitSize() const
 {
     switch (type)
@@ -386,12 +401,26 @@ void FileFormat::serialize(MemoryBuffer & out) const
         ::serialize(out, separate);
         ::serialize(out, quote);
         ::serialize(out, terminate);
-        ::serialize(out, escape);
         ::serialize(out, rowTag);
         break;
     }
 }
 
+void FileFormat::serializeExtra(MemoryBuffer & out, unsigned version) const
+{
+    switch (type)
+    {
+    case FFTcsv:
+    case FFTutf:
+    case FFTutf8: case FFTutf8n:
+    case FFTutf16: case FFTutf16be: case FFTutf16le:
+    case FFTutf32: case FFTutf32be: case FFTutf32le:
+        if (version == 1)
+            ::serialize(out, escape);
+        break;
+    }
+}
+
 void FileFormat::set(const FileFormat & src) 
 { 
     type = src.type; 

dali/ft/ftslave.cpp

@@ -83,6 +83,12 @@ bool processPartitionCommand(ISocket * masterSocket, MemoryBuffer & msg, MemoryB
     StringAttr decryptkey;
     if (msg.remaining())
         msg.read(decryptkey);
+    if (msg.remaining())
+    {
+        srcFormat.deserializeExtra(msg, 1);
+        tgtFormat.deserializeExtra(msg, 1);
+    }
+
     StringBuffer text;
     fullPath.getRemotePath(text);
     LOG(MCdebugProgress, unknownJob, "Process partition %d(%s)", whichInput, text.str());

dali/ft/fttransform.cpp

@@ -631,7 +631,11 @@ void TransferServer::deserializeAction(MemoryBuffer & msg, unsigned action)
         msg.read(transferBufferSize);
     if (msg.remaining()) 
         msg.read(encryptKey).read(decryptKey);
-
+    if (msg.remaining())
+    {
+        srcFormat.deserializeExtra(msg, 1);
+        tgtFormat.deserializeExtra(msg, 1);
+    }
 
     LOG(MCdebugProgress, unknownJob, "throttle(%d), transferBufferSize(%d)", throttleNicSpeed, transferBufferSize);
     PROGLOG("compressedInput(%d), compressedOutput(%d), copyCompressed(%d)", compressedInput?1:0, compressOutput?1:0, copyCompressed?1:0);

ecl/hql/hqltrans.ipp

@@ -150,7 +150,7 @@ public:
 #endif
         noteMemory();
         unlockTransformMutex(); 
-        endTime();          // must come afetr unlockTransformMutex
+        endTime();          // must come after unlockTransformMutex
 #ifdef TRANSFORM_STATS
         info.tally(stats);
 #endif

ecl/hql/hqlutil.cpp

@@ -4726,6 +4726,44 @@ IHqlExpression * removeVirtualFields(IHqlExpression * record)
     return record->clone(args);
 }
 
+static HqlTransformerInfo fieldPropertyRemoverInfo("FieldPropertyRemover");
+class FieldPropertyRemover : public NewHqlTransformer
+{
+public:
+    FieldPropertyRemover(_ATOM _name) : NewHqlTransformer(fieldPropertyRemoverInfo), name(_name) {}
+
+    virtual IHqlExpression * createTransformed(IHqlExpression * expr)
+    {
+        switch (expr->getOperator())
+        {
+        //By default fields within the following are not transformed...
+        case no_record:
+        case no_ifblock:
+        case no_select: // Ensure fields used by ifblocks get transformed
+            return completeTransform(expr);
+
+        case no_field:
+            {
+                OwnedHqlExpr transformed = transformField(expr);
+                while (transformed->hasProperty(name))
+                    transformed.setown(removeProperty(transformed, name));
+                return transformed.getClear();
+            }
+
+        default:
+            return NewHqlTransformer::createTransformed(expr);
+        }
+    }
+
+private:
+    _ATOM name;
+};
+
+IHqlExpression * removePropertyFromFields(IHqlExpression * expr, _ATOM name)
+{
+    FieldPropertyRemover remover(name);
+    return remover.transformRoot(expr);
+}
 
 #if 0
 void VirtualReplacer::createProjectAssignments(HqlExprArray & assigns, IHqlExpression * expr, IHqlExpression * tgtSelector, IHqlExpression * srcSelector, IHqlExpression * dataset)

ecl/hql/hqlutil.hpp

@@ -67,6 +67,7 @@ extern HQL_API void gatherIndexBuildSortOrder(HqlExprArray & sorts, IHqlExpressi
 extern HQL_API bool recordContainsBlobs(IHqlExpression * record);
 inline bool recordIsEmpty(IHqlExpression * record) { return queryLastField(record) == NULL; }
 extern HQL_API IHqlExpression * queryVirtualFileposField(IHqlExpression * record);
+extern HQL_API IHqlExpression * removePropertyFromFields(IHqlExpression * expr, _ATOM name);
 
 extern HQL_API IHqlExpression * flattenListOwn(IHqlExpression * list);
 extern HQL_API void flattenListOwn(HqlExprArray & out, IHqlExpression * list);

ecl/hqlcpp/hqlcpp.cpp

@@ -1710,6 +1710,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.showSeqInGraph,"showSeqInGraph",false),  // For tracking down why projects are not commoned up
         DebugOption(options.normalizeSelectorSequence,"normalizeSelectorSequence",false),  // For tracking down why projects are not commoned up
         DebugOption(options.transformCaseToChoose,"transformCaseToChoose",true),
+        DebugOption(options.removeXpathFromOutput,"removeXpathFromOutput",false),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -718,6 +718,7 @@ struct HqlCppOptions
     bool                showSeqInGraph;
     bool                normalizeSelectorSequence;
     bool                transformCaseToChoose;
+    bool                removeXpathFromOutput;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlcpputil.cpp

@@ -445,3 +445,11 @@ IHqlExpression * mapInternalFunctionParameters(IHqlExpression * expr)
 
     return LINK(expr);
 }
+
+
+bool mustInitializeField(IHqlExpression * field)
+{
+    if (hasLinkCountedModifier(field))
+        return true;
+    return false;
+}

ecl/hqlcpp/hqlcpputil.hpp

@@ -51,6 +51,8 @@ extern bool canCreateTemporary(IHqlExpression * expr);
 extern IHqlExpression * projectCreateSetDataset(IHqlExpression * createsetExpr);
 extern IHqlExpression * mapInternalFunctionParameters(IHqlExpression * expr);
 
+extern bool mustInitializeField(IHqlExpression * field);
+
 //Common types and expressions...
 extern ITypeInfo * boolType;
 extern ITypeInfo * sizetType;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -10771,7 +10771,11 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputWorkunit(BuildCtx & ctx,
             buildReturn(namectx, name, constUnknownVarStringType);
         }
 
-        Owned<IWUResult> result = createDatasetResultSchema(seq, name, record, true, false);
+        LinkedHqlExpr cleanedRecord = record;
+        if (options.removeXpathFromOutput)
+            cleanedRecord.setown(removePropertyFromFields(cleanedRecord, xpathAtom));
+
+        Owned<IWUResult> result = createDatasetResultSchema(seq, name, cleanedRecord, true, false);
         if (result)
         {
             result->setResultRowLimit(-1);
@@ -11093,7 +11097,6 @@ void HqlCppTranslator::generateSortCompare(BuildCtx & nestedctx, BuildCtx & ctx,
     }
 }
 
-
 void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression * record, IHqlExpression * selector, IHqlExpression * selfSelect, IHqlExpression * leftSelect, const DatasetReference & srcDataset, const DatasetReference & tgtDataset, HqlExprArray & srcSelects, HqlExprArray & tgtSelects, bool needToClear)
 {
     ForEachChild(i, record)
@@ -11117,7 +11120,7 @@ void HqlCppTranslator::generateSerializeAssigns(BuildCtx & ctx, IHqlExpression *
                 {
                     generateSerializeAssigns(ctx, cur->queryRecord(), selected, selfSelect, leftSelect, srcDataset, tgtDataset, srcSelects, tgtSelects, needToClear);
                 }
-                else if (needToClear)
+                else if (needToClear || mustInitializeField(cur))
                 {
                     //MORE: Might want to recurse if a record
                     Owned<IHqlExpression> self = tgtDataset.mapScalar(selected, selfSelect);

ecl/hqlcpp/hqliproj.cpp

@@ -142,7 +142,13 @@ void UsedFieldSet::appendField(IHqlExpression & ownedField)
 
 void UsedFieldSet::clone(const UsedFieldSet & source)
 {
-    assertex(originalFields == source.originalFields);
+    if (originalFields != source.originalFields)
+    {
+        //In very rare circumstances it is possible to have non-identical records with identical structures
+        //A typical case is the same structure defined in two different places, using a symbol to specify a maximum
+        //length.  The locations of the symbols differ, so the records do not match exactly.
+        assertex(recordTypesMatch(queryOriginalRecord(), source.queryOriginalRecord()));
+    }
     ForEachItemIn(i, source.fields)
         appendField(OLINK(source.fields.item(i)));
 

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -464,46 +464,49 @@ void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
 {
     try
     {
+        IArrayOf<IEspECLHelpFile> helpers;
+
         Owned <IConstWUQuery> query = cw->getQuery();
         if(!query)
         {
             ERRLOG("Cannot get Query for this workunit.");
             info.setHelpersDesc("Cannot get Query for this workunit.");
         }
-
-        SCMStringBuffer qname;
-        query->getQueryShortText(qname);
-        if(qname.length())
+        else
         {
-            if((flags & WUINFO_TruncateEclTo64k) && (qname.length() > 64000))
-                qname.setLen(qname.str(), 64000);
+            SCMStringBuffer qname;
+            query->getQueryShortText(qname);
+            if(qname.length())
+            {
+                if((flags & WUINFO_TruncateEclTo64k) && (qname.length() > 64000))
+                    qname.setLen(qname.str(), 64000);
 
-            IEspECLQuery* q=&info.updateQuery();
-            q->setText(qname.str());
-        }
+                IEspECLQuery* q=&info.updateQuery();
+                q->setText(qname.str());
+            }
 
-        if (version > 1.34)
-        {
-            SCMStringBuffer mainDefinition;
-            query->getQueryMainDefinition(mainDefinition);
-            if(mainDefinition.length())
+            if (version > 1.34)
             {
-                IEspECLQuery* q=&info.updateQuery();
-                q->setQueryMainDefinition(mainDefinition.str());
+                SCMStringBuffer mainDefinition;
+                query->getQueryMainDefinition(mainDefinition);
+                if(mainDefinition.length())
+                {
+                    IEspECLQuery* q=&info.updateQuery();
+                    q->setQueryMainDefinition(mainDefinition.str());
+                }
             }
-        }
 
-        if (version > 1.30)
-        {
-            SCMStringBuffer qText;
-            query->getQueryText(qText);
-            if ((qText.length() > 0) && isArchiveQuery(qText.str()))
-                info.setHasArchiveQuery(true);
-        }
+            if (version > 1.30)
+            {
+                SCMStringBuffer qText;
+                query->getQueryText(qText);
+                if ((qText.length() > 0) && isArchiveQuery(qText.str()))
+                    info.setHasArchiveQuery(true);
+            }
 
-        IArrayOf<IEspECLHelpFile> helpers;
-        for (unsigned i = 0; i < FileTypeSize; i++)
-            getHelpFiles(query, (WUFileType) i, helpers);
+            for (unsigned i = 0; i < FileTypeSize; i++)
+                getHelpFiles(query, (WUFileType) i, helpers);
+        }
 
         getWorkunitThorLogInfo(helpers, info);
 

githooks/commit-msg

@@ -38,7 +38,7 @@ Use git commit -s -t $1 to reopen previous message for editing\n"
   exit 1
 }
 
-[ $(cat "$1" | wc -L ) -le 80 ] || {
+[ $(grep -v '^#' "$1" | wc -L ) -le 80 ] || {
   echo >&2 "\n ERROR: Commit message lines too long (max 80 chars)."
   echo >&2 $usage
   exit 1

roxie/ccd/ccdquery.cpp

@@ -1237,6 +1237,7 @@ public:
         catch (...)
         {
             ::Release(activities);
+            allActivities.kill();
             throw;
         }
         return activities;

roxie/ccd/ccdserver.cpp

@@ -13388,12 +13388,12 @@ public:
         ActivityTimer t(totalCycles, timeActivities, ctx->queryDebugContext());
         loop
         {
-            const void * in = input->nextInGroup();
+            OwnedConstRoxieRow in = input->nextInGroup();
             if (!in)
             {
                 recordCount = 0;
                 if (numProcessedLastGroup == processed)
-                    in = input->nextInGroup();
+                    in.setown(input->nextInGroup());
                 if (!in)
                 {
                     numProcessedLastGroup = processed;
@@ -13409,7 +13409,6 @@ public:
                     outSize = ((IHThorCountProjectArg &) basehelper).transform(rowBuilder, in, ++recordCount);
                 else
                     outSize = ((IHThorProjectArg &) basehelper).transform(rowBuilder, in);
-                ReleaseRoxieRow(in);
                 if (outSize)
                 {
                     processed++;

testing/ecl/roxie/soapcall.ecl

@@ -52,7 +52,7 @@ END;
 
 // Test some failure cases
 
-output(SORT(SOAPCALL(d, 'http://127.0.0.1:9876|http://127.0.0.1:9875','soapbase', { unkname }, DATASET(ServiceOutRecord), onFail(doError(LEFT)),RETRY(0), log('SOAP: ' + unkname)), record));
+output(SORT(SOAPCALL(d, 'http://127.0.0.1:9876|http://127.0.0.1:9875','soapbase', { unkname }, DATASET(ServiceOutRecord), onFail(doError(LEFT)),RETRY(0), log('SOAP: ' + unkname),TIMEOUT(1)), record));
 output(SORT(SOAPCALL('http://127.0.0.1:9876','soapbase', { string unkname := 'FAIL' }, dataset(ServiceOutRecord),onFail(doError2),RETRY(0), LOG(MIN)),record));
 output(SORT(SOAPCALL(d, 'http://127.0.0.1:9876','soapbaseNOSUCHQUERY', { unkname }, DATASET(ServiceOutRecord), onFail(doError(LEFT)),MERGE(25),RETRY(0), LOG(MIN)), record));