Merge branch 'candidate-5.6.0' into candidate-6.0.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -189,6 +189,7 @@ if(TOP_LEVEL_PROJECT)
     set(CPACK_RPM_PACKAGE_RELEASE "${stagever}")
     set(CPACK_RPM_PACKAGE_VENDOR "HPCC Systems®")
     set(CPACK_PACKAGE_VENDOR "HPCC Systems®")
+    set(CPACK_PACKAGE_VENDOR_WITHOUT_TRADEMARK "HPCC Systems")
 
     if(${ARCH64BIT} EQUAL 1)
         set(CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
@@ -388,7 +389,7 @@ if(TOP_LEVEL_PROJECT)
             set(CPACK_NSIS_URL_INFO_ABOUT "http:\\\\\\\\hpccsystems.com")
             set(CPACK_NSIS_CONTACT ${CPACK_PACKAGE_CONTACT})
             set(CPACK_NSIS_DEFINES "
-                !define MUI_STARTMENUPAGE_DEFAULTFOLDER \\\"${CPACK_PACKAGE_VENDOR}\\\\${version}\\\\${CPACK_NSIS_DISPLAY_NAME}\\\"
+                !define MUI_STARTMENUPAGE_DEFAULTFOLDER \\\"${CPACK_PACKAGE_VENDOR_WITHOUT_TRADEMARK}\\\\${version}\\\\${CPACK_NSIS_DISPLAY_NAME}\\\"
                 !define MUI_FINISHPAGE_NOAUTOCLOSE
             ")
 

dali/base/dadfs.cpp

@@ -3074,7 +3074,23 @@ public:
     {
         return !checkProtectAttr(logicalName.get(),root,reason);
     }
-
+    bool checkOwned(StringBuffer &error)
+    {
+        Owned<IPropertyTreeIterator> iter = root->getElements("SuperOwner");
+        if (iter->first())
+        {
+            error.append("Cannot remove file ").append(logicalName.get()).append(" as owned by SuperFile(s): ");
+            loop
+            {
+                error.append(iter->query().queryProp("@name"));
+                if (!iter->next())
+                    break;
+                error.append(", ");
+            }
+            return true;
+        }
+        return false;
+    }
     bool canRemove(StringBuffer &reason,bool ignoresub=false)
     {
         CriticalBlock block(sect);
@@ -3085,31 +3101,25 @@ public:
             reason.appendf("empty filename");
             return false;
         }
-        if (logicalName.isQuery()) {
+        if (logicalName.isQuery())
+        {
             reason.appendf("%s is query",logicalname);
             return false;
         }
-        if (logicalName.isForeign()) {
+        if (logicalName.isForeign())
+        {
             reason.appendf("%s is foreign",logicalname);
             return false;
         }
-        if (logicalName.isMulti()) {
+        if (logicalName.isMulti())
+        {
             reason.appendf("%s is multi",logicalname);
             return false;
         }
-        if (!ignoresub) {
-            // And has super owners
-            Owned<IPropertyTreeIterator> iter = root->getElements("SuperOwner");
-            if (iter->first()) {
-                reason.append("Cannot remove file ").append(logicalname).append(" as owned by SuperFile(s): ");
-                loop {
-                    reason.append(iter->query().queryProp("@name"));
-                    if (!iter->next())
-                        break;
-                    reason.append(", ");
-                }
+        if (!ignoresub)
+        {
+            if (checkOwned(reason))
                 return false;
-            }
         }
         return true;
     }
@@ -5185,6 +5195,10 @@ public:
 
     void checkFormatAttr(IDistributedFile *sub, const char* exprefix="")
     {
+        IDistributedSuperFile *superSub = sub->querySuperFile();
+        if (superSub && (0 == superSub->numSubFiles(true)))
+            return;
+
         // only check sub files not siblings, which is excessive (format checking is really only debug aid)
         checkSubFormatAttr(sub,exprefix);
     }
@@ -5520,7 +5534,10 @@ public:
         assertex(conn.get()); // must be attached
         CriticalBlock block(sect);
         checkModify("CDistributedSuperFile::detach");
-        subfiles.kill();    
+        StringBuffer reason;
+        if (checkOwned(reason))
+            throw MakeStringException(-1, "detach: %s", reason.str());
+        subfiles.kill();
 
         // Remove from SDS
 

esp/src/eclwatch/ESPLogicalFile.js

@@ -66,7 +66,7 @@ define([
                 case "RecordCount":
                     request.Sortby = "Records";
                     break;
-                case "Totalsize":
+                case "IntSize":
                     request.Sortby = "FileSize";
                     break;
             }
@@ -420,4 +420,4 @@ define([
             return Observable(store);
         }
     };
-});
+});

esp/src/eclwatch/FileBelongsToWidget.js

@@ -18,12 +18,17 @@ define([
     "dojo/i18n",
     "dojo/i18n!./nls/hpcc",
 
+    "dgrid/selector",
+
+    "hpcc/DelayLoadWidget",
     "hpcc/GridDetailsWidget",
     "hpcc/ESPLogicalFile",
-    "hpcc/ESPUtil"
+    "hpcc/ESPUtil",
+    "hpcc/SFDetailsWidget",
 
 ], function (declare, i18n, nlsHPCC,
-                GridDetailsWidget, ESPLogicalFile, ESPUtil) {
+                selector,
+                DelayLoadWidget, GridDetailsWidget, ESPLogicalFile, ESPUtil, SFDetailsWidget) {
     return declare("FileBelongsToWidget", [GridDetailsWidget], {
         i18n: nlsHPCC,
         logicalFile: null,
@@ -42,12 +47,31 @@ define([
             var retVal = new declare([ESPUtil.Grid(false, true)])({
                 store: this.store,
                 columns: {
-                    Name: { label: this.i18n.Name }
+                    sel: selector({
+                        width: 27,
+                        selectorType: 'checkbox'
+                    }),
+                   Name: { label: this.i18n.Name }
                 }
             }, domID);
             return retVal;
         },
 
+        createDetail: function (id, row, params) {
+            return new DelayLoadWidget({
+                id: id,
+                title: row.Name,
+                closable: true,
+                delayWidget: "SFDetailsWidget",
+                hpcc: {
+                    type: "SFDetailsWidget",
+                    params: {
+                        Name: row.Name
+                    }
+                }
+            });
+         },
+
         refreshGrid: function (args) {
             var context = this;
             if (this.logicalFile.Superfiles.DFULogicalFile) {

esp/src/eclwatch/LFDetailsWidget.js

@@ -331,6 +331,7 @@ define([
                 domClass.add(this.id + "StateIdImage", this.logicalFile.getStateIconClass());
                 domAttr.set(this.id + "Name", "innerHTML", this.logicalFile.Name + (this.logicalFile.isDeleted() ? " (" + this.i18n.Deleted + ")" : "" ));
             } else if (name === "Superfiles") {
+                this.fileBelongsToWidget.set("title", this.i18n.Superfile + " (" + newValue.DFULogicalFile.length + ")");
             } else if (name === "__hpcc_changedCount" && newValue > 0) {
                 this.refreshActionState();
                 //  Force Icon to Show (I suspect its not working due to Circular Reference Loading)

roxie/ccd/ccdactivities.cpp

@@ -4607,7 +4607,7 @@ public:
         m.setBuffer(indexLayoutSize, indexLayoutMeta.getdata());
         activityMeta.setown(deserializeRecordMeta(m, true));
         layoutTranslators.setown(new TranslatorArray);
-        bool variableFileName = allFilesDynamic || queryFactory.isDynamic() || ((helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0);
+        bool variableFileName = allFilesDynamic || queryFactory.isDynamic() || ((helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename|JFindexfromactivity)) != 0);
         if (!variableFileName)
         {
             bool isOpt = (helper->getJoinFlags() & JFindexoptional) != 0;
@@ -4648,7 +4648,7 @@ public:
         : factory(_aFactory), CRoxieKeyedActivity(_logctx, _packet, _hFactory, _aFactory)
     {
         helper = (IHThorKeyedJoinArg *) basehelper;
-        variableFileName = allFilesDynamic || basefactory->queryQueryFactory().isDynamic() || ((helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename)) != 0);
+        variableFileName = allFilesDynamic || basefactory->queryQueryFactory().isDynamic() || ((helper->getJoinFlags() & (JFvarindexfilename|JFdynamicindexfilename|JFindexfromactivity)) != 0);
         inputDone = 0;
         processed = 0;
         candidateCount = 0;

roxie/ccd/ccdserver.cpp

@@ -24148,7 +24148,7 @@ public:
         if (keepLimit == 0) keepLimit = (unsigned)-1;
         getLimitType(joinFlags, limitFail, limitOnFail);
         cloneLeft = (joinFlags & JFtransformmatchesleft) != 0;
-        if (joinFlags & JFleftouter)
+        if (joinFlags & JFleftouter || limitOnFail)
             createDefaultRight();
     }
 

system/jlib/jsocket.cpp

@@ -4965,6 +4965,12 @@ public:
         }
     }
 
+    void set_keep_alive(bool keepalive)
+    {
+        if (sock)
+            sock->set_keep_alive(keepalive);
+    }
+
     bool connect(unsigned timeoutms)
     {
         CriticalBlock block(crit);

system/jlib/jsocket.hpp

@@ -536,6 +536,7 @@ extern jlib_decl bool catchWriteBuffer(ISocket * socket, MemoryBuffer & buffer);
 interface IConversation: extends IInterface
 {
     virtual bool accept(unsigned timeoutms)=0;                      // one side accepts
+    virtual void set_keep_alive(bool keepalive)=0;                  // enable keepalive for socket
     virtual bool connect(unsigned timeoutms)=0;                     // other side connects
     virtual bool send(MemoryBuffer &mb)=0;                          // 0 length buffer can be sent
     virtual bool recv(MemoryBuffer &mb, unsigned timeoutms)=0;      // up to protocol to terminate conversation (e.g. by zero length buffer)

testing/regress/ecl/key/rollupgroup.xml

@@ -1,5 +1,4 @@
 <Dataset name='Result 1'>
- <Row><did>1</did><names><Row><name>Gavin               </name></Row><Row><name>Gavin Halliday      </name></Row><Row><name>Halliday            </name></Row></names><secondname>Gavin Halliday      </secondname></Row>
  <Row><did>2</did><names><Row><name>Richard             </name></Row><Row><name>Richard Chapman     </name></Row><Row><name>Richard             </name></Row></names><secondname>Richard Chapman     </secondname></Row>
  <Row><did>3</did><names><Row><name>Jake Smith          </name></Row></names><secondname>                    </secondname></Row>
  <Row><did>4</did><names></names><secondname>                    </secondname></Row>

testing/regress/ecl/rollupgroup.ecl

@@ -63,7 +63,7 @@ j3 := join(dsg, i3, left.did = right.did, transform(outrec, self := left; self :
 //perform 3 soap calls in parallel
 combined := regroup(j1, j2, j3);
 
-finalRec doRollup(outRec l, dataset(outRec) allRows) := transform
+finalRec doRollup(outRec l, dataset(outRec) allRows) := transform,skip(l.did=1)
     self.did  := l.did;
     self.names := project(allRows(score != 0), transform(nameRec, self := left));
     self.secondName := allRows(score != 0)[2].name;

thorlcr/activities/rollup/throllupslave.cpp

@@ -598,25 +598,29 @@ public:
     CATCH_NEXTROW()
     {
         ActivityTimer t(totalCycles, timeActivities);
-        if (!eoi)
+        if (eoi)
+            return NULL;
+
+        loop
         {
             CThorExpandingRowArray rows(*this, queryRowInterfaces(input));
             Owned<IRowStream> rowStream = groupLoader->loadGroup(input, abortSoon, &rows);
             unsigned count = rows.ordinality();
-            if (count)
+            if (0 == count)
             {
-                RtlDynamicRowBuilder row(queryRowAllocator());
-                size32_t sz = helper->transform(row, count, rows.getRowArray());
-                rows.kill();
-                if (sz)
-                {
-                    dataLinkIncrement();
-                    return row.finalizeRowClear(sz);
-                }
+                eoi = true;
+                return NULL;
+            }
+
+            RtlDynamicRowBuilder row(queryRowAllocator());
+            size32_t sz = helper->transform(row, count, rows.getRowArray());
+            rows.kill();
+            if (sz)
+            {
+                dataLinkIncrement();
+                return row.finalizeRowClear(sz);
             }
         }
-        eoi = true;
-        return NULL;
     }
     virtual bool isGrouped() { return false; }
     virtual void getMetaInfo(ThorDataLinkMetaInfo &info)

thorlcr/activities/soapcall/thsoapcallslave.cpp

@@ -36,6 +36,7 @@ class CWscRowCallSlaveActivity : public CSlaveActivity, public CThorDataLink, im
 {
     bool eof;
     Owned<IWSCHelper> wscHelper;
+    StringBuffer authToken;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
@@ -45,37 +46,38 @@ public:
     // IThorSlaveActivity overloaded methods
     virtual void init(MemoryBuffer &data, MemoryBuffer &slaveData)
     {
-        StringBuffer authToken;
         buildAuthToken(queryJob().queryUserDescriptor(), authToken);
         appendOutputLinked(this);
+    }
+    // IThorDataLink methods
+    virtual void start()
+    {
+        ActivityTimer s(totalCycles, timeActivities);
+        eof = false;
         if (container.queryLocalOrGrouped() || firstNode())
         {
             switch (container.getKind())
             {
                 case TAKsoap_rowdataset:
-                    wscHelper.setown(createSoapCallHelper(this, queryRowAllocator(), authToken.str(), SCrow, NULL, queryDummyContextLogger(),NULL));
+                    wscHelper.setown(createSoapCallHelper(this, queryRowAllocator(), authToken.str(), SCrow, NULL, queryDummyContextLogger(), NULL));
                     break;
                 case TAKhttp_rowdataset:
-                    wscHelper.setown(createHttpCallHelper(this, queryRowAllocator(), authToken.str(), SCrow, NULL, queryDummyContextLogger(),NULL));
+                    wscHelper.setown(createHttpCallHelper(this, queryRowAllocator(), authToken.str(), SCrow, NULL, queryDummyContextLogger(), NULL));
                     break;
                 default:
                     throwUnexpected();
             }
+            wscHelper->start();
         }
-    }
-    // IThorDataLink methods
-    virtual void start()
-    {
-        ActivityTimer s(totalCycles, timeActivities);
-        eof = false;
         dataLinkStart();
-        if (wscHelper)
-            wscHelper->start();
     }
     virtual void stop()
     {
         if (wscHelper)
+        {
             wscHelper->waitUntilDone();
+            wscHelper.clear();
+        }
         dataLinkStop();
     }
     CATCH_NEXTROW()

thorlcr/graph/thgraphmaster.cpp

@@ -222,6 +222,7 @@ void CSlaveMessageHandler::main()
                         DBGLOG("%s", msg.str());
                         parentExtract = graph->setParentCtx(parentExtractSz, parentExtract);
                     }
+                    Owned<IException> exception;
                     loop
                     {
                         activity_id id;
@@ -230,23 +231,41 @@ void CSlaveMessageHandler::main()
                             break;
                         CMasterGraphElement *element = (CMasterGraphElement *)graph->queryElement(id);
                         assertex(element);
-                        element->deserializeStartContext(msg);
-                        element->doCreateActivity(parentExtractSz, parentExtract);
+                        try
+                        {
+                            element->deserializeStartContext(msg);
+                            element->doCreateActivity(parentExtractSz, parentExtract);
+                        }
+                        catch (IException *e)
+                        {
+                            EXCLOG(e, NULL);
+                            exception.setown(e);
+                            break;
+                        }
                         CActivityBase *activity = element->queryActivity();
                         if (activity && activity->needReInit())
                             element->sentActInitData->set(slave, 0); // clear to permit serializeActivityInitData to resend
                         toSerialize.append(*LINK(element));
                     }
                     msg.clear();
-                    CMessageBuffer replyMsg;
                     mptag_t replyTag = job.queryJobChannel(0).queryMPServer().createReplyTag();
                     msg.append(replyTag); // second reply
-                    replyMsg.setReplyTag(replyTag);
-                    CGraphElementArrayIterator iter(toSerialize);
-                    graph->serializeActivityInitData(slave, msg, iter);
+                    if (exception)
+                    {
+                        msg.append(true);
+                        serializeException(exception, msg);
+                    }
+                    else
+                    {
+                        msg.append(false);
+                        CGraphElementArrayIterator iter(toSerialize);
+                        graph->serializeActivityInitData(slave, msg, iter);
+                    }
                     job.queryJobChannel(0).queryJobComm().reply(msg);
                     if (!job.queryJobChannel(0).queryJobComm().recv(msg, slave+1, replyTag, NULL, MEDIUMTIMEOUT))
                         throwUnexpected();
+                    if (exception)
+                        throw exception.getClear();
                     bool error;
                     msg.read(error);
                     if (error)

thorlcr/graph/thgraphslave.cpp

@@ -470,6 +470,17 @@ bool CSlaveGraph::recvActivityInitData(size32_t parentExtractSz, const byte *par
             if (!queryJobChannel().queryJobComm().sendRecv(msg, 0, queryJob().querySlaveMpTag(), LONGTIMEOUT))
                 throwUnexpected();
             replyTag = queryJobChannel().deserializeMPTag(msg);
+            bool error;
+            msg.read(error);
+            if (error)
+            {
+                Owned<IException> e = deserializeException(msg);
+                EXCLOG(e, "Master hit exception");
+                msg.clear();
+                if (!queryJobChannel().queryJobComm().send(msg, 0, replyTag, LONGTIMEOUT))
+                    throw MakeStringException(0, "Timeout sending init data back to master");
+                throw e.getClear();
+            }
             msg.read(len);
         }
         try

thorlcr/master/thgraphmanager.cpp

@@ -591,6 +591,7 @@ void CJobManager::run()
                                     Owned<IConversation> acceptconv = createSingletonSocketConnection(ep.port,&ep);
                                     if (acceptconv->connect(60*1000)) // shouldn't need that long
                                     {
+                                        acceptconv->set_keep_alive(true);
                                         item.setown(_item.getClear());
                                         conversation.setown(acceptconv.getClear());
                                     }