Merge branch 'candidate-5.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/remote/sockfile.cpp

@@ -855,11 +855,6 @@ class CRemoteBase: public CInterface
     bool                    useSSL;
     void connectSocket(SocketEndpoint &ep)
     {
-#ifdef _DEBUG
-        StringBuffer sb;
-        ep.getUrlStr(sb);
-        DBGLOG("Client connecting %sto dafilesrv %s", useSSL?"SECURE ":"", sb.str());
-#endif
         sRFTM tm;
         // called in CConnectionTable::crit
         unsigned retries = 3;

docs/CMakeLists.txt

@@ -47,7 +47,6 @@ add_subdirectory(common)
 add_subdirectory(BuildTools)
 add_subdirectory(resources)
 
-
 # Doc dirs to include.
 add_subdirectory(DynamicESDL)
 add_subdirectory(ECLLanguageReference)

docs/ECLWatch/ECLWa_mods/ECLWatchSrc.xml

@@ -694,8 +694,8 @@
             <title>Stats</title>
 
             <para>On the workunit Timers tab is another tab for Stats. The
-            Stats tab is a visual representation of all the workunit timings.
-            </para>
+            Stats tab is a visual representation of all the workunit
+            timings.</para>
 
             <figure>
               <title>Stats</title>
@@ -708,7 +708,7 @@
             </figure>
 
             <para>Select the timer values from the drop list on the Stats tab
-            to view the various charts and graphs. </para>
+            to view the various charts and graphs.</para>
           </sect4>
         </sect3>
 
@@ -1298,6 +1298,11 @@
             </listitem>
 
             <listitem>
+              <para><emphasis role="bold">Index</emphasis> - include only
+              Index files if checked.</para>
+            </listitem>
+
+            <listitem>
               <para><emphasis role="bold">Cluster</emphasis> - filter files by
               cluster. Select the cluster from the drop list.</para>
             </listitem>

docs/HPCCMonitoring/MonRep-Mods/MonRep-VM.xml

@@ -7,7 +7,7 @@
   <para>The complete details for installing and running HPCC in a virtual
   machine are available in the document: <emphasis role="bold">Running HPCC in
   a Virtual Machine</emphasis>, available from <ulink
-  url="hpccsystems.com/download/docs">hpccsystems.com/download/docs</ulink>
+  url="hpccsystems.com/download/docs">http://hpccsystems.com/download/docs</ulink>
   .</para>
 
   <para>The following steps are a quick summary, assuming you have some

ecl/eclcmd/eclcmd_common.hpp

@@ -91,6 +91,7 @@ typedef IEclCommand *(*EclCommandFactory)(const char *cmdname);
 #define ECLOPT_DELETE_FILES "--delete"
 #define ECLOPT_DELETE_SUBFILES "--delete-subfiles"
 #define ECLOPT_DELETE_RECURSIVE "--delete-recursive"
+#define ECLOPT_RETRY "--retry"
 
 #define ECLOPT_MAIN "--main"
 #define ECLOPT_MAIN_S "-main"  //eclcc compatible format

ecl/eclcmd/roxie/ecl-roxie.cpp

@@ -251,7 +251,7 @@ private:
 class EclCmdRoxieCheckOrReload : public EclCmdCommon
 {
 public:
-    EclCmdRoxieCheckOrReload(bool _reload) : optMsToWait(10000), reload(_reload)
+    EclCmdRoxieCheckOrReload(bool _reload) : optMsToWait(10000), reload(_reload), optRetry(false)
     {
     }
     virtual bool parseCommandLineOptions(ArgvIterator &iter)
@@ -272,6 +272,8 @@ public:
             }
             if (iter.matchOption(optMsToWait, ECLOPT_WAIT))
                 continue;
+            if (reload && iter.matchFlag(optRetry, ECLOPT_RETRY))
+                continue;
             if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
                 return false;
         }
@@ -295,7 +297,12 @@ public:
         Owned<IClientRoxieControlCmdRequest> req = client->createRoxieControlCmdRequest();
         req->setWait(optMsToWait);
         req->setProcessCluster(optProcess);
-        req->setCommand((reload) ? CRoxieControlCmd_RELOAD : CRoxieControlCmd_STATE);
+        if (!reload)
+            req->setCommand(CRoxieControlCmd_STATE);
+        else if (optRetry)
+            req->setCommand(CRoxieControlCmd_RELOAD_RETRY);
+        else
+            req->setCommand(CRoxieControlCmd_RELOAD);
 
         Owned<IClientRoxieControlCmdResponse> resp = client->RoxieControlCmd(req);
         if (resp->getExceptions().ordinality())
@@ -358,6 +365,7 @@ private:
     StringAttr optProcess;
     unsigned optMsToWait;
     bool reload;
+    bool optRetry;
 };
 
 class EclCmdRoxieUnusedFiles : public EclCmdCommon

esp/scm/ws_smc.ecm

@@ -313,7 +313,8 @@ ESPenum RoxieControlCmd : string
     ATTACH("Attach"),
     DETACH("Detach"),
     STATE("State"),
-    RELOAD("Reload")
+    RELOAD("Reload"),
+    RELOAD_RETRY("ReloadRetry")
 };
 
 ESPrequest RoxieControlCmdRequest

esp/services/ws_smc/ws_smcService.cpp

@@ -2103,6 +2103,8 @@ inline const char *controlCmdMessage(int cmd)
     case CRoxieControlCmd_DETACH:
         return "<control:lockDali/>";
     case CRoxieControlCmd_RELOAD:
+        return "<control:reload/>";
+    case CRoxieControlCmd_RELOAD_RETRY:
         return "<control:reload forceRetry='1' />";
     case CRoxieControlCmd_STATE:
         return "<control:state/>";

esp/src/eclwatch/ESPWorkunit.js

@@ -263,6 +263,9 @@ define([
         isComplete: function () {
             return this.hasCompleted;
         },
+        isFailed: function () {
+            return this.StateID === 4;
+        },
         isDeleted: function () {
             return this.StateID === 999;
         },
@@ -281,6 +284,14 @@ define([
                 });
             }
         },
+        doDeschedule: function () {
+            return this._action("Deschedule").then(function(response) {
+            });
+        },
+        doReschedule: function () {
+            return this._action("Reschedule").then(function(response) {
+            });
+        },
         create: function (ecl) {
             var context = this;
             WsWorkunits.WUCreate({

esp/src/eclwatch/WUDetailsWidget.js

@@ -179,6 +179,12 @@ define([
         _onRecover: function (event) {
             this.wu.recover();
         },
+        _onDeschedule: function (event) {
+            this.wu.doDeschedule();
+        },
+        _onReschedule: function (event) {
+            this.wu.doReschedule();
+        },
         _onPublish: function (event) {
             var allowForeign = registry.byId(this.id + "AllowForeignFiles");
             if (allowForeign.checked == true) {
@@ -496,6 +502,8 @@ define([
             registry.byId(this.id + "Recover").set("disabled", isArchived || !this.wu.isComplete() || this.wu.isDeleted());
             registry.byId(this.id + "Publish").set("disabled", isArchived || !this.wu.isComplete() || this.wu.isDeleted());
             registry.byId(this.id + "ZapReport").set("disabled", this.wu.isDeleted());
+            registry.byId(this.id + "Reschedule").set("disabled", !this.wu.isComplete() || this.wu.isFailed());
+            registry.byId(this.id + "Deschedule").set("disabled", this.wu.isComplete() || this.wu.isFailed());
 
             registry.byId(this.id + "Jobname").set("readOnly", !this.wu.isComplete() || this.wu.isDeleted());
             registry.byId(this.id + "Description").set("readOnly", !this.wu.isComplete() || this.wu.isDeleted());

esp/src/eclwatch/WUQueryWidget.js

@@ -155,9 +155,11 @@ define([
         },
 
         _onReschedule: function (event) {
+            WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Reschedule");
         },
 
         _onDeschedule: function (event) {
+            WsWorkunits.WUAction(this.workunitsGrid.getSelected(), "Deschedule");
         },
 
         _onRowDblClick: function (wuid) {
@@ -473,6 +475,8 @@ define([
             var hasNotFailed = false;
             var hasCompleted = false;
             var hasNotCompleted = false;
+            var isScheduled = false;
+            var isNotScheduled = false;
             for (var i = 0; i < selection.length; ++i) {
                 hasSelection = true;
                 if (selection[i] && selection[i].Protected !== null) {
@@ -494,6 +498,11 @@ define([
                         hasNotCompleted = true;
                     }
                 }
+                if (selection[i].EventSchedule === 2) {
+                    isScheduled = true;
+                } else if (selection[i].EventSchedule === 1) {
+                    isNotScheduled = true;
+                }
             }
 
             registry.byId(this.id + "Open").set("disabled", !hasSelection);
@@ -502,8 +511,8 @@ define([
             registry.byId(this.id + "SetToFailed").set("disabled", !hasNotProtected);
             registry.byId(this.id + "Protect").set("disabled", !hasNotProtected);
             registry.byId(this.id + "Unprotect").set("disabled", !hasProtected);
-            registry.byId(this.id + "Reschedule").set("disabled", true);    //TODO
-            registry.byId(this.id + "Deschedule").set("disabled", true);    //TODO
+            registry.byId(this.id + "Reschedule").set("disabled", !isNotScheduled);
+            registry.byId(this.id + "Deschedule").set("disabled", !isScheduled);
 
             this.menuProtect.set("disabled", !hasNotProtected);
             this.menuUnprotect.set("disabled", !hasProtected);

esp/src/eclwatch/nls/es/hpcc.js

@@ -424,7 +424,7 @@ define(
     SampleRequest: "Ejemplo del requerimiento",
     SampleResponse: "Ejemplo de la respuesta",
     Save: "Guardar",
-    Scope: "Alcance",
+    Scope: "Ámbito",
     SearchResults: "Resultados de búsqueda",
     SecondsRemaining: "Segundos que faltan",
     Security: "Seguridad",

esp/src/eclwatch/templates/WUDetailsWidget.html

@@ -10,6 +10,9 @@
                     <div id="${id}Delete" data-dojo-attach-event="onClick:_onDelete" data-dojo-type="dijit.form.Button">${i18n.Delete}</div>
                     <div id="${id}Restore" data-dojo-attach-event="onClick:_onRestore" data-dojo-type="dijit.form.Button">${i18n.Restore}</div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>
+                    <div id="${id}Reschedule" data-dojo-attach-event="onClick:_onReschedule" data-dojo-type="dijit.form.Button">${i18n.Reschedule}</div>
+                    <div id="${id}Deschedule" data-dojo-attach-event="onClick:_onDeschedule" data-dojo-type="dijit.form.Button">${i18n.Deschedule}</div>
+                    <span data-dojo-type="dijit.ToolbarSeparator"></span>
                     <div id="${id}SetToFailed" data-dojo-attach-event="onClick:_onSetToFailed" data-dojo-type="dijit.form.Button">${i18n.SetToFailed}</div>
                     <div id="${id}Abort" data-dojo-attach-event="onClick:_onAbort" data-dojo-type="dijit.form.Button">${i18n.Abort}</div>
                     <span data-dojo-type="dijit.ToolbarSeparator"></span>

plugins/pyembed/pyembed.cpp

@@ -156,6 +156,8 @@ public:
     {
         PyEval_RestoreThread(threadState);
         script.clear();
+        module.clear();
+        lru.clear();
     }
 
     inline PyObject * importFunction(size32_t lenChars, const char *utf)
@@ -328,7 +330,9 @@ public:
     PyObject *getNamedTupleType(const RtlTypeInfo *type)
     {
         // It seems the customized namedtuple types leak, and they are slow to create, so take care to reuse
-        CriticalBlock b(lock);  // Not sure if this is really needed, as we have effectively locked out other threads using the GIL
+        // Note - we do not need (and must not have) a lock protecting this. It is protected by the Python GIL,
+        // and if we add our own lock we are liable to deadlock as the code within Py_CompileStringFlags may
+        // temporarily release then re-acquire the GIL.
         if (!namedtuple)
         {
             namedtupleTypes.setown(PyDict_New());
@@ -369,7 +373,9 @@ public:
     }
     PyObject *compileScript(const char *text)
     {
-        CriticalBlock b(lock);  // Not sure if this is really needed, as we have effectively locked out other threads using the GIL
+        // Note - we do not need (and must not have) a lock protecting this. It is protected by the Python GIL,
+        // and if we add our own lock we are liable to deadlock as the code within Py_CompileStringFlags may
+        // temporarily release then re-acquire the GIL.
         if (!compiledScripts)
             compiledScripts.setown(PyDict_New());
         OwnedPyObject code;
@@ -377,7 +383,6 @@ public:
         if (!code)
         {
             code.setown(Py_CompileString(text, "", Py_eval_input));
-
             if (!code)
             {
                 PyErr_Clear();
@@ -412,7 +417,6 @@ protected:
     OwnedPyObject namedtuple;      // collections.namedtuple
     OwnedPyObject namedtupleTypes; // dictionary of return values from namedtuple()
     OwnedPyObject compiledScripts; // dictionary of previously compiled scripts
-    CriticalSection lock;
 } globalState;
 
 MODULE_INIT(INIT_PRIORITY_STANDARD)

roxie/ccd/ccdquery.cpp

@@ -696,7 +696,7 @@ protected:
         case TAKpipewrite:
             return createRoxieServerPipeWriteActivityFactory(id, subgraphId, *this, helperFactory, kind, usageCount(node), isRootAction(node));
         case TAKpull:
-            throwUnexpected(); //code generator strips for non-thor
+            return createRoxieServerPullActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKlinkedrawiterator:
             return createRoxieServerLinkedRawIteratorActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKremoteresult:

roxie/ccd/ccdserver.cpp

@@ -18350,6 +18350,94 @@ IRoxieServerActivityFactory *createRoxieServerCatchActivityFactory(unsigned _id,
 
 //=================================================================================
 
+class CRoxieServerPullActivity : public CRoxieServerActivity
+{
+    ConstPointerArray buff;
+    bool started;
+    unsigned index;
+
+public:
+    CRoxieServerPullActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
+        : CRoxieServerActivity(_factory, _probeManager)
+    {
+        started = false;
+        index = 0;
+    }
+
+    virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
+    {
+        started = false;
+        index = 0;
+        CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
+    }
+
+    virtual void reset()
+    {
+        while (buff.isItem(index))
+            ReleaseRoxieRow(buff.item(index++));
+        buff.kill();
+        started = false;
+        index = 0;
+        CRoxieServerActivity::reset();
+    }
+
+    virtual const void *nextInGroup()
+    {
+        ActivityTimer t(totalCycles, timeActivities);
+        if (!started)
+            pullInput();
+        if (buff.isItem(index))
+        {
+            const void * next = buff.item(index++);
+            if (next)
+                processed++;
+            return next;
+        }
+        return NULL;
+    }
+
+protected:
+    void pullInput()
+    {
+        bool EOGseen = false;
+        loop
+        {
+            const void * next = input->nextInGroup();
+            buff.append(next);
+            if (next == NULL)
+            {
+                if (EOGseen)
+                    break;
+                EOGseen = true;
+            }
+            else
+                EOGseen = false;
+        }
+        started = true;
+    }
+};
+
+class CRoxieServerPullActivityFactory : public CRoxieServerActivityFactory
+{
+public:
+    CRoxieServerPullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind)
+        : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind)
+    {
+    }
+
+    virtual IRoxieServerActivity *createActivity(IProbeManager *_probeManager) const
+    {
+        return new CRoxieServerPullActivity(this, _probeManager);
+    }
+};
+
+IRoxieServerActivityFactory *createRoxieServerPullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind)
+{
+    return new CRoxieServerPullActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind);
+}
+
+//=================================================================================
+
 class CRoxieServerCaseActivity : public CRoxieServerActivity
 {
     IHThorCaseArg &helper;

roxie/ccd/ccdserver.hpp

@@ -427,6 +427,7 @@ extern IRoxieServerActivityFactory *createRoxieServerWhenActivityFactory(unsigne
 extern IRoxieServerActivityFactory *createRoxieServerWhenActionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, bool _isRoot);
 
 extern IRoxieServerActivityFactory *createRoxieServerDistributionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, bool _isRoot);
+extern IRoxieServerActivityFactory *createRoxieServerPullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
 
 extern void throwRemoteException(IMessageUnpackCursor *extra);
 

thorlcr/graph/thgraphslave.cpp

@@ -129,7 +129,7 @@ void CSlaveActivity::setInput(unsigned index, CActivityBase *inputActivity, unsi
 {
     CActivityBase::setInput(index, inputActivity, inputOutIdx);
     Linked<IThorDataLink> outLink;
-    if (!inputActivity)
+    if (!inputActivity || container.isEof)
     {
         Owned<CActivityBase> nullAct = container.factory(TAKnull);