Merge remote-tracking branch 'origin/candidate-4.0.2'

Merged-by: Gavin Halliday <ghalliday@hpccsystems.com>

dali/base/dacoven.cpp

@@ -915,8 +915,9 @@ ICoven &queryCoven()
 {
     if (coven==NULL)
     {
-        ERRLOG("No active dali server connection available");
-        throw MakeStringException(-1,"No active dali server connection available");
+        Owned<IException> e = MakeStringException(-1, "No access to Dali - this normally means a plugin call is being called from a thorslave");
+        EXCLOG(e, NULL);
+        throw e.getClear();
     }
     return *coven;
 }

dali/daliadmin/daliadmin.cpp

@@ -2168,7 +2168,7 @@ static void dodalilocks(const char *pattern,const char *obj,Int64Array *conn,boo
                     while (*x)
                         curxpath.append(*(x++));
                     if (begins(ln,"/Files")) {
-                        while (*ln&&(begins(ln,"/Scope[@name=\"")||begins(ln,"/File[@name=\""))) {
+                        while (*ln&&(begins(ln,"/Scope[@name=\"")||begins(ln,"/File[@name=\"")||begins(ln,"/SuperFile[@name=\""))) {
                             if (curfile.length())
                                 curfile.append("::");
                             while (*ln&&(*ln!='"'))

docs/BuildTools/fo.xsl

@@ -103,6 +103,10 @@
    <fo:block break-after='page'/>
 </xsl:template>
   
+<xsl:template match="processing-instruction('linebreak')">
+  <fo:block/>
+</xsl:template>  
+  
 <xsl:template match="programlisting[@role='tab']">
   <fo:block xsl:use-attribute-sets="monospace.verbatim.properties">
    <xsl:attribute name="font-family">serif</xsl:attribute>

docs/BuildTools/fo.xsl.in

@@ -103,6 +103,10 @@
    <fo:block break-after='page'/>
 </xsl:template>
   
+<xsl:template match="processing-instruction('linebreak')">
+  <fo:block/>
+</xsl:template>  
+  
 <xsl:template match="programlisting[@role='tab']">
   <fo:block xsl:use-attribute-sets="monospace.verbatim.properties">
    <xsl:attribute name="font-family">serif</xsl:attribute>

docs/RunningHPCCinaVirtualMachine/RunningHPCCinaVirtualMachine.xml

@@ -137,8 +137,8 @@
 
         <listitem>
           <para>A virtualization software package:
-          VMware<superscript>®</superscript> Player or Server (version 3.0 or
-          later) or Oracle VM VirtualBox (version 4.0 or later). </para>
+          VMware<superscript>®</superscript> Player or Server (version 4.0 or
+          later) or Oracle VM VirtualBox (version 4.0 or later).</para>
         </listitem>
 
         <listitem>
@@ -163,7 +163,7 @@
 
     <itemizedlist>
       <listitem>
-        <para>VMware's Virtual Machine </para>
+        <para>VMware's Virtual Machine</para>
       </listitem>
 
       <listitem>
@@ -194,7 +194,7 @@
             <para>Open and import the image in the VMware Player</para>
           </listitem>
         </itemizedlist>Once you have completed these steps, you can evaluate
-      the HPCC Platform and learn how to use it. </para>
+      the HPCC Platform and learn how to use it.</para>
 
       <sect2>
         <title>Download and Install the VMware Player</title>
@@ -257,7 +257,7 @@
             </listitem>
 
             <listitem>
-              <para>Open the VMware Player. </para>
+              <para>Open the VMware Player.</para>
             </listitem>
 
             <listitem>
@@ -270,7 +270,7 @@
               <para>Go to the folder where you saved the downloaded file and
               select it.</para>
 
-              <para> For example (<emphasis
+              <para>For example (<emphasis
               role="bluebold">HPCCSystemsVM-<emphasis>n.n.n.n</emphasis>.ova</emphasis>
               , where n.n.n.n is the version number).</para>
 
@@ -290,7 +290,7 @@
 
             <listitem>
               <para>Press the <emphasis role="bold">Import</emphasis> button
-              to import the virtual machine into the VMware Player. </para>
+              to import the virtual machine into the VMware Player.</para>
 
               <para>Wait for the HPCC virtual machine to load to the desktop
               in the VMware player. This may take a few minutes.</para>
@@ -303,7 +303,7 @@
                     <para>The first time you use the VM Player, you must
                     accept the license agreement. You may also be prompted to
                     install add-ons, but they are not necessary for the HPCC
-                    virtual machine. </para>
+                    virtual machine.</para>
                   </listitem>
                 </varlistentry>
               </variablelist>
@@ -377,7 +377,7 @@
       <title>VM VirtualBox</title>
 
       <para>Oracle's virtualization software, VM VirtualBox is supported for
-      running the HPCC virtual machine image. </para>
+      running the HPCC virtual machine image.</para>
 
       <para>In the following section(s), you will:</para>
 
@@ -392,7 +392,7 @@
         </listitem>
 
         <listitem>
-          <para>Open and import the image in VM VirtualBox </para>
+          <para>Open and import the image in VM VirtualBox</para>
         </listitem>
       </itemizedlist>
 
@@ -419,8 +419,8 @@
           </listitem>
 
           <listitem>
-            <para>Double-click on the installation file to install VirtualBox.
-            </para>
+            <para>Double-click on the installation file to install
+            VirtualBox.</para>
           </listitem>
 
           <listitem>
@@ -443,7 +443,7 @@
                     <imagedata fileref="images/vmimg16.jpg" />
                   </imageobject>
                 </mediaobject>
-              </figure> </para>
+              </figure></para>
           </listitem>
 
           <listitem>
@@ -488,7 +488,7 @@
 
           <listitem>
             <para>From the <emphasis role="bold">File</emphasis> menu, select
-            <emphasis role="bold">Preferences</emphasis>... </para>
+            <emphasis role="bold">Preferences</emphasis>...</para>
 
             <para>The <emphasis>VirtualBox - Settings</emphasis> dialog
             displays.<figure id="VBoxSettings">
@@ -504,12 +504,12 @@
 
           <listitem>
             <para>Select<emphasis role="bold"> Network</emphasis> from the
-            list on the left side of the window. </para>
+            list on the left side of the window.</para>
           </listitem>
 
           <listitem>
             <para>Double-click on the <emphasis>VirtualBox Host-Only Ethernet
-            Adaptor</emphasis>. </para>
+            Adaptor</emphasis>.</para>
           </listitem>
 
           <listitem>
@@ -526,13 +526,13 @@
                 </mediaobject>
               </figure><variablelist>
                 <varlistentry>
-                  <term>Note: </term>
+                  <term>Note:</term>
 
                   <listitem>
                     <para>You may need to add IP Address values to the Lower
                     Address Bound and Upper Address Bound fields. The address
                     the VM uses is in the private IP address range of
-                    192.168.x.x </para>
+                    192.168.x.x</para>
                   </listitem>
                 </varlistentry>
               </variablelist></para>
@@ -542,7 +542,7 @@
             <para>Press the <emphasis role="bold">OK</emphasis> button on the
             <emphasis>Network Details</emphasis> dialog box, then press the
             <emphasis role="bold">OK</emphasis> button on the
-            <emphasis>VirtualBox - Settings</emphasis> box. </para>
+            <emphasis>VirtualBox - Settings</emphasis> box.</para>
           </listitem>
         </orderedlist>
       </sect2>
@@ -585,8 +585,7 @@
 
             <listitem>
               <para>Press the <emphasis role="bold">Open appliance</emphasis>
-              button from the dialog to select the appliance to import.
-              </para>
+              button from the dialog to select the appliance to import.</para>
             </listitem>
 
             <listitem>
@@ -602,7 +601,7 @@
               <para>Press the <emphasis role="bold">Open</emphasis> button to
               start the import. Follow the prompts in the import process
               pressing <emphasis role="bold">Next</emphasis> as appropriate,
-              then press <emphasis role="bold">Import</emphasis>. </para>
+              then press <emphasis role="bold">Import</emphasis>.</para>
 
               <para>Wait for the HPCC virtual machine to import. This may take
               a few minutes.</para>
@@ -610,20 +609,20 @@
 
             <listitem>
               <para>At the main window, the newly imported virtual machine is
-              now listed. </para>
+              now listed.</para>
             </listitem>
 
             <listitem>
               <para>Double-click on that virtual machine to start it.
               <variablelist>
                   <varlistentry>
-                    <term>TIP: </term>
+                    <term>TIP:</term>
 
                     <listitem>
                       <para>If you get any Network Error messages, please
                       review the steps in <xref
                       linkend="Install_VM_VirtualBox" /> and verify that all
-                      the network settings are set appropriately. </para>
+                      the network settings are set appropriately.</para>
                     </listitem>
                   </varlistentry>
                 </variablelist></para>

esp/services/ws_dfu/ws_dfuService.cpp

@@ -1158,23 +1158,35 @@ bool CWsDfuEx::DFUDeleteFiles(IEspContext &context, IEspDFUArrayActionRequest &r
 
             if (j>0)
             { // 2nd pass, now we want to skip superfiles and the files which cannot do the lookup.
-
-                if (superFileNames.contains(filename))
-                    continue;
-
-                if (filesCannotBeDeleted.contains(filename))
+                if (superFileNames.contains(filename) || filesCannotBeDeleted.contains(filename))
                     continue;
             }
 
-            Owned<IDistributedFile> df;
             try
             {
-                df.setown(queryDistributedFileDirectory().lookup(filename, userdesc, true));
-                if(!df)
+                IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
                 {
-                    returnStr.appendf("<Message><Value>Cannot delete %s: file not found</Value></Message>", filename);
-                    filesCannotBeDeleted.append(filename);
-                    continue;
+                    Owned<IDistributedFile> df = fdir.lookup(filename, userdesc, true);
+                    if(!df)
+                    {
+                        returnStr.appendf("<Message><Value>Cannot delete %s: file not found</Value></Message>", filename);
+                        filesCannotBeDeleted.append(filename);
+                        continue;
+                    }
+                    if (0==j) // skip non-super files on 1st pass
+                    {
+                        if(!df->querySuperFile())
+                            continue;
+
+                        superFileNames.append(filename);
+                    }
+                }
+                if (!fdir.removeEntry(filename, userdesc, NULL, REMOVE_FILE_SDS_CONNECT_TIMEOUT))
+                    returnStr.appendf("<Message><Value>Failed to delete %s</Value></Message>", filename);
+                else
+                {
+                    PROGLOG("Deleted Logical File: %s by: %s\n",filename, username.str());
+                    returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>", filename);
                 }
             }
             catch(IException* e)
@@ -1187,37 +1199,12 @@ bool CWsDfuEx::DFUDeleteFiles(IEspContext &context, IEspDFUArrayActionRequest &r
                     emsg.replaceString("Create ", "Delete ");
 
                 returnStr.appendf("<Message><Value>Cannot delete %s: %s</Value></Message>", filename, emsg.str());
+                e->Release();
             }
             catch(...)
             {
                 returnStr.appendf("<Message><Value>Cannot delete %s: unknown exception.</Value></Message>", filename);
             }
-            if (df)
-            {
-                if (0==j) // skip non-super files on 1st pass
-                {
-                    if(!df->querySuperFile())
-                        continue;
-
-                    superFileNames.append(filename);
-                }
-
-                DBGLOG("CWsDfuEx::DFUDeleteFiles User=%s Action=Delete File=%s",username.str(), filename);
-                try
-                {
-                    df->detach(REMOVE_FILE_SDS_CONNECT_TIMEOUT);
-                    PROGLOG("Deleted Logical File: %s\n",filename);
-                    returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>", filename);
-                }
-                catch (IException *e)
-                {
-                    StringBuffer errorMsg;
-                    e->errorMessage(errorMsg);
-                    PROGLOG("%s", errorMsg.str());
-                    e->Release();
-                    returnStr.appendf("<Message><Value>%s</Value></Message>", errorMsg.str());
-                }
-            }
         }
     }
 

esp/smc/SMCLib/LogicFileWrapper.cpp

@@ -67,16 +67,23 @@ bool LogicFileWrapper::doDeleteFile(const char* logicalName,const char *cluster,
 
     try
     {
-        Owned<IDistributedFile> df = queryDistributedFileDirectory().lookup(cname.str(), udesc, true) ;
-        if(!df)
+        IDistributedFileDirectory &fdir = queryDistributedFileDirectory();
         {
-            returnStr.appendf("<Message><Value>File %s not found</Value></Message>", cname.str());
-            return false;
+            Owned<IDistributedFile> df = fdir.lookup(cname.str(), udesc, true) ;
+            if(!df)
+            {
+                returnStr.appendf("<Message><Value>File %s not found</Value></Message>", cname.str());
+                return false;
+            }
         }
 
-        df->detach();
-        returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>", cname.str());
-        DBGLOG("%s", returnStr.str());
+        if (!fdir.removeEntry(cname.str(), udesc))
+            returnStr.appendf("<Message><Value>Failed to delete %s</Value></Message>", cname.str());
+        else
+        {
+            returnStr.appendf("<Message><Value>Deleted File %s</Value></Message>", cname.str());
+            DBGLOG("%s", returnStr.str());
+        }
         return true;
     }
     catch (IException *e)

roxie/ccd/ccddali.cpp

@@ -98,9 +98,14 @@ public:
     virtual void notify(SubscriptionId subid, const char *xpath, SDSNotifyFlags flags, unsigned valueLen, const void *valueData)
     {
         Linked<CDaliPackageWatcher> me = this;  // Ensure that I am not released by the notify call (which would then access freed memory to release the critsec)
-        CriticalBlock b(crit);
-        if (notifier)
-            notifier->notify(subid, xpath, flags, valueLen, valueData);
+        Linked<ISDSSubscription> myNotifier;
+        {
+            CriticalBlock b(crit);
+            myNotifier.set(notifier);
+            // allow crit to be released, allowing this to be unsubscribed, to avoid deadlocking when other threads via notify call unsubscribe
+        }
+        if (myNotifier)
+            myNotifier->notify(subid, xpath, flags, valueLen, valueData);
     }
 };
 

thorlcr/activities/msort/thsortu.cpp

@@ -1297,7 +1297,7 @@ public:
         loop {
             CRollingCacheElem * r = cache->mid(0);
             if (!r)
-                break; // kit eos              
+                break; // hit eos
             int c = cmp->docompare(left,r->row);
             if (c==0) {
                 r->cmp = limitedcmp->docompare(left,r->row);
@@ -1408,7 +1408,7 @@ public:
     ThorActivityKind kind;
     Owned<IException> exc;
     CriticalSection sect;
-    bool eos;
+    bool eos, selfJoin;
 
 
     void setException(IException *e,const char *title)
@@ -1479,7 +1479,7 @@ public:
     void doMatch(cWorkItem &work,SimpleInterThreadQueueOf<cOutItem,false> &outqueue) 
     {
         MemoryBuffer rmatchedbuf;  
-        CThorExpandingRowArray &rgroup = (kind==TAKselfjoin||kind==TAKselfjoinlight)?work.lgroup:work.rgroup;
+        CThorExpandingRowArray &rgroup = selfJoin?work.lgroup:work.rgroup;
         bool *rmatched;
         if (rightouter) {
             rmatched = (bool *)rmatchedbuf.clear().reserve(rgroup.ordinality());
@@ -1519,7 +1519,7 @@ public:
         }
     }
 
-    CMultiCoreJoinHelperBase(CActivityBase &_activity, unsigned numthreads, IJoinHelper *_jhelper, IHThorJoinArg *_helper, IEngineRowAllocator *_allocator)
+    CMultiCoreJoinHelperBase(CActivityBase &_activity, unsigned numthreads, bool _selfJoin, IJoinHelper *_jhelper, IHThorJoinArg *_helper, IEngineRowAllocator *_allocator)
         : activity(_activity), allocator(_allocator)
     {
         kind = activity.queryContainer().getKind();
@@ -1529,6 +1529,7 @@ public:
         leftouter = (flags & JFleftouter) != 0;
         rightouter = (flags & JFrightouter) != 0;
         exclude = (flags & JFexclude) != 0;
+        selfJoin = _selfJoin;
         numworkers = numthreads;
         eos = false;
     }
@@ -1628,7 +1629,6 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase
         {
             PROGLOG("CMultiCoreJoinHelper::cWorker started");
             MemoryBuffer rmatchedbuf;  
-            bool selfjoin = (parent->kind==TAKselfjoin||parent->kind==TAKselfjoinlight);
             loop {
                 work.clear();
                 workready.signal();
@@ -1661,8 +1661,8 @@ class CMultiCoreJoinHelper: public CMultiCoreJoinHelperBase
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CMultiCoreJoinHelper(CActivityBase &activity, unsigned numthreads, IJoinHelper *_jhelper, IHThorJoinArg *_helper, IEngineRowAllocator *_allocator)
-        : CMultiCoreJoinHelperBase(activity, numthreads, _jhelper, _helper, _allocator)
+    CMultiCoreJoinHelper(CActivityBase &activity, unsigned numthreads, bool selfJoin, IJoinHelper *_jhelper, IHThorJoinArg *_helper, IEngineRowAllocator *_allocator)
+        : CMultiCoreJoinHelperBase(activity, numthreads, selfJoin, _jhelper, _helper, _allocator)
     {
         reader.parent = this;
         workers = new cWorker *[numthreads];
@@ -1802,7 +1802,6 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase
     {
         CMultiCoreUnorderedJoinHelper *parent;
     public:
-        SimpleInterThreadQueueOf<cOutItem,false> outqueue;          // used in ordered
         cWorker(CMultiCoreUnorderedJoinHelper *_parent)
             : Thread("CMulticoreUnorderedJoinHelper::cWorker"), parent(_parent)
         {
@@ -1838,8 +1837,8 @@ class CMultiCoreUnorderedJoinHelper: public CMultiCoreJoinHelperBase
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CMultiCoreUnorderedJoinHelper(CActivityBase &activity, unsigned numthreads, IJoinHelper *_jhelper, IHThorJoinArg *_helper, IEngineRowAllocator *_allocator)
-        : CMultiCoreJoinHelperBase(activity, numthreads, _jhelper, _helper, _allocator)
+    CMultiCoreUnorderedJoinHelper(CActivityBase &activity, unsigned numthreads, bool selfJoin, IJoinHelper *_jhelper, IHThorJoinArg *_helper, IEngineRowAllocator *_allocator)
+        : CMultiCoreJoinHelperBase(activity, numthreads, selfJoin, _jhelper, _helper, _allocator)
     {
         reader.parent = this;
         stoppedworkers = 0;
@@ -1944,8 +1943,8 @@ IJoinHelper *createJoinHelper(CActivityBase &activity, IHThorJoinArg *helper, IE
         return jhelper;
     unsigned numthreads = getAffinityCpus();
     if (unsortedoutput)
-        return new CMultiCoreUnorderedJoinHelper(activity, numthreads, jhelper, helper, allocator);
-    return new CMultiCoreJoinHelper(activity, numthreads, jhelper, helper, allocator);
+        return new CMultiCoreUnorderedJoinHelper(activity, numthreads, false, jhelper, helper, allocator);
+    return new CMultiCoreJoinHelper(activity, numthreads, false, jhelper, helper, allocator);
 }
 
 
@@ -1962,8 +1961,8 @@ IJoinHelper *createSelfJoinHelper(CActivityBase &activity, IHThorJoinArg *helper
         return jhelper;
     unsigned numthreads = getAffinityCpus();
     if (unsortedoutput)
-        return new CMultiCoreUnorderedJoinHelper(activity, numthreads, jhelper, helper, allocator);
-    return new CMultiCoreJoinHelper(activity, numthreads, jhelper, helper, allocator);
+        return new CMultiCoreUnorderedJoinHelper(activity, numthreads, true, jhelper, helper, allocator);
+    return new CMultiCoreJoinHelper(activity, numthreads, true, jhelper, helper, allocator);
 }