Merge remote-tracking branch 'origin/candidate-4.2.4' into closedown-4.2.x

Conflicts:
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorsoapcall.cpp

@@ -217,12 +217,13 @@ typedef IArrayOf<Url> UrlArray;
 
 //=================================================================================================
 
+//MORE: This whole class should really be replaced with a single function, avoiding at least one string clone.
 class UrlListParser
 {
 public:
     UrlListParser(const char * text)
     {
-        fullText = strdup(text);
+        fullText = text ? strdup(text) : NULL;
     }
 
     ~UrlListParser()
@@ -232,17 +233,20 @@ public:
 
     unsigned getUrls(UrlArray &array)
     {
-        char *copyFullText = strdup(fullText);
-
-        char *saveptr;
-        char *url = strtok_r(copyFullText, "|", &saveptr);
-        while (url != NULL)
+        if (fullText)
         {
-            array.append(*new Url(url));
-            url = strtok_r(NULL, "|", &saveptr);
-        }
+            char *copyFullText = strdup(fullText);
+
+            char *saveptr;
+            char *url = strtok_r(copyFullText, "|", &saveptr);
+            while (url != NULL)
+            {
+                array.append(*new Url(url));
+                url = strtok_r(NULL, "|", &saveptr);
+            }
 
-        free(copyFullText);
+            free(copyFullText);
+        }
         return array.ordinality();
     }
 

common/workunit/referencedfilelist.cpp

@@ -242,6 +242,7 @@ void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstC
     }
     else
     {
+        flags |= RefSubFile;
         if (!dstCluster || !*dstCluster)
             return;
         if (df->findCluster(dstCluster)==NotFound)

common/workunit/workunit.cpp

@@ -4554,7 +4554,7 @@ extern WORKUNIT_API IStringIterator *getTargetClusters(const char *processType,
     {
         StringBuffer xpath;
         xpath.appendf("%s", processType ? processType : "*");
-        if (processName)
+        if (processName && *processName)
             xpath.appendf("[@process=\"%s\"]", processName);
         Owned<IPropertyTreeIterator> targets = conn->queryRoot()->getElements("Software/Topology/Cluster");
         ForEach(*targets)

dali/sasha/saxref.cpp

@@ -2308,7 +2308,7 @@ public:
                 {
                     if (0 == expireDays)
                     {
-                        bool isPersist = attr.queryProp("@persist");
+                        bool isPersist = attr.getPropBool("@persistent");
                         expireDays = isPersist ? defaultPersistExpireDays : defaultExpireDays;
                     }
                     CDateTime now;

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -124,7 +124,7 @@
           url="http://hpccsystems.com/download/free-community-edition">http://hpccsystems.com/download/free-community-edition</ulink>
           and install.</para>
 
-          <para>On CentOS/Red Hat/SuSe:</para>
+          <para>On CentOS/Red Hat:</para>
 
           <para><programlisting> sudo rpm -Uvh &lt;rpm file name&gt;</programlisting>On
           Ubuntu/Debian:</para>
@@ -135,7 +135,7 @@
         <listitem>
           <para>Start your HPCC System.</para>
 
-          <para>On CentOS/Red Hat/SuSe:</para>
+          <para>On CentOS/Red Hat:</para>
 
           <para><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
@@ -544,10 +544,6 @@
             </listitem>
 
             <listitem>
-              <para>64-bit openSUSE 11.3 or 11.4</para>
-            </listitem>
-
-            <listitem>
               <para>64-bit Debian 6.x (Squeeze)</para>
             </listitem>
           </itemizedlist></para>
@@ -753,7 +749,7 @@
         instructions:</para>
 
         <sect3 id="Installing_CentOS-RedHat-Suse">
-          <title>CentOS/Red Hat/SuSe</title>
+          <title>CentOS/Red Hat</title>
 
           <para>Install RPM with the -Uvh switch.</para>
 
@@ -763,18 +759,6 @@
 
           <programlisting>sudo rpm -Uvh &lt;rpm file name&gt;</programlisting>
 
-          <blockquote>
-            <para><emphasis role="bold">Note:</emphasis> For ANY version of
-            SuSe you must set a password for the hpcc user on all nodes. One
-            way to do do this is to issue the following command:</para>
-
-            <para><programlisting>sudo passwd hpcc</programlisting>Be sure to
-            set the password on ALL nodes.</para>
-
-            <para><emphasis role="bold">Note:</emphasis> For CentOS
-            installations, the Fedora EPEL repository is required.</para>
-          </blockquote>
-
           <sect4 id="Optional_Plug-ins">
             <title>Optional Plug-ins</title>
 
@@ -845,7 +829,7 @@
             <para>Start the system using the default configuration.</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1278,7 +1262,7 @@
             command:</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1303,7 +1287,7 @@
 
                       <entry>You can use this command to confirm HPCC
                       processes are stopped (on Centos/Red
-                      Hat/SuSe):<para><programlisting>sudo /sbin/service hpcc-init status</programlisting><phrase>For
+                      Hat):<para><programlisting>sudo /sbin/service hpcc-init status</programlisting><phrase>For
                       Ubuntu</phrase><programlisting>sudo service hpcc-init status
 </programlisting><phrase>For Debian 6 (Squeeze)</phrase><programlisting>sudo /etc/init.d/hpcc-init status</programlisting></para></entry>
                     </row>
@@ -1618,7 +1602,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
             the HPCC system on an individual node:</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1676,7 +1660,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <title>To start the system:</title>
 
           <para><emphasis role="bold">Centos/Red
-          Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
+          Hat</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
           <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1691,7 +1675,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <title>To stop the system:</title>
 
           <para><emphasis role="bold">Centos/Red
-          Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
+          Hat</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
 
           <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1731,7 +1715,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           in the init system as follows.</para>
 
           <para><emphasis role="bold">Centos/Red
-          Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init -c &lt;component name&gt; &lt;command&gt;</programlisting></para>
+          Hat</emphasis><programlisting>sudo /sbin/service hpcc-init -c &lt;component name&gt; &lt;command&gt;</programlisting></para>
 
           <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1777,7 +1761,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
             command on <emphasis role="bold">every</emphasis> node:</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -2866,7 +2850,7 @@ OUTPUT(ValidWords)
       your system. If necessary, do so on each node that it is installed
       on.</para>
 
-      <para><emphasis role="bold">Centos/Red Hat/SuSe</emphasis></para>
+      <para><emphasis role="bold">Centos/Red Hat</emphasis></para>
 
       <programlisting>sudo rpm -e hpccsystems-platform </programlisting>
 
@@ -3106,7 +3090,7 @@ sudo /sbin/service hpcc-init -c esp start
       <para><emphasis role="bold">RPM-based systems:</emphasis></para>
 
       <para>If you are interested in using external languages for RPM-based
-      systems (CentOS/Red Hat/SuSe), you need to download and install the
+      systems (CentOS/Red Hat), you need to download and install the
       appropriate platform installation distribution <emphasis
       role="bold">with plug-ins</emphasis> option from the downloads
       site.</para>
@@ -3279,8 +3263,8 @@ add1(10);
             <para>JavaScript support is available for CentOS 6.x or later (not
             available for CentOS 5.x).</para>
 
-            <para>On an RPM-based system (CentOS/Red Hat/SuSe) install
-            <emphasis role="bold">v8embed.</emphasis></para>
+            <para>On an RPM-based system (CentOS/Red Hat) install <emphasis
+            role="bold">v8embed.</emphasis></para>
 
             <para><emphasis role="bold">Debian-based
             systems:</emphasis></para>
@@ -3397,7 +3381,7 @@ split_words('Once upon a time');
 
             <para><emphasis role="bold">RPM-based systems:</emphasis></para>
 
-            <para>On an RPM-based system (CentOS/Red Hat/SuSe) use <emphasis
+            <para>On an RPM-based system (CentOS/Red Hat) use <emphasis
             role="bold">R-core</emphasis> and <emphasis
             role="bold">R-core-devel</emphasis></para>
 

ecl/eclcc/eclcc.cpp

@@ -278,7 +278,6 @@ public:
     void processBatchedFile(IFile & file, bool multiThreaded);
 
     virtual void noteCluster(const char *clusterName);
-    virtual void registerFile(const char * filename, const char * description);
     virtual bool allowAccess(const char * category);
 
 protected:
@@ -1661,9 +1660,6 @@ bool EclCompileInstance::reportErrorSummary()
 void EclCC::noteCluster(const char *clusterName)
 {
 }
-void EclCC::registerFile(const char * filename, const char * description)
-{
-}
 bool EclCC::allowAccess(const char * category)
 {
     ForEachItemIn(idx1, deniedPermissions)

ecl/hql/hqlfold.cpp

@@ -6069,7 +6069,8 @@ IHqlExpression * foldScopedHqlExpression(IHqlExpression * dataset, IHqlExpressio
 
     CExprFolderTransformer folder(NULL, foldOptions);
 
-    folder.setScope(dataset);
+    if (dataset)
+        folder.setScope(dataset);
 
     IHqlExpression * ret = folder.transformRoot(expr);
 

ecl/hql/hqlopt.cpp

@@ -282,15 +282,8 @@ IHqlExpression * CTreeOptimizer::swapNodeWithChild(IHqlExpression * parent)
 IHqlExpression * CTreeOptimizer::forceSwapNodeWithChild(IHqlExpression * parent)
 {
     OwnedHqlExpr swapped = swapNodeWithChild(parent);
-    return replaceOwnedAttribute(swapped, getNoHoistAttr());
-}
-
-IHqlExpression * CTreeOptimizer::getNoHoistAttr()
-{
-    //Ensure the attribute is unique for each call to the optimizer - otherwise it stops items being hoisted that could be.
-    if (!noHoistAttr)
-        noHoistAttr.setown(createAttribute(_noHoist_Atom, createUniqueId()));
-    return LINK(noHoistAttr);
+    queryBodyExtra(swapped)->setStopHoist();
+    return swapped.getClear();
 }
 
 IHqlExpression * CTreeOptimizer::swapNodeWithChild(IHqlExpression * parent, unsigned childIndex)
@@ -320,26 +313,40 @@ IHqlExpression * CTreeOptimizer::swapIntoIf(IHqlExpression * expr, bool force)
     OwnedHqlExpr newLeft = replaceChildDataset(body, left, 0);
     OwnedHqlExpr newRight = replaceChildDataset(body, right, 0);
 
-    OwnedHqlExpr transformedLeft = transform(newLeft);
-    OwnedHqlExpr transformedRight = transform(newRight);
+    HqlExprArray args;
+    args.append(*LINK(cond));
+    args.append(*LINK(newLeft));
+    args.append(*LINK(newRight));
+    OwnedHqlExpr newIf = child->clone(args);
+
+    if (!alreadyHasUsage(newIf))
+    {
+        incUsage(newLeft);
+        incUsage(newRight);
+    }
 
-    //Don't bother moving the condition over the if if it doesn't improve the code elsewhere
-    if (force || (newLeft != transformedLeft) || (newRight != transformedRight))
+    OwnedHqlExpr transformedIf = transform(newIf);
+    if (force || (newIf != transformedIf))
     {
         //Need to call dec on all expressions that are no longer used... left and right still used by newLeft/newRight
-        noteUnused(child);
-        DBGLOG("Optimizer: Swap %s and %s", queryNode0Text(expr), queryNode1Text(child));
-        HqlExprArray args;
-        args.append(*LINK(cond));
-        args.append(*LINK(transformedLeft));
-        args.append(*LINK(transformedRight));
-        OwnedHqlExpr ret = child->clone(args);
-        if (!alreadyHasUsage(ret))
+        if (!alreadyHasUsage(newIf))
         {
-            incUsage(transformedLeft);
-            incUsage(transformedRight);
+            //This may possibly leave left/right linked once if transformed(newLeft) doesn't use left any more.
+            //But a recursiveDecUsage could cause too much to be decremented.
+            if (newLeft != transformedIf->queryChild(1))
+                decUsage(newLeft);
+            if (newRight != transformedIf->queryChild(2))
+                decUsage(newRight);
         }
-        return ret.getClear();
+        noteUnused(child);
+        DBGLOG("Optimizer: Swap %s and %s", queryNode0Text(expr), queryNode1Text(child));
+        return transformedIf.getClear();
+    }
+
+    if (!alreadyHasUsage(newIf))
+    {
+        decUsage(newLeft);
+        decUsage(newRight);
     }
     return LINK(expr);
 }
@@ -1464,8 +1471,13 @@ IHqlExpression * CTreeOptimizer::optimizeProjectInlineTable(IHqlExpression * tra
         if (!next || monitor.isComplex())
             return NULL;
 
-        if (onlyFoldConstant && !isConstantTransform(next))
-            return NULL;
+        if (onlyFoldConstant)
+        {
+            next.setown(foldScopedHqlExpression(NULL, next));
+
+            if (!isConstantTransform(next))
+                return NULL;
+        }
         newValues.append(*ensureTransformType(next, no_transform));
     }
 
@@ -1543,6 +1555,10 @@ IHqlExpression * CTreeOptimizer::inheritUsage(IHqlExpression * newExpr, IHqlExpr
 {
     OptTransformInfo * newExtra = queryBodyExtra(newExpr);
     OptTransformInfo * oldExtra = queryBodyExtra(oldExpr);
+
+    if (oldExtra->getStopHoist())
+        newExtra->setStopHoist();
+
 #ifdef TRACE_USAGE
     if (newExpr->isDataset() || newExpr->isDatarow())
         DBGLOG("%lx inherit %d,%d (from %lx) [%s]", (unsigned)newExpr, newExtra->useCount, oldExtra->useCount, (unsigned)oldExpr, queryNode0Text(newExpr));
@@ -1719,7 +1735,7 @@ bool CTreeOptimizer::childrenAreShared(IHqlExpression * expr)
 
 bool CTreeOptimizer::isWorthMovingProjectOverLimit(IHqlExpression * project)
 {
-    if (noHoistAttr && project->queryAttribute(_noHoist_Atom) == noHoistAttr)
+    if (queryBodyExtra(project)->getStopHoist())
         return false;
 
     IHqlExpression * expr = project->queryChild(0);
@@ -1905,8 +1921,8 @@ ANewTransformInfo * CTreeOptimizer::createTransformInfo(IHqlExpression * expr)
 IHqlExpression * CTreeOptimizer::expandFields(TableProjectMapper * mapper, IHqlExpression * expr, IHqlExpression * oldDataset, IHqlExpression * newDataset, IExpandCallback * _expandCallback)
 {
     OwnedHqlExpr expandedFilter = mapper->expandFields(expr, oldDataset, newDataset, _expandCallback);
-    if (options & HOOfold)
-        expandedFilter.setown(foldHqlExpression(expandedFilter));
+    //There used to be code to constant fold filters here - but it can cause dataset expressions to become duplicated
+    //causing code to be duplicated.  Only fold expressions that are reduced to constants.
     return expandedFilter.getClear();
 }
 
@@ -3840,10 +3856,12 @@ IHqlExpression * optimizeHqlExpression(IHqlExpression * expr, unsigned options)
     unwindCommaCompound(args, expr);
     optimizeHqlExpression(newArgs, args, options);
     OwnedHqlExpr optimized = createActionList(newArgs);
+
+    if (expr == optimized)
+        return optimized.getClear();
+
     //If the graph was optimized then it is highly likely there are now constant folding opportunities
-    if (expr != optimized)
-        return foldHqlExpression(optimized);
-    return optimized.getClear();
+    return foldHqlExpression(optimized);
 }
 
 

ecl/hql/hqlopt.ipp

@@ -32,6 +32,9 @@ class OptTransformInfo : public NewTransformInfo
 public:
     OptTransformInfo(IHqlExpression * _expr) : NewTransformInfo(_expr) { useCount = 0; }
 
+    bool getStopHoist() { return spareByte1; }
+    void setStopHoist() { spareByte1 = true; }
+
 public:
     unsigned useCount;
 };
@@ -117,15 +120,12 @@ protected:
     inline const char * queryNode0Text(IHqlExpression * expr) { return queryChildNodeTraceText(nodeText[0], expr); }
     inline const char * queryNode1Text(IHqlExpression * expr) { return queryChildNodeTraceText(nodeText[1], expr); }
 
-    IHqlExpression * getNoHoistAttr();
-
     inline bool isAlwaysLocal() const { return (options & HOOalwayslocal) != 0; }
 
 protected:
     typedef NewHqlTransformer PARENT;
     unsigned options;
     StringBuffer nodeText[2];
-    HqlExprAttr noHoistAttr;
 };
 
 

ecl/hqlcpp/hqlcpp.cpp

@@ -1267,8 +1267,6 @@ void HqlCppInstance::addHint(const char * hintXml, ICodegenContextCallback * ctx
 
         Owned<IWUQuery> query = workunit->updateQuery();
         associateLocalFile(query, FileTypeCpp, hintFilename.str(), "Hints", 0);
-
-        ctxCallback->registerFile(hintFilename.str(), "Hints");
     }
     appendHintText(hintXml);
 }
@@ -1334,7 +1332,6 @@ void HqlCppInstance::flushResources(const char *filename, ICodegenContextCallbac
         {
             Owned<IWUQuery> query = workunit->updateQuery();
             associateLocalFile(query, FileTypeHintXml, resTextName, "Workunit resource text", 0);
-            ctxCallback->registerFile(resTextName, "Workunit Res.txt");
         }
         useLibrary(filename);
     }

ecl/hqlcpp/hqlcpp.hpp

@@ -87,7 +87,6 @@ The following debug options are currently supported by the code generator:
 interface ICodegenContextCallback : public IInterface
 {
     virtual void noteCluster(const char *clusterName) = 0;
-    virtual void registerFile(const char * filename, const char * description) = 0;
     virtual bool allowAccess(const char * category) = 0;
 };
 

ecl/hqlcpp/hqlecl.cpp

@@ -53,7 +53,6 @@ class NullContextCallback : public CInterface, implements ICodegenContextCallbac
     IMPLEMENT_IINTERFACE
 
     virtual void noteCluster(const char *clusterName) {}
-    virtual void registerFile(const char * filename, const char * description) {}
     virtual bool allowAccess(const char * category) { return true; }
 };
 
@@ -135,7 +134,6 @@ void HqlDllGenerator::addCppName(const char * filename)
     {
         Owned<IWUQuery> query = wu->updateQuery();
         associateLocalFile(query, FileTypeCpp, filename, pathTail(filename), 0);
-        ctxCallback->registerFile(filename, "Workunit CPP");
     }
 }
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5902,8 +5902,6 @@ void HqlCppTranslator::writeFieldUsage(const char * targetDir, IPropertyTree * s
 
         Owned<IWUQuery> query = wu()->updateQuery();
         associateLocalFile(query, FileTypeXml, fullname, "FieldUsage", 0);
-
-        ctxCallback->registerFile(fullname.str(), "FieldUsage");
     }
 }
 
@@ -16819,19 +16817,19 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
     }
 
     //virtual unsigned numParallelThreads()
-    doBuildUnsignedFunction(instance->classctx, "numParallelThreads", queryAttributeChild(expr, parallelAtom, 0));
+    doBuildUnsignedFunction(instance->startctx, "numParallelThreads", queryAttributeChild(expr, parallelAtom, 0));
 
     //virtual unsigned numRecordsPerBatch()
-    doBuildUnsignedFunction(instance->classctx, "numRecordsPerBatch", queryAttributeChild(expr, mergeAtom, 0));
+    doBuildUnsignedFunction(instance->startctx, "numRecordsPerBatch", queryAttributeChild(expr, mergeAtom, 0));
 
     //virtual int numRetries()
-    doBuildSignedFunction(instance->classctx, "numRetries", queryAttributeChild(expr, retryAtom, 0));
+    doBuildSignedFunction(instance->startctx, "numRetries", queryAttributeChild(expr, retryAtom, 0));
 
     //virtual double getTimeout()
-    doBuildDoubleFunction(instance->classctx, "getTimeout", queryAttributeChild(expr, timeoutAtom, 0));
+    doBuildDoubleFunction(instance->startctx, "getTimeout", queryAttributeChild(expr, timeoutAtom, 0));
 
     //virtual double getTimeLimit()
-    doBuildDoubleFunction(instance->classctx, "getTimeLimit", queryAttributeChild(expr, timeLimitAtom, 0));
+    doBuildDoubleFunction(instance->startctx, "getTimeLimit", queryAttributeChild(expr, timeLimitAtom, 0));
 
     if (namespaceAttr)
     {

esp/eclwatch/ws_XSLT/CMakeLists.txt

@@ -77,6 +77,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/queryfiledetails.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/queryfilelist.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/queryfilelistdone.xslt
+    ${CMAKE_CURRENT_SOURCE_DIR}/QueriesUsingFile.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/result.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/result_lib.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/result_lib1.xslt

esp/eclwatch/ws_XSLT/dfu.xslt

@@ -142,7 +142,7 @@
                 ]);
             }
 
-            if (roxiecluster != 0) {
+            if (roxiecluster != 0 || cluster=="") {
                 oMenu.addItems([
                     { text: "Used By", onclick: { fn: queriesUsingDFUFile } }
                 ]);

esp/eclwatch/ws_XSLT/wuid.xslt

@@ -650,14 +650,16 @@
 
                 var exceptionHtml = "Workunit successfully published.";
                 var headerText = "Workunit Published";
-                var i = o.responseText.indexOf('<h3>Exception');
+                var hasException = 0;
+                var i = o.responseText.indexOf('<Exception>');
                 if (i > -1) {
+                    hasException = 1;
                     headerText = "Error Publishing Workunit";
-                    var j = o.responseText.indexOf('<table');
+                    var j = o.responseText.indexOf('<Message>');
                     if (j > -1) {
-                        var k = o.responseText.indexOf('</table>');
+                        var k = o.responseText.indexOf('</Message>');
                         if (k > -1) {
-                            exceptionHtml = o.responseText.substring(j, k+8);
+                            exceptionHtml = o.responseText.substring(j+9, k);
                         }
                     }
                 } else {
@@ -690,7 +692,7 @@
                 }
                 var publishDialog =
                      new YAHOO.widget.SimpleDialog("publishDialog",
-                      { width: "300px",
+                      { width: hasException ? null : "300px",
                         fixedcenter: true,
                         visible: false,
                         draggable: false,

esp/scm/ws_workunits.ecm

@@ -1108,7 +1108,7 @@ ESPrequest [nil_remove] WUPublishWorkunitRequest
     string Comment;
     bool DontCopyFiles(false);
     string SourceProcess;
-    bool AllowForeignFiles(true);
+    bool AllowForeignFiles(false);
 };
 
 ESPresponse [exceptions_inline] WUPublishWorkunitResponse

esp/services/ws_workunits/ws_workunitsAuditLogs.cpp

@@ -1475,6 +1475,17 @@ int CWsWorkunitsSoapBindingEx::onGet(CHttpRequest* request, CHttpResponse* respo
             streamJobQueueListResponse(*ctx, cluster, startDate, endDate, response, xls.str());
             return 0;
         }
+        else if(!strnicmp(path.str(), "/WsWorkunits/filesInUse", 28))
+        {
+            StringBuffer xml;
+            wswService->filesInUse.toStr(xml);
+
+            response->setContent(xml);
+            response->setContentType("text/xml");
+            response->setStatus(HTTP_STATUS_OK);
+            response->send();
+            return 0;
+        }
     }
     catch(IException* e)
     {

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -328,16 +328,18 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
             if (aborting)
                 return;
             IReferencedFile &rf = files->query();
-            if (!(rf.getFlags() & RefSubFile))
-                continue;
+            //if (!(rf.getFlags() & RefSubFile))
+            //    continue;
             const char *lfn = rf.getLogicalName();
             if (!lfn || !*lfn)
                 continue;
 
-            if (!queryTree->hasProp(xpath.setf("SubFile[@lfn='%s']", lfn)))
+            if (!queryTree->hasProp(xpath.setf("File[@lfn='%s']", lfn)))
             {
-                IPropertyTree *fileTree = queryTree->addPropTree("SubFile", createPTree("SubFile"));
+                IPropertyTree *fileTree = queryTree->addPropTree("File", createPTree("File"));
                 fileTree->setProp("@lfn", lfn);
+                if (rf.getFlags() & RefFileSuper)
+                    fileTree->setPropBool("@super", true);
                 const char *fpkgid = rf.queryPackageId();
                 if (fpkgid && *fpkgid)
                     fileTree->setProp("@pkgid", fpkgid);
@@ -368,7 +370,7 @@ IPropertyTreeIterator *QueryFilesInUse::findQueriesUsingFile(const char *target,
     if (!targetTree)
         return NULL;
 
-    VStringBuffer xpath("Query[SubFile/@lfn='%s']", lfn);
+    VStringBuffer xpath("Query[File/@lfn='%s']", lfn);
     return targetTree->getElements(xpath);
 }
 
@@ -1295,7 +1297,7 @@ bool CWsWorkunitsEx::onWUListQueriesUsingFile(IEspContext &context, IEspWUListQu
             Owned<IEspQueryUsingFile> q = createQueryUsingFile();
             q->setId(query.queryProp("@id"));
 
-            VStringBuffer xpath("SubFile[@lfn='%s']/@pkgid", lfn.str());
+            VStringBuffer xpath("File[@lfn='%s']/@pkgid", lfn.str());
             if (query.hasProp(xpath))
                 q->setPackage(query.queryProp(xpath));
             respQueries.append(*q.getClear());

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -114,6 +114,12 @@ public:
         CriticalBlock b(crit);
     }
     IPropertyTreeIterator *findQueriesUsingFile(const char *target, const char *lfn);
+    StringBuffer &toStr(StringBuffer &s)
+    {
+        CriticalBlock b(crit);
+        return toXML(tree, s);
+    }
+
 };
 
 class QueryFilesInUseUpdateThread : public Thread
@@ -246,6 +252,7 @@ private:
     WUSchedule m_sched;
     unsigned short port;
     Owned<IPropertyTree> directories;
+public:
     QueryFilesInUse filesInUse;
 };
 
@@ -254,6 +261,7 @@ class CWsWorkunitsSoapBindingEx : public CWsWorkunitsSoapBinding
 public:
     CWsWorkunitsSoapBindingEx(IPropertyTree *cfg, const char *name, const char *process, http_soap_log_level llevel) : CWsWorkunitsSoapBinding(cfg, name, process, llevel)
     {
+        wswService = NULL;
         VStringBuffer xpath("Software/EspProcess[@name=\"%s\"]/EspBinding[@name=\"%s\"]/BatchWatch", process, name);
         batchWatchFeaturesOnly = cfg->getPropBool(xpath.str(), false);
     }
@@ -281,15 +289,15 @@ public:
 
     virtual void addService(const char * name, const char * host, unsigned short port, IEspService & service)
     {
-        CWsWorkunitsEx* srv = dynamic_cast<CWsWorkunitsEx*>(&service);
-        if (srv)
-            srv->setPort(port);
+        wswService = dynamic_cast<CWsWorkunitsEx*>(&service);
+        if (wswService)
+            wswService->setPort(port);
         CWsWorkunitsSoapBinding::addService(name, host, port, service);
     }
 
-
 private:
     bool batchWatchFeaturesOnly;
+    CWsWorkunitsEx *wswService;
 };
 
 #endif

roxie/roxiemem/roxiemem.cpp

@@ -2854,18 +2854,11 @@ public:
 
     virtual const char *cloneVString(size32_t len, const char *str)
     {
-        if (str)
-        {
-            char *ret = (char *) allocate(len+1, 0);
-            memcpy(ret, str, len);
-            ret[len] = 0;
-            return (const char *) ret;
-        }
-        else
-        {
-            assertex(len==0);
-            return NULL;
-        }
+        //Converting a empty string to a vstring should return a real string - not NULL
+        char *ret = (char *) allocate(len+1, 0);
+        memcpy(ret, str, len);
+        ret[len] = 0;
+        return (const char *) ret;
     }
 
     virtual const char *cloneVString(const char *str)

sourcedoc.xml

@@ -201,7 +201,7 @@
                 
                 This will first do a make to ensure everything is up to date, then will 
                 create the appropriate package for your operating system, Currently supported
-                package formats are rpm (for RedHat/Centos and SuSE) and  .deb (for Debian and
+                package formats are rpm (for RedHat/Centos) and  .deb (for Debian and
                 Ubuntu). If the operating system is not one of the above, or is not recognized,
                 make package will create a tarball.
             </para>