Merge remote-tracking branch 'origin/closedown-4.2.x'

Conflicts:
	ecl/hql/hqlopt.cpp

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorsoapcall.cpp

@@ -217,12 +217,13 @@ typedef IArrayOf<Url> UrlArray;
 
 //=================================================================================================
 
+//MORE: This whole class should really be replaced with a single function, avoiding at least one string clone.
 class UrlListParser
 {
 public:
     UrlListParser(const char * text)
     {
-        fullText = strdup(text);
+        fullText = text ? strdup(text) : NULL;
     }
 
     ~UrlListParser()
@@ -232,17 +233,20 @@ public:
 
     unsigned getUrls(UrlArray &array)
     {
-        char *copyFullText = strdup(fullText);
-
-        char *saveptr;
-        char *url = strtok_r(copyFullText, "|", &saveptr);
-        while (url != NULL)
+        if (fullText)
         {
-            array.append(*new Url(url));
-            url = strtok_r(NULL, "|", &saveptr);
-        }
+            char *copyFullText = strdup(fullText);
+
+            char *saveptr;
+            char *url = strtok_r(copyFullText, "|", &saveptr);
+            while (url != NULL)
+            {
+                array.append(*new Url(url));
+                url = strtok_r(NULL, "|", &saveptr);
+            }
 
-        free(copyFullText);
+            free(copyFullText);
+        }
         return array.ordinality();
     }
 

common/workunit/referencedfilelist.cpp

@@ -241,6 +241,7 @@ void ReferencedFile::processLocalFileInfo(IDistributedFile *df, const char *dstC
     }
     else
     {
+        flags |= RefSubFile;
         if (!dstCluster || !*dstCluster)
             return;
         if (df->findCluster(dstCluster)==NotFound)

common/workunit/workunit.cpp

@@ -4562,7 +4562,7 @@ extern WORKUNIT_API IStringIterator *getTargetClusters(const char *processType,
     {
         StringBuffer xpath;
         xpath.appendf("%s", processType ? processType : "*");
-        if (processName)
+        if (processName && *processName)
             xpath.appendf("[@process=\"%s\"]", processName);
         Owned<IPropertyTreeIterator> targets = conn->queryRoot()->getElements("Software/Topology/Cluster");
         ForEach(*targets)

dali/sasha/saxref.cpp

@@ -2308,7 +2308,7 @@ public:
                 {
                     if (0 == expireDays)
                     {
-                        bool isPersist = attr.queryProp("@persist");
+                        bool isPersist = attr.getPropBool("@persistent");
                         expireDays = isPersist ? defaultPersistExpireDays : defaultExpireDays;
                     }
                     CDateTime now;

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -124,7 +124,7 @@
           url="http://hpccsystems.com/download/free-community-edition">http://hpccsystems.com/download/free-community-edition</ulink>
           and install.</para>
 
-          <para>On CentOS/Red Hat/SuSe:</para>
+          <para>On CentOS/Red Hat:</para>
 
           <para><programlisting> sudo rpm -Uvh &lt;rpm file name&gt;</programlisting>On
           Ubuntu/Debian:</para>
@@ -135,7 +135,7 @@
         <listitem>
           <para>Start your HPCC System.</para>
 
-          <para>On CentOS/Red Hat/SuSe:</para>
+          <para>On CentOS/Red Hat:</para>
 
           <para><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
@@ -549,10 +549,6 @@
             </listitem>
 
             <listitem>
-              <para>64-bit openSUSE 11.3 or 11.4</para>
-            </listitem>
-
-            <listitem>
               <para>64-bit Debian 6.x (Squeeze)</para>
             </listitem>
           </itemizedlist></para>
@@ -758,7 +754,7 @@
         instructions:</para>
 
         <sect3 id="Installing_CentOS-RedHat-Suse">
-          <title>CentOS/Red Hat/SuSe</title>
+          <title>CentOS/Red Hat</title>
 
           <para>Install RPM with the -Uvh switch.</para>
 
@@ -768,18 +764,6 @@
 
           <programlisting>sudo rpm -Uvh &lt;rpm file name&gt;</programlisting>
 
-          <blockquote>
-            <para><emphasis role="bold">Note:</emphasis> For ANY version of
-            SuSe you must set a password for the hpcc user on all nodes. One
-            way to do do this is to issue the following command:</para>
-
-            <para><programlisting>sudo passwd hpcc</programlisting>Be sure to
-            set the password on ALL nodes.</para>
-
-            <para><emphasis role="bold">Note:</emphasis> For CentOS
-            installations, the Fedora EPEL repository is required.</para>
-          </blockquote>
-
           <sect4 id="Optional_Plug-ins">
             <title>Optional Plug-ins</title>
 
@@ -850,7 +834,7 @@
             <para>Start the system using the default configuration.</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1284,7 +1268,7 @@
             command:</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1309,7 +1293,7 @@
 
                       <entry>You can use this command to confirm HPCC
                       processes are stopped (on Centos/Red
-                      Hat/SuSe):<para><programlisting>sudo /sbin/service hpcc-init status</programlisting><phrase>For
+                      Hat):<para><programlisting>sudo /sbin/service hpcc-init status</programlisting><phrase>For
                       Ubuntu</phrase><programlisting>sudo service hpcc-init status
 </programlisting><phrase>For Debian 6 (Squeeze)</phrase><programlisting>sudo /etc/init.d/hpcc-init status</programlisting></para></entry>
                     </row>
@@ -1624,7 +1608,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
             the HPCC system on an individual node:</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1682,7 +1666,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <title>To start the system:</title>
 
           <para><emphasis role="bold">Centos/Red
-          Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
+          Hat</emphasis><programlisting>sudo /sbin/service hpcc-init start</programlisting></para>
 
           <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1697,7 +1681,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           <title>To stop the system:</title>
 
           <para><emphasis role="bold">Centos/Red
-          Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
+          Hat</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
 
           <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1737,7 +1721,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
           in the init system as follows.</para>
 
           <para><emphasis role="bold">Centos/Red
-          Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init -c &lt;component name&gt; &lt;command&gt;</programlisting></para>
+          Hat</emphasis><programlisting>sudo /sbin/service hpcc-init -c &lt;component name&gt; &lt;command&gt;</programlisting></para>
 
           <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -1783,7 +1767,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
             command on <emphasis role="bold">every</emphasis> node:</para>
 
             <para><emphasis role="bold">Centos/Red
-            Hat/SuSe</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
+            Hat</emphasis><programlisting>sudo /sbin/service hpcc-init stop</programlisting></para>
 
             <para><emphasis role="bold">Ubuntu</emphasis></para>
 
@@ -2872,7 +2856,7 @@ OUTPUT(ValidWords)
       your system. If necessary, do so on each node that it is installed
       on.</para>
 
-      <para><emphasis role="bold">Centos/Red Hat/SuSe</emphasis></para>
+      <para><emphasis role="bold">Centos/Red Hat</emphasis></para>
 
       <programlisting>sudo rpm -e hpccsystems-platform </programlisting>
 
@@ -3112,7 +3096,7 @@ sudo /sbin/service hpcc-init -c esp start
       <para><emphasis role="bold">RPM-based systems:</emphasis></para>
 
       <para>If you are interested in using external languages for RPM-based
-      systems (CentOS/Red Hat/SuSe), you need to download and install the
+      systems (CentOS/Red Hat), you need to download and install the
       appropriate platform installation distribution <emphasis
       role="bold">with plug-ins</emphasis> option from the downloads
       site.</para>
@@ -3285,8 +3269,8 @@ add1(10);
             <para>JavaScript support is available for CentOS 6.x or later (not
             available for CentOS 5.x).</para>
 
-            <para>On an RPM-based system (CentOS/Red Hat/SuSe) install
-            <emphasis role="bold">v8embed.</emphasis></para>
+            <para>On an RPM-based system (CentOS/Red Hat) install <emphasis
+            role="bold">v8embed.</emphasis></para>
 
             <para><emphasis role="bold">Debian-based
             systems:</emphasis></para>
@@ -3403,7 +3387,7 @@ split_words('Once upon a time');
 
             <para><emphasis role="bold">RPM-based systems:</emphasis></para>
 
-            <para>On an RPM-based system (CentOS/Red Hat/SuSe) use <emphasis
+            <para>On an RPM-based system (CentOS/Red Hat) use <emphasis
             role="bold">R-core</emphasis> and <emphasis
             role="bold">R-core-devel</emphasis></para>
 

ecl/eclcc/eclcc.cpp

@@ -275,7 +275,6 @@ public:
     void processBatchedFile(IFile & file, bool multiThreaded);
 
     virtual void noteCluster(const char *clusterName);
-    virtual void registerFile(const char * filename, const char * description);
     virtual bool allowAccess(const char * category);
 
 protected:
@@ -1705,9 +1704,6 @@ bool EclCompileInstance::reportErrorSummary()
 void EclCC::noteCluster(const char *clusterName)
 {
 }
-void EclCC::registerFile(const char * filename, const char * description)
-{
-}
 bool EclCC::allowAccess(const char * category)
 {
     ForEachItemIn(idx1, deniedPermissions)

ecl/hqlcpp/hqlcpp.cpp

@@ -1271,8 +1271,6 @@ void HqlCppInstance::addHint(const char * hintXml, ICodegenContextCallback * ctx
 
         Owned<IWUQuery> query = workunit->updateQuery();
         associateLocalFile(query, FileTypeCpp, hintFilename.str(), "Hints", 0);
-
-        ctxCallback->registerFile(hintFilename.str(), "Hints");
     }
     appendHintText(hintXml);
 }
@@ -1340,7 +1338,6 @@ void HqlCppInstance::flushResources(const char *filename, ICodegenContextCallbac
         {
             Owned<IWUQuery> query = workunit->updateQuery();
             associateLocalFile(query, FileTypeHintXml, resTextName, "Workunit resource text", 0);
-            ctxCallback->registerFile(resTextName, "Workunit Res.txt");
         }
         useLibrary(filename);
     }

ecl/hqlcpp/hqlcpp.hpp

@@ -87,7 +87,6 @@ The following debug options are currently supported by the code generator:
 interface ICodegenContextCallback : public IInterface
 {
     virtual void noteCluster(const char *clusterName) = 0;
-    virtual void registerFile(const char * filename, const char * description) = 0;
     virtual bool allowAccess(const char * category) = 0;
 };
 

ecl/hqlcpp/hqlecl.cpp

@@ -53,7 +53,6 @@ class NullContextCallback : public CInterface, implements ICodegenContextCallbac
     IMPLEMENT_IINTERFACE
 
     virtual void noteCluster(const char *clusterName) {}
-    virtual void registerFile(const char * filename, const char * description) {}
     virtual bool allowAccess(const char * category) { return true; }
 };
 
@@ -135,7 +134,6 @@ void HqlDllGenerator::addCppName(const char * filename)
     {
         Owned<IWUQuery> query = wu->updateQuery();
         associateLocalFile(query, FileTypeCpp, filename, pathTail(filename), 0);
-        ctxCallback->registerFile(filename, "Workunit CPP");
     }
 }
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -5982,8 +5982,6 @@ void HqlCppTranslator::writeFieldUsage(const char * targetDir, IPropertyTree * s
 
         Owned<IWUQuery> query = wu()->updateQuery();
         associateLocalFile(query, FileTypeXml, fullname, "FieldUsage", 0);
-
-        ctxCallback->registerFile(fullname.str(), "FieldUsage");
     }
 }
 
@@ -16948,19 +16946,19 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySOAP(BuildCtx & ctx, IHqlExpre
     }
 
     //virtual unsigned numParallelThreads()
-    doBuildUnsignedFunction(instance->classctx, "numParallelThreads", queryAttributeChild(expr, parallelAtom, 0));
+    doBuildUnsignedFunction(instance->startctx, "numParallelThreads", queryAttributeChild(expr, parallelAtom, 0));
 
     //virtual unsigned numRecordsPerBatch()
-    doBuildUnsignedFunction(instance->classctx, "numRecordsPerBatch", queryAttributeChild(expr, mergeAtom, 0));
+    doBuildUnsignedFunction(instance->startctx, "numRecordsPerBatch", queryAttributeChild(expr, mergeAtom, 0));
 
     //virtual int numRetries()
-    doBuildSignedFunction(instance->classctx, "numRetries", queryAttributeChild(expr, retryAtom, 0));
+    doBuildSignedFunction(instance->startctx, "numRetries", queryAttributeChild(expr, retryAtom, 0));
 
     //virtual double getTimeout()
-    doBuildDoubleFunction(instance->classctx, "getTimeout", queryAttributeChild(expr, timeoutAtom, 0));
+    doBuildDoubleFunction(instance->startctx, "getTimeout", queryAttributeChild(expr, timeoutAtom, 0));
 
     //virtual double getTimeLimit()
-    doBuildDoubleFunction(instance->classctx, "getTimeLimit", queryAttributeChild(expr, timeLimitAtom, 0));
+    doBuildDoubleFunction(instance->startctx, "getTimeLimit", queryAttributeChild(expr, timeLimitAtom, 0));
 
     if (namespaceAttr)
     {

esp/eclwatch/ws_XSLT/CMakeLists.txt

@@ -77,6 +77,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/queryfiledetails.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/queryfilelist.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/queryfilelistdone.xslt
+    ${CMAKE_CURRENT_SOURCE_DIR}/QueriesUsingFile.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/result.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/result_lib.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/result_lib1.xslt

esp/eclwatch/ws_XSLT/dfu.xslt

@@ -142,7 +142,7 @@
                 ]);
             }
 
-            if (roxiecluster != 0) {
+            if (roxiecluster != 0 || cluster=="") {
                 oMenu.addItems([
                     { text: "Used By", onclick: { fn: queriesUsingDFUFile } }
                 ]);

esp/eclwatch/ws_XSLT/wuid.xslt

@@ -650,14 +650,16 @@
 
                 var exceptionHtml = "Workunit successfully published.";
                 var headerText = "Workunit Published";
-                var i = o.responseText.indexOf('<h3>Exception');
+                var hasException = 0;
+                var i = o.responseText.indexOf('<Exception>');
                 if (i > -1) {
+                    hasException = 1;
                     headerText = "Error Publishing Workunit";
-                    var j = o.responseText.indexOf('<table');
+                    var j = o.responseText.indexOf('<Message>');
                     if (j > -1) {
-                        var k = o.responseText.indexOf('</table>');
+                        var k = o.responseText.indexOf('</Message>');
                         if (k > -1) {
-                            exceptionHtml = o.responseText.substring(j, k+8);
+                            exceptionHtml = o.responseText.substring(j+9, k);
                         }
                     }
                 } else {
@@ -690,7 +692,7 @@
                 }
                 var publishDialog =
                      new YAHOO.widget.SimpleDialog("publishDialog",
-                      { width: "300px",
+                      { width: hasException ? null : "300px",
                         fixedcenter: true,
                         visible: false,
                         draggable: false,

esp/scm/ws_workunits.ecm

@@ -1115,7 +1115,7 @@ ESPrequest [nil_remove] WUPublishWorkunitRequest
     string Comment;
     bool DontCopyFiles(false);
     string SourceProcess;
-    bool AllowForeignFiles(true);
+    bool AllowForeignFiles(false);
 };
 
 ESPresponse [exceptions_inline] WUPublishWorkunitResponse

esp/services/ws_workunits/ws_workunitsAuditLogs.cpp

@@ -1470,6 +1470,17 @@ int CWsWorkunitsSoapBindingEx::onGet(CHttpRequest* request, CHttpResponse* respo
             streamJobQueueListResponse(*ctx, cluster, startDate, endDate, response, xls.str());
             return 0;
         }
+        else if(!strnicmp(path.str(), "/WsWorkunits/filesInUse", 28))
+        {
+            StringBuffer xml;
+            wswService->filesInUse.toStr(xml);
+
+            response->setContent(xml);
+            response->setContentType("text/xml");
+            response->setStatus(HTTP_STATUS_OK);
+            response->send();
+            return 0;
+        }
     }
     catch(IException* e)
     {

esp/services/ws_workunits/ws_workunitsQuerySets.cpp

@@ -328,16 +328,18 @@ void QueryFilesInUse::loadTarget(IPropertyTree *t, const char *target, unsigned
             if (aborting)
                 return;
             IReferencedFile &rf = files->query();
-            if (!(rf.getFlags() & RefSubFile))
-                continue;
+            //if (!(rf.getFlags() & RefSubFile))
+            //    continue;
             const char *lfn = rf.getLogicalName();
             if (!lfn || !*lfn)
                 continue;
 
-            if (!queryTree->hasProp(xpath.setf("SubFile[@lfn='%s']", lfn)))
+            if (!queryTree->hasProp(xpath.setf("File[@lfn='%s']", lfn)))
             {
-                IPropertyTree *fileTree = queryTree->addPropTree("SubFile", createPTree("SubFile"));
+                IPropertyTree *fileTree = queryTree->addPropTree("File", createPTree("File"));
                 fileTree->setProp("@lfn", lfn);
+                if (rf.getFlags() & RefFileSuper)
+                    fileTree->setPropBool("@super", true);
                 const char *fpkgid = rf.queryPackageId();
                 if (fpkgid && *fpkgid)
                     fileTree->setProp("@pkgid", fpkgid);
@@ -368,7 +370,7 @@ IPropertyTreeIterator *QueryFilesInUse::findQueriesUsingFile(const char *target,
     if (!targetTree)
         return NULL;
 
-    VStringBuffer xpath("Query[SubFile/@lfn='%s']", lfn);
+    VStringBuffer xpath("Query[File/@lfn='%s']", lfn);
     return targetTree->getElements(xpath);
 }
 
@@ -1288,7 +1290,7 @@ bool CWsWorkunitsEx::onWUListQueriesUsingFile(IEspContext &context, IEspWUListQu
             Owned<IEspQueryUsingFile> q = createQueryUsingFile();
             q->setId(query.queryProp("@id"));
 
-            VStringBuffer xpath("SubFile[@lfn='%s']/@pkgid", lfn.str());
+            VStringBuffer xpath("File[@lfn='%s']/@pkgid", lfn.str());
             if (query.hasProp(xpath))
                 q->setPackage(query.queryProp(xpath));
             respQueries.append(*q.getClear());

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -114,6 +114,12 @@ public:
         CriticalBlock b(crit);
     }
     IPropertyTreeIterator *findQueriesUsingFile(const char *target, const char *lfn);
+    StringBuffer &toStr(StringBuffer &s)
+    {
+        CriticalBlock b(crit);
+        return toXML(tree, s);
+    }
+
 };
 
 class QueryFilesInUseUpdateThread : public Thread
@@ -246,6 +252,7 @@ private:
     WUSchedule m_sched;
     unsigned short port;
     Owned<IPropertyTree> directories;
+public:
     QueryFilesInUse filesInUse;
 };
 
@@ -254,6 +261,7 @@ class CWsWorkunitsSoapBindingEx : public CWsWorkunitsSoapBinding
 public:
     CWsWorkunitsSoapBindingEx(IPropertyTree *cfg, const char *name, const char *process, http_soap_log_level llevel) : CWsWorkunitsSoapBinding(cfg, name, process, llevel)
     {
+        wswService = NULL;
         VStringBuffer xpath("Software/EspProcess[@name=\"%s\"]/EspBinding[@name=\"%s\"]/BatchWatch", process, name);
         batchWatchFeaturesOnly = cfg->getPropBool(xpath.str(), false);
     }
@@ -281,15 +289,15 @@ public:
 
     virtual void addService(const char * name, const char * host, unsigned short port, IEspService & service)
     {
-        CWsWorkunitsEx* srv = dynamic_cast<CWsWorkunitsEx*>(&service);
-        if (srv)
-            srv->setPort(port);
+        wswService = dynamic_cast<CWsWorkunitsEx*>(&service);
+        if (wswService)
+            wswService->setPort(port);
         CWsWorkunitsSoapBinding::addService(name, host, port, service);
     }
 
-
 private:
     bool batchWatchFeaturesOnly;
+    CWsWorkunitsEx *wswService;
 };
 
 #endif

roxie/roxiemem/roxiemem.cpp

@@ -2910,18 +2910,11 @@ public:
 
     virtual const char *cloneVString(size32_t len, const char *str)
     {
-        if (str)
-        {
-            char *ret = (char *) allocate(len+1, 0);
-            memcpy(ret, str, len);
-            ret[len] = 0;
-            return (const char *) ret;
-        }
-        else
-        {
-            assertex(len==0);
-            return NULL;
-        }
+        //Converting a empty string to a vstring should return a real string - not NULL
+        char *ret = (char *) allocate(len+1, 0);
+        memcpy(ret, str, len);
+        ret[len] = 0;
+        return (const char *) ret;
     }
 
     virtual const char *cloneVString(const char *str)

sourcedoc.xml

@@ -190,7 +190,7 @@
                 
                 This will first do a make to ensure everything is up to date, then will 
                 create the appropriate package for your operating system, Currently supported
-                package formats are rpm (for RedHat/Centos and SuSE) and  .deb (for Debian and
+                package formats are rpm (for RedHat/Centos) and  .deb (for Debian and
                 Ubuntu). If the operating system is not one of the above, or is not recognized,
                 make package will create a tarball.
             </para>