Merge remote-tracking branch 'origin/closedown-4.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/fileview2/fvsource.cpp

@@ -43,7 +43,7 @@ inline void appendNextXpathName(StringBuffer &s, const char *&xpath)
 
 void splitXmlTagNamesFromXPath(const char *xpath, StringAttr &inner, StringAttr *outer=NULL)
 {
-    if (!xpath || !*xpath)
+    if (!xpath)
         return;
 
     StringBuffer s1;

dali/base/dadfs.cpp

@@ -1311,13 +1311,6 @@ class CDistributedFileTransaction: public CInterface, implements IDistributedFil
                     }
                 }
             }
-            SuperHashIteratorOf<HTMapping> iter(subFilesByName);
-            ForEach(iter)
-            {
-                HTMapping &map = iter.query();
-                PROGLOG("subfile: %s", map.queryFindString());
-            }
-
             return false;
         }
         const void *queryFindParam() const { return &file; }
@@ -4792,11 +4785,12 @@ protected:
         unsigned n = root->getPropInt("@numsubfiles");
         if (n == 0)
             return;
-        try {
+        try
+        {
             // Find all reported indexes and bail on bad range (before we lock any file)
             Owned<IPropertyTreeIterator> subit = root->getElements("SubFile");
-            OwnedMalloc<unsigned> subFiles(n, true);
-            unsigned expectedN = 1;
+            // Adding a sub 'before' another get the list out of order (but still valid)
+            OwnedMalloc<IPropertyTree *> orderedSubFiles(n, true);
             ForEach (*subit)
             {
                 IPropertyTree &sub = subit->query();
@@ -4805,23 +4799,19 @@ protected:
                     ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: bad subfile part number %d of %d", logicalName.get(), sn, n);
                 if (sn > n)
                     ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: out-of-range subfile part number %d of %d", logicalName.get(), sn, n);
-                if (subFiles[sn-1])
+                if (orderedSubFiles[sn-1])
                     ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: duplicated subfile part number %d of %d", logicalName.get(), sn, n);
-                if (sn != expectedN)
-                    ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: bad part number %d, expected %d", logicalName.get(), sn, expectedN);
-                subFiles[sn-1] = 1;
-                ++expectedN;
+                orderedSubFiles[sn-1] = &sub;
             }
             for (unsigned i=0; i<n; i++)
             {
-                if (!subFiles[i])
+                if (!orderedSubFiles[i])
                     ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: missing subfile part number %d of %d", logicalName.get(), i+1, n);
             }
-
             // Now try to resolve them all (file/superfile)
-            ForEach (*subit)
+            for (unsigned f=0; f<n; f++)
             {
-                IPropertyTree &sub = subit->query();
+                IPropertyTree &sub = *(orderedSubFiles[f]);
                 sub.getProp("@name",subname.clear());
                 Owned<IDistributedFile> subfile;
                 subfile.setown(transaction?transaction->lookupFile(subname.str(),timeout):parent->lookup(subname.str(), udesc, false, false, transaction, timeout));
@@ -4852,7 +4842,8 @@ protected:
                 root->setPropInt("@numsubfiles", subfiles.ordinality());
             }
         }
-        catch (IException *) {
+        catch (IException *)
+        {
             partscache.kill();
             subfiles.kill();    // one out, all out
             throw;
@@ -5094,40 +5085,55 @@ public:
         CriticalBlock block (sect);
         if (subfiles.ordinality()==1)
             return subfiles.item(0).getFileDescriptor(clustername);
-        // superfiles assume consistant replication
+        // superfiles assume consistant replication & compression
         UnsignedArray subcounts;  
         bool mixedwidth = false;
+        unsigned width = 0;
+        bool first = true;
         Owned<IPropertyTree> at = getEmptyAttr();
-        if (subfiles.ordinality()!=0) {
-            Owned<IAttributeIterator> ait = subfiles.item(0).queryAttributes().getAttributes();
-            ForEach(*ait) {
-                const char *name = ait->queryName();
-                if ((stricmp(name,"@size")!=0)&&(stricmp(name,"@recordCount")!=0)) {
-                    const char *v = ait->queryValue();
-                    if (!v)
-                        continue;
-                    bool ok = true;
-                    for (unsigned i=1;i<subfiles.ordinality();i++) {
-                        const char *p = subfiles.item(i).queryAttributes().queryProp(name);
-                        if (!p||(strcmp(p,v)!=0)) {
-                            ok = false;
-                            break;
+        Owned<IDistributedFileIterator> fiter = getSubFileIterator(true);
+        ForEach(*fiter)
+        {
+            IDistributedFile &file = fiter->query();
+            if (first)
+            {
+                first = false;
+                Owned<IAttributeIterator> ait = file.queryAttributes().getAttributes();
+                ForEach(*ait)
+                {
+                    const char *name = ait->queryName();
+                    if ((stricmp(name,"@size")!=0)&&(stricmp(name,"@recordCount")!=0))
+                    {
+                        const char *v = ait->queryValue();
+                        if (!v)
+                            continue;
+                        bool ok = true;
+                        // add attributes that are common
+                        for (unsigned i=1;i<subfiles.ordinality();i++)
+                        {
+                            IDistributedFile &file = subfiles.item(i);
+                            IDistributedSuperFile *sFile = file.querySuperFile();
+                            if (!sFile || sFile->numSubFiles(true)) // skip empty super files
+                            {
+                                const char *p = file.queryAttributes().queryProp(name);
+                                if (!p||(strcmp(p,v)!=0))
+                                {
+                                    ok = false;
+                                    break;
+                                }
+                            }
                         }
+                        if (ok)
+                            at->setProp(name,v);
                     }
-                    if (ok)
-                        at->setProp(name,v);
                 }
             }
-            unsigned width = 0;
-            Owned<IDistributedFileIterator> fiter = getSubFileIterator(true);
-            ForEach(*fiter) {
-                unsigned np = fiter->query().numParts();
-                if (width) 
-                    width = np;
-                else if (np!=width) 
-                    mixedwidth = true;
-                subcounts.append(np);
-            }
+            unsigned np = file.numParts();
+            if (width)
+                width = np;
+            else if (np!=width)
+                mixedwidth = true;
+            subcounts.append(np);
         }
 
         // need common attributes

dali/daliadmin/daliadmin.cpp

@@ -1153,7 +1153,6 @@ static void checksuperfile(const char *lfn,bool fix=false)
         root->setPropInt("@numsubfiles",subnum);
     i = 0;
     byte fixstate = 0;
-    bool outOfSequence = false;
     loop {
         bool err = false;
         IPropertyTree *sub = root->queryPropTree(path.clear().appendf("SubFile[%d]",i+1).str());
@@ -1172,13 +1171,6 @@ static void checksuperfile(const char *lfn,bool fix=false)
                     i--;
                 }
             }
-            else if (pn != i+1) {
-                if (!outOfSequence)
-                    ERRLOG("SuperFile %s: corrupt, subfile file part @num values out of sequence, starting at part: %d", lname.get(), pn);
-                if (fix && (outOfSequence || doFix()))
-                    sub->setPropInt("@num", i+1);
-                outOfSequence = true;
-            }
         }
         else
             break;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -13240,7 +13240,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityAggregate(BuildCtx & ctx, IHql
             doBuildAggregateFirstFunc(instance->startctx, expr);
             doBuildAggregateNextFunc(instance->startctx, expr);
 
-            if (targetThor() && !isGrouped(dataset) && !expr->hasAttribute(localAtom))
+            if (targetThor() && !isGroupedActivity(expr) && !expr->hasAttribute(localAtom))
                 doBuildAggregateMergeFunc(instance->startctx, expr, requiresOrderedMerge);
         }
     }

esp/eclwatch/ws_XSLT/index.xslt

@@ -540,7 +540,7 @@
                         <tr>
                             <td align="left">
                                 <xsl:choose>
-                                    <xsl:when test="SuperUser">
+                                    <xsl:when test="SuperUser=1">
                                         <A href="javascript:void(0)" onclick="SetBanner();">
                                             <h3>Existing Activity on Servers:</h3>
                                         </A>
@@ -651,7 +651,7 @@
                     </xsl:for-each>
                     <xsl:apply-templates select="DFUJobs"/>
                 </form>
-                <xsl:if test="SuperUser">
+                <xsl:if test="SuperUser=1">
                     <span id="SetBannerFrame"   style="display:none; visibility:hidden">
                         <form id="SetBannerForm">
                             <table>

esp/files/scripts/WUDetailsWidget.js

@@ -99,6 +99,9 @@ define([
         buildVersion: null,
         espIPAddress: null,
         thorIPAddress: null,
+        zapDescription: null,
+        warnHistory: null,
+        warnTimings: null,
 
         prevState: "",
 
@@ -116,6 +119,9 @@ define([
             this.playgroundWidget = registry.byId(this.id + "_Playground");
             this.xmlWidget = registry.byId(this.id + "_XML");
             this.publishForm = registry.byId(this.id + "PublishForm");
+            this.zapDescription = registry.byId(this.id + "ZapDescription");
+            this.warnHistory = registry.byId(this.id + "WarnHistory");
+            this.warnTimings = registry.byId(this.id + "WarnTimings");
 
             this.infoGridWidget = registry.byId(this.id + "InfoContainer");
             this.zapDialog = registry.byId(this.id + "ZapDialog");
@@ -214,7 +220,7 @@ define([
 
         onZapSubmit: function (event) {
             var frame = iframe.create("ZapDownload" + uniqueID++);
-            var url = ESPRequest.getBaseURL("WsWorkunits") + "/WUCreateZAPInfo?WUID=" + this.wu.Wuid + "&ESPIPAddress=" + this.espIPAddress + "&ThorIPAddress=" + this.thorIPAddress + "&BuildVersion=" + encodeURIComponent(this.buildVersion);
+            var url = ESPRequest.getBaseURL("WsWorkunits") + "/WUCreateZAPInfo?WUID=" + this.wu.Wuid + "&ESPIPAddress=" + this.espIPAddress + "&ThorIPAddress=" + this.thorIPAddress + "&BuildVersion=" + encodeURIComponent(this.buildVersion) + "&ProblemDescription=" + encodeURIComponent(this.zapDescription.value) + "&WhatChanged=" + encodeURIComponent(this.warnHistory.value) + "&WhereSlow=" + encodeURIComponent(this.warnTimings.value);
             iframe.setSrc(frame, url, true);
             this.zapDialog.hide();
         },

plugins/pyembed/pyembed.cpp

@@ -236,6 +236,7 @@ static class Python27GlobalState
 public:
     Python27GlobalState()
     {
+        pythonLibrary = (HINSTANCE) 0;
 #ifndef _WIN32
         // If Py_Initialize is called when stdin is set to a directory, it calls exit()
         // We don't want that to happen - just disable Python support in such situations
@@ -246,6 +247,33 @@ public:
             return;
         }
 #endif
+#ifndef _WIN32
+        // We need to ensure all symbols in the python2.6 so are loaded - due to bugs in some distro's python installations
+        FILE *diskfp = fopen("/proc/self/maps", "r");
+        if (diskfp)
+        {
+            char ln[_MAX_PATH];
+            while (fgets(ln, sizeof(ln), diskfp))
+            {
+                if (strstr(ln, "libpython2"))
+                {
+                    const char *fullName = strchr(ln, '/');
+                    if (fullName)
+                    {
+                        char * lf = (char *) strchr(fullName, '\n');
+                        if (lf)
+                        {
+                            *lf = 0;
+                            pythonLibrary = dlopen((char *)fullName, RTLD_NOW|RTLD_GLOBAL);
+//                            DBGLOG("dlopen %s returns %"I64F"x", fullName, (__uint64) pythonLibrary);
+                            break;
+                        }
+                    }
+                }
+            }
+            fclose(diskfp);
+        }
+#endif
         // Initialize the Python Interpreter
         Py_Initialize();
         PyEval_InitThreads();
@@ -263,6 +291,8 @@ public:
             // Finish the Python Interpreter
             Py_Finalize();
         }
+        if (pythonLibrary)
+            FreeSharedObject(pythonLibrary);
     }
     bool isInitialized()
     {
@@ -271,6 +301,7 @@ public:
 protected:
     PyThreadState *tstate;
     bool initialized;
+    HINSTANCE pythonLibrary;
 } globalState;
 
 // Each call to a Python function will use a new Python27EmbedFunctionContext object

testing/ecl/persist_replicate.ecl

@@ -39,7 +39,7 @@ small_rec trans(rec r) := TRANSFORM
 END;
 
 // In Thor, this created a file that could not be replicated
-ds2 := PROJECT(ds, trans(LEFT)) : PERSIST('~temp::PersistReplicate');
+ds2 := PROJECT(ds, trans(LEFT)) : PERSIST('~temp::PersistReplicate', SINGLE);
 OUTPUT(ds2);
 
 // Now, replicate

thorlcr/thorutil/thmem.cpp

@@ -207,7 +207,6 @@ public:
     {
         rows.swap(inRows);
         useCompression = false;
-        activity.queryJob().queryRowManager()->addRowBuffer(this);
     }
     ~CSpillableStreamBase()
     {
@@ -292,6 +291,7 @@ public:
     CSharedSpillableRowSet(CActivityBase &_activity, CThorSpillableRowArray &inRows, IRowInterfaces *_rowIf, bool _preserveNulls)
         : CSpillableStreamBase(_activity, inRows, _rowIf, _preserveNulls)
     {
+        activity.queryJob().queryRowManager()->addRowBuffer(this);
     }
     IRowStream *createRowStream()
     {
@@ -329,6 +329,7 @@ public:
         // a small amount of rows to read from swappable rows
         roxiemem::IRowManager *rowManager = activity.queryJob().queryRowManager();
         readRows = static_cast<const void * *>(rowManager->allocate(granularity * sizeof(void*), activity.queryContainer().queryId()));
+        activity.queryJob().queryRowManager()->addRowBuffer(this);
     }
     ~CSpillableStream()
     {