Merge remote-tracking branch 'origin/closedown-4.2.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/dependencies/saucy.cmake

@@ -1 +1 @@
-set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.53.0, libicu48, libxalanc111, libxerces-c3.1, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync")
+set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex1.53.0, libicu48, libxalan-c111, libxerces-c3.1, binutils, libldap-2.4-2, openssl, zlib1g, g++, openssh-client, openssh-server, expect, libarchive13, rsync")

dali/base/dadfs.cpp

@@ -2805,6 +2805,24 @@ public:
                     queryLogicalName(), superRepO);
     }
 
+    virtual void getSuperOwners(StringArray &owners)
+    {
+        if (root)
+        {
+            StringBuffer owner;
+            Owned<IPropertyTreeIterator> iter = root->getElements("SuperOwner");
+            ForEach (*iter)
+            {
+                iter->query().getProp("@name", owner.clear());
+                if (owner.length())
+                {
+                    if (NotFound == owners.find(owner))
+                        owners.append(owner);
+                }
+            }
+        }
+    }
+
     void linkSuperOwner(const char *superfile,bool link)
     {
         if (!superfile||!*superfile)
@@ -4495,7 +4513,87 @@ class CDistributedSuperFile: public CDistributedFileBase<IDistributedSuperFile>
                     }
                 }
                 // Now we clean the subs
-                sf->doRemoveSubFile(subfile.get(),transaction);
+                if (subfile.get())
+                    sf->doRemoveSubFile(subfile.get(), transaction);
+                else
+                    sf->doRemoveSubFiles(transaction);
+            }
+        }
+    };
+
+    /**
+     * Removes all subfiles exclusively owned by named superfile within a transaction.
+     */
+    class cRemoveOwnedSubFilesAction: public CDFAction
+    {
+        StringAttr parentlname;
+        Owned<IDistributedSuperFile> parent;
+        bool remsub;
+    public:
+        cRemoveOwnedSubFilesAction(IDistributedFileTransaction *_transaction, const char *_parentlname,bool _remsub=false)
+            : parentlname(_parentlname), remsub(_remsub)
+        {
+        }
+        bool prepare()
+        {
+            parent.setown(transaction->lookupSuperFile(parentlname,true));
+            if (!parent)
+                throw MakeStringException(-1,"removeOwnedSubFiles: SuperFile %s cannot be found", parentlname.get());
+            // Try to lock all files
+            addFileLock(parent);
+            bool dirty=false;
+            if (lock(&dirty))
+            {
+                if (dirty)
+                {
+                    // in the process of previous attempt to lock for exclusive access, locks were released
+                    // need to reload to ensure position and # of files is correct
+                    CDistributedSuperFile *sf = dynamic_cast<CDistributedSuperFile *>(parent.get());
+                    if (sf)
+                        sf->loadSubFiles(transaction, SDS_TRANSACTION_RETRY);
+                }
+                return true;
+            }
+            unlock();
+            return false;
+        }
+        void run()
+        {
+            CDistributedSuperFile *sf = QUERYINTERFACE(parent.get(),CDistributedSuperFile);
+            if (sf)
+            {
+                StringArray toRemove;
+                Owned<IDistributedFileIterator> iter = parent->getSubFileIterator(false);
+                ForEach (*iter)
+                {
+                    IDistributedFile *file = &iter->query();
+                    CDistributedFile *_file = QUERYINTERFACE(file, CDistributedFile);
+                    StringArray owners;
+                    _file->getSuperOwners(owners);
+
+                    if (NotFound == owners.find(parentlname))
+                        ThrowStringException(-1, "removeOwnedSubFiles: SuperFile %s, subfile %s - subfile not owned by superfile", parentlname.get(), file->queryLogicalName());
+                    if (1 == owners.ordinality()) // just me
+                    {
+                        const char *logicalName = file->queryLogicalName();
+                        toRemove.append(logicalName);
+                        // Delay the deletion of the subs until commit
+                        if (remsub)
+                        {
+                            CDfsLogicalFileName lname;
+                            lname.set(logicalName);
+                            transaction->addDelayedDelete(lname, SDS_SUB_LOCK_TIMEOUT);
+                        }
+                    }
+                }
+                // Now we clean the subs
+                if (sf->numSubFiles(false) == toRemove.ordinality())
+                    sf->doRemoveSubFiles(transaction); // remove all
+                else
+                {
+                    ForEachItemIn(r, toRemove)
+                        sf->doRemoveSubFile(toRemove.item(r), transaction);
+                }
             }
         }
     };
@@ -4658,6 +4756,27 @@ protected:
             throw exceptions.getClear();
     }
 
+    virtual void getSuperOwners(StringArray &owners)
+    {
+        ForEachItemIn(i, subfiles)
+        {
+            IDistributedFile *file = &subfiles.item(i);
+            IDistributedSuperFile *super = file->querySuperFile();
+            if (super)
+            {
+                CDistributedSuperFile *_super = QUERYINTERFACE(super, CDistributedSuperFile);
+                if (_super)
+                    _super->getSuperOwners(owners);
+            }
+            else
+            {
+                CDistributedFile *_file = QUERYINTERFACE(file, CDistributedFile);
+                if (_file)
+                    _file->getSuperOwners(owners);
+            }
+        }
+
+    }
     static StringBuffer &getSubPath(StringBuffer &path,unsigned idx)
     {
         return path.append("SubFile[@num=\"").append(idx+1).append("\"]");
@@ -4676,8 +4795,8 @@ protected:
         try {
             // Find all reported indexes and bail on bad range (before we lock any file)
             Owned<IPropertyTreeIterator> subit = root->getElements("SubFile");
-            // Adding a sub 'before' another get the list out of order (but still valid)
             OwnedMalloc<unsigned> subFiles(n, true);
+            unsigned expectedN = 1;
             ForEach (*subit)
             {
                 IPropertyTree &sub = subit->query();
@@ -4688,7 +4807,10 @@ protected:
                     ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: out-of-range subfile part number %d of %d", logicalName.get(), sn, n);
                 if (subFiles[sn-1])
                     ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: duplicated subfile part number %d of %d", logicalName.get(), sn, n);
+                if (sn != expectedN)
+                    ThrowStringException(-1, "CDistributedSuperFile: SuperFile %s: bad part number %d, expected %d", logicalName.get(), sn, expectedN);
                 subFiles[sn-1] = 1;
+                ++expectedN;
             }
             for (unsigned i=0; i<n; i++)
             {
@@ -4761,14 +4883,13 @@ protected:
         root->setPropInt("@numsubfiles",subfiles.ordinality());
     }
 
-    void removeItem(unsigned pos, StringBuffer &subname)
+    void removeItem(unsigned pos)
     {
         partscache.kill();
         StringBuffer path;
         IPropertyTree* sub = root->queryPropTree(getSubPath(path,pos).str());
         if (!sub)
             throw MakeStringException(-1,"CDistributedSuperFile(3): Corrupt subfile file part %d cannot be found",pos+1);
-        sub->getProp("@name",subname);
         root->removeTree(sub);
         // now renumber all above
         for (unsigned i=pos+1; i<subfiles.ordinality(); i++) {
@@ -5601,56 +5722,56 @@ private:
         linkSubFile(pos, transaction);
     }
 
+    bool doRemoveSubFiles(IDistributedFileTransactionExt *transaction)
+    {
+        // have to be quite careful here
+        unsigned pos = subfiles.ordinality();
+        if (pos)
+        {
+            DistributedFilePropertyLock lock(this);
+            if (lock.needsReload())
+                loadSubFiles(transaction,1000*60*10);
+            pos = subfiles.ordinality();
+            if (pos)
+            {
+                do
+                {
+                    pos--;
+                    unlinkSubFile(pos,transaction);
+                    removeItem(pos);
+                } while (pos);
+                setModified();
+                updateFileAttrs();
+                lock.unlock();
+                updateParentFileAttrs(transaction);
+            }
+        }
+        return true;
+    }
+
     bool doRemoveSubFile(const char *subfile,
                          IDistributedFileTransactionExt *transaction)
     {
         // have to be quite careful here
-        StringAttrArray subnames;
-        unsigned pos;
-        StringBuffer subname;
-        if (subfile) {
-            unsigned pos=findSubFileOrd(subfile);
+        unsigned pos=findSubFileOrd(subfile);
+        if ((pos==NotFound)||(pos>=subfiles.ordinality()))
+            pos = findSubFile(subfile);
+        if (pos==NotFound)
+            return false;
+        {
+            DistributedFilePropertyLock lock(this);
+            // don't reload subfiles here
+            pos=findSubFileOrd(subfile);
             if ((pos==NotFound)||(pos>=subfiles.ordinality()))
                 pos = findSubFile(subfile);
             if (pos==NotFound)
                 return false;
-            {
-                DistributedFilePropertyLock lock(this);
-                // don't reload subfiles here
-                pos=findSubFileOrd(subfile);
-                if ((pos==NotFound)||(pos>=subfiles.ordinality()))
-                    pos = findSubFile(subfile);
-                if (pos==NotFound)
-                    return false;
-                unlinkSubFile(pos,transaction);
-                removeItem(pos,subname.clear());
-                subnames.append(* new StringAttrItem(subname.str()));
-                setModified();
-                updateFileAttrs();
-            }
-            updateParentFileAttrs(transaction);
-        }
-        else {
-            pos = subfiles.ordinality();
-            if (pos) {
-                DistributedFilePropertyLock lock(this);
-                if (lock.needsReload())
-                    loadSubFiles(transaction,1000*60*10);
-                pos = subfiles.ordinality();
-                if (pos) {
-                    do {
-                        pos--;
-                        unlinkSubFile(pos,transaction);
-                        removeItem(pos,subname.clear());
-                        subnames.append(* new StringAttrItem(subname.str()));
-                    } while (pos);
-                    setModified();
-                    updateFileAttrs();
-                    lock.unlock();
-                    updateParentFileAttrs(transaction);
-                }
-            }
+            unlinkSubFile(pos,transaction);
+            removeItem(pos);
+            setModified();
+            updateFileAttrs();
         }
+        updateParentFileAttrs(transaction);
         return true;
     }
 
@@ -5802,6 +5923,34 @@ public:
         return true;
     }
 
+    virtual bool removeOwnedSubFiles(bool remsub, // if true removes subfiles from DFS
+                                     IDistributedFileTransaction *transaction)
+    {
+        CriticalBlock block (sect);
+        checkModify("removeOwnedSubFiles");
+        partscache.kill();
+
+        // Create a local transaction that will be destroyed (MORE: make transaction compulsory)
+        Linked<IDistributedFileTransactionExt> localtrans;
+        if (transaction)
+        {
+            IDistributedFileTransactionExt *_transaction = dynamic_cast<IDistributedFileTransactionExt *>(transaction);
+            localtrans.set(_transaction);
+        }
+        else
+            localtrans.setown(new CDistributedFileTransaction(udesc));
+
+        // Make sure this file is in cache (reuse below)
+        localtrans->addFile(this);
+
+        cRemoveOwnedSubFilesAction *action = new cRemoveOwnedSubFilesAction(localtrans, queryLogicalName(), remsub);
+        localtrans->addAction(action); // takes ownership
+        localtrans->autoCommit();
+
+        // MORE - auto-commit will throw an exception, change this to void
+        return true;
+    }
+
     virtual bool swapSuperFile( IDistributedSuperFile *_file,
                                 IDistributedFileTransaction *transaction)
     {

dali/base/dadfs.hpp

@@ -312,6 +312,9 @@ interface IDistributedSuperFile: extends IDistributedFile
                                 bool remcontents=false,     // if true removes contents of subfile (assuming it is a superfile)
                                 IDistributedFileTransaction *transaction=NULL)=0;
                             // Note does not delete subfile
+    virtual bool removeOwnedSubFiles(bool remsub,           // if true removes subfiles from DFS
+                                     IDistributedFileTransaction *transaction=NULL)=0;
+                            // Note does not delete subfile
     virtual bool swapSuperFile( IDistributedSuperFile *_file,               // swaps sub files
                                 IDistributedFileTransaction *transaction)=0;
 

dali/base/dasess.cpp

@@ -880,8 +880,9 @@ public:
         int ret=-1;
         if (mb.remaining()>=sizeof(ret)) {
             mb.read(ret);
-            int e = 0;
-            if (mb.remaining()>=sizeof(e)) {
+            if (mb.remaining()>=sizeof(int)) {
+                int e = 0;
+                mb.read(e);
                 if (err)
                     *err = e;
                 else if (e) 

dali/daliadmin/daliadmin.cpp

@@ -1153,6 +1153,7 @@ static void checksuperfile(const char *lfn,bool fix=false)
         root->setPropInt("@numsubfiles",subnum);
     i = 0;
     byte fixstate = 0;
+    bool outOfSequence = false;
     loop {
         bool err = false;
         IPropertyTree *sub = root->queryPropTree(path.clear().appendf("SubFile[%d]",i+1).str());
@@ -1171,6 +1172,13 @@ static void checksuperfile(const char *lfn,bool fix=false)
                     i--;
                 }
             }
+            else if (pn != i+1) {
+                if (!outOfSequence)
+                    ERRLOG("SuperFile %s: corrupt, subfile file part @num values out of sequence, starting at part: %d", lname.get(), pn);
+                if (fix && (outOfSequence || doFix()))
+                    sub->setPropInt("@num", i+1);
+                outOfSequence = true;
+            }
         }
         else
             break;

docs/ECLLanguageReference/ECLR_mods/Recrd-DATASET.xml

@@ -59,7 +59,8 @@
   </emphasis><emphasis role="bold">)</emphasis></para>
 
   <para><emphasis role="bold">DATASET(</emphasis><emphasis> count, transform
-  </emphasis><emphasis role="bold"> [, DISTRIBUTED ] )</emphasis></para>
+  </emphasis><emphasis role="bold"> [, DISTRIBUTED | LOCAL ]
+  )</emphasis></para>
 
   <informaltable colsep="1" frame="all" rowsep="1">
     <tgroup cols="2">
@@ -277,6 +278,13 @@
           all nodes of the cluster. If omitted, all records are created on
           node 1.</entry>
         </row>
+
+        <row>
+          <entry><emphasis role="bold">LOCAL</emphasis></entry>
+
+          <entry>Optional. Specifies records are created on every
+          node.</entry>
+        </row>
       </tbody>
     </tgroup>
   </informaltable>
@@ -1423,19 +1431,23 @@ OUTPUT(ds);         </programlisting>
     <title>DATASET from TRANSFORM</title>
 
     <para><emphasis role="bold">DATASET(</emphasis><emphasis> count, transform
-    </emphasis><emphasis role="bold"> [, DISTRIBUTED ] )</emphasis></para>
+    </emphasis><emphasis role="bold"> [, DISTRIBUTED | LOCAL ]
+    )</emphasis></para>
 
     <para>This form uses the<emphasis> transform </emphasis> to create the
     records. The result type of the <emphasis>transform</emphasis> function
     determines the structure. The integer COUNTER can be used to number each
     iteration of the <emphasis>transform</emphasis> function.</para>
 
+    <para>LOCAL executes separately and independently on each node.</para>
+
     <para><emphasis role="bold">Example:</emphasis></para>
 
     <programlisting>IMPORT STD;
 msg(UNSIGNED c) := 'Rec ' + (STRING)c + ' on node ' + (STRING)(STD.system.Thorlib.Node()+1);
 
-DS := DATASET(CLUSTERSIZE * 2, 
+// DISTRIBUTED example
+DS := DATASET(CLUSTERSIZE * 2,
               TRANSFORM({STRING line}, 
                         SELF.line := msg(COUNTER)), 
               DISTRIBUTED);
@@ -1447,6 +1459,24 @@ DS;
    Rec 4 on node 2
    Rec 5 on node 3
    Rec 6 on node 3 
+*/
+
+// LOCAL example
+
+DS2 := DATASET(2,
+              TRANSFORM({STRING line},
+                        SELF.line := msg(COUNTER)),
+              LOCAL);
+DS2;
+
+/* An alternative (and clearer) way
+creates a result like this:
+   Rec 1 on node 1
+   Rec 2 on node 1
+   Rec 1 on node 2
+   Rec 2 on node 2
+   Rec 1 on node 3
+   Rec 2 on node 3
 */</programlisting>
 
     <para>See Also: <link linkend="RECORD_Structure">RECORD Structure</link>,

ecl/hqlcpp/hqlcpp.ipp

@@ -1581,7 +1581,7 @@ public:
     IHqlExpression * buildGetLocalResult(BuildCtx & ctx, IHqlExpression * expr);
 
     IHqlExpression * queryOptimizedExists(BuildCtx & ctx, IHqlExpression * expr, IHqlExpression * dataset);
-    void doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * dataset, IHqlExpression * doneFirstVar);
+    void doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * dataset, IHqlExpression * doneFirstVar, bool multiPath);
 
     void validateExprScope(BuildCtx & ctx, IHqlExpression * dataset, IHqlExpression * expr, const char * opName, const char * argName);
 

ecl/hqlcpp/hqlcppds.cpp

@@ -714,7 +714,7 @@ void HqlCppTranslator::doBuildExprAggregate(BuildCtx & ctx, IHqlExpression * exp
 }
 
 
-void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * dataset, IHqlExpression * doneFirstVar)
+void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoundTarget & target, IHqlExpression * expr, IHqlExpression * dataset, IHqlExpression * doneFirstVar, bool multiPath)
 {
     node_operator op = expr->getOperator();
     switch (op)
@@ -724,7 +724,17 @@ void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoun
             OwnedHqlExpr optimized = queryOptimizedExists(ctx, expr, dataset);
             if (optimized)
             {
-                assignBound(ctx, target, optimized);
+                if (matchesBoolean(optimized, false))
+                    return;
+
+                if (multiPath)
+                {
+                    BuildCtx condctx(ctx);
+                    buildFilter(condctx, optimized);
+                    assignBound(condctx, target, queryBoolExpr(true));
+                }
+                else
+                    assignBound(ctx, target, optimized);
                 return;
             }
             break;
@@ -748,15 +758,15 @@ void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoun
         {
             BuildCtx subctx(ctx);
             IHqlStmt * stmt = buildFilterViaExpr(subctx, dataset->queryChild(0));
-            doBuildAssignAggregateLoop(subctx, target, expr, dataset->queryChild(1), doneFirstVar);
+            doBuildAssignAggregateLoop(subctx, target, expr, dataset->queryChild(1), doneFirstVar, multiPath);
             subctx.selectElse(stmt);
-            doBuildAssignAggregateLoop(subctx, target, expr, dataset->queryChild(2), doneFirstVar);
+            doBuildAssignAggregateLoop(subctx, target, expr, dataset->queryChild(2), doneFirstVar, multiPath);
             return;
         }
     case no_addfiles:
         {
-            doBuildAssignAggregateLoop(ctx, target, expr, dataset->queryChild(0), doneFirstVar);
-            doBuildAssignAggregateLoop(ctx, target, expr, dataset->queryChild(1), doneFirstVar);
+            doBuildAssignAggregateLoop(ctx, target, expr, dataset->queryChild(0), doneFirstVar, true);
+            doBuildAssignAggregateLoop(ctx, target, expr, dataset->queryChild(1), doneFirstVar, true);
             return;
         }
     case no_chooseds:
@@ -778,7 +788,7 @@ void HqlCppTranslator::doBuildAssignAggregateLoop(BuildCtx & ctx, const CHqlBoun
                 else
                     subctx.addDefault(switchstmt);
 
-                doBuildAssignAggregateLoop(subctx, target, expr, cur, doneFirstVar);
+                doBuildAssignAggregateLoop(subctx, target, expr, cur, doneFirstVar, multiPath);
             }
             return;
         }
@@ -918,7 +928,7 @@ void HqlCppTranslator::doBuildAssignAggregate(BuildCtx & ctx, const CHqlBoundTar
             doneFirstVar.setown(ctx.getTempDeclare(queryBoolType(), queryBoolExpr(false)));
         }
 
-        doBuildAssignAggregateLoop(ctx, target, expr, dataset, doneFirstVar);
+        doBuildAssignAggregateLoop(ctx, target, expr, dataset, doneFirstVar, false);
     }
     else
     {
@@ -1011,22 +1021,29 @@ IHqlExpression * HqlCppTranslator::queryOptimizedExists(BuildCtx & ctx, IHqlExpr
         break;
     }
 
+    OwnedHqlExpr test;
     if (specialCase)
     {
         CHqlBoundExpr temp;
         buildDataset(ctx, dataset, temp, FormatNatural);
-        IHqlExpression * test;
         if (temp.count)
-            test = LINK(temp.count);
+            test.set(temp.count);
         else
-            test = getBoundSize(temp);
-        return createBoolExpr(op, test, createConstant(test->queryType()->castFrom(false, 0)));
+            test.setown(getBoundSize(temp));
     }
     else if (canOptimizeCount)
     {
-        IHqlExpression * count = optimized.expr;
-        return createBoolExpr(op, LINK(count), createConstant(count->queryType()->castFrom(false, 0)));
+        test.set(optimized.expr);
     }
+
+    if (test)
+    {
+        OwnedHqlExpr cond = createBoolExpr(op, LINK(test), createConstant(test->queryType()->castFrom(false, 0)));
+        if (cond->isConstant())
+            return foldHqlExpression(cond);
+        return cond.getClear();
+    }
+
     return NULL;
 }
 

ecllibrary/std/File.ecl

@@ -473,6 +473,9 @@ EXPORT RemoveSuperFile(varstring lsuperfn, varstring lfn, boolean del=false, boo
 EXPORT ClearSuperFile(varstring lsuperfn, boolean del=false) :=
     lib_fileservices.FileServices.ClearSuperFile(lsuperfn, del);
 
+EXPORT RemoveOwnedSubFiles(varstring lsuperfn, boolean del=false) :=
+    lib_fileservices.FileServices.RemoveOwnedSubFiles(lsuperfn, del);
+
 EXPORT SwapSuperFile(varstring lsuperfn1, varstring lsuperfn2) :=
     lib_fileservices.FileServices.SwapSuperFile(lsuperfn1, lsuperfn2);
 

esp/eclwatch/ws_XSLT/CMakeLists.txt

@@ -133,7 +133,7 @@ FOREACH ( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/dropzonefile.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/hpccresourcelist.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/dropzonefilelist.xslt
-    ${CMAKE_CURRENT_SOURCE_DIR}/WUBugReportForm.xslt
+    ${CMAKE_CURRENT_SOURCE_DIR}/WUZAPInfoForm.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/WUCopyLogicalFiles.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/WUDeployWorkunit.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/access_accountpermissions.xslt

esp/eclwatch/ws_XSLT/WUBugReportForm.xslt

@@ -18,7 +18,7 @@
 
 <xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
     <xsl:output method="html"/>
-    <xsl:template match="/WUGetBugReportInfoResponse">
+    <xsl:template match="/WUGetZAPInfoResponse">
         <html xmlns="http://www.w3.org/1999/xhtml" xml:lang="en" lang="en">
             <head>
                 <meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
@@ -66,7 +66,7 @@
                                 var desc = document.getElementById("ProblemDescription").value;
                                 var history = document.getElementById("WhatChanged").value;
                                 var timing = document.getElementById("WhereSlow").value;
-                                opener.createBugReport(wuid, espIP, thorIP, buildVersion, desc, history, timing);
+                                opener.createZAPInfo(wuid, espIP, thorIP, buildVersion, desc, history, timing);
                             }
                             window.close();
                         }
@@ -86,7 +86,7 @@
                 </script>
             </head>
             <body class="yui-skin-sam" onload="nof5();onLoad()">
-                <h3 style="text-align: center;">Report</h3>
+                <h3 style="text-align: center;">Zipped Analysis Package</h3>
                 <p/>
                 <form action="" method="POST">
                     <input type="hidden" id="WUID" name="WUID" value="{WUID}"/>

esp/eclwatch/ws_XSLT/wuid.xslt

@@ -738,18 +738,18 @@
 </soap:Envelope>
         */
 
-                    function popupBugReportForm()
+                    function popupZAPInfoForm()
                     {
-                        mywindow = window.open ("/WsWorkunits/WUGetBugReportInfo?WUID="+wid,
+                        mywindow = window.open ("/WsWorkunits/WUGetZAPInfo?WUID="+wid,
                             "mywindow", "location=0,status=1,scrollbars=1,resizable=1,width=800,height=760");
                         if (mywindow.opener == null)
                             mywindow.opener = window;
                         mywindow.focus();
                         return false;
                     }
-                    function createBugReport(wuid, espIP, thorIP, ESPBuildVersion, problemDesciption, history, timingInfo)
+                    function createZAPInfo(wuid, espIP, thorIP, ESPBuildVersion, problemDesciption, history, timingInfo)
                     {
-                        var href = "/WsWorkunits/WUReportBug?WUID=" + wuid;
+                        var href = "/WsWorkunits/WUCreateZAPInfo?WUID=" + wuid;
                         href += "&ESPIPAddress=" + espIP;
                         if (thorIP != '')
                             href += "&ThorIPAddress=" + thorIP;

esp/eclwatch/ws_XSLT/wuidcommon.xslt

@@ -788,7 +788,7 @@
                           <xsl:attribute name="disabled">disabled</xsl:attribute>
                         </xsl:if>
                       </input>
-              <input type="button" name="ZAPReport" style="width: 120px" value="Z.A.P. Report" class="sbutton" onclick="return popupBugReportForm()"/>
+              <input type="button" name="ZAPReport" style="width: 120px" value="Z.A.P. Report" class="sbutton" onclick="return popupZAPInfoForm()"/>
             </td>
           </tr>
           <tr>
@@ -1292,7 +1292,7 @@
           </a>
         </td>
         <td>
-          <a href="javascript:void(0)" onclick="getOptions('eclagent.log', '/WsWorkunits/WUFile/EclAgentLog?Wuid={$wuid}&amp;Name={Name}&amp;Type=EclAgentLog', false); return false;">
+          <a href="javascript:void(0)" onclick="getOptions('eclagent.log', '/WsWorkunits/WUFile/EclAgentLog?Wuid={$wuid}&amp;Name={Name}&amp;Process={PID}&amp;Type=EclAgentLog', false); return false;">
             download
           </a>
         </td>

esp/files/scripts/GraphsWidget.js

@@ -117,7 +117,16 @@ define([
                     },
                     Label: { label: "Label", sortable: true },
                     Complete: { label: "Completed", width: 72, sortable: true },
-                    Time: { label: "Time", width: 72, sortable: true },
+                    Time: {
+                        label: "Time", width: 90, sortable: true,
+                        formatter: function (totalSeconds, idx) {
+                            var hours = Math.floor(totalSeconds / 3600);
+                            totalSeconds %= 3600;
+                            var minutes = Math.floor(totalSeconds / 60);
+                            var seconds = (totalSeconds % 60).toFixed(2);
+                            return (hours < 10 ? "0" : "") + hours + ":" + (minutes < 10 ? "0" : "") + minutes + ":" + (seconds < 10 ? "0" : "") + seconds;
+                        }
+                    },
                     Type: { label: "Type", width: 72, sortable: true }
                 }
             }, domID);

esp/files/scripts/HPCCPlatformWidget.js

@@ -16,6 +16,7 @@
 define([
     "dojo/_base/declare",
     "dojo/_base/lang",
+    "dojo/_base/array",
     "dojo/dom",
     "dojo/dom-style",
 
@@ -27,6 +28,7 @@ define([
     "hpcc/_TabContainerWidget",
     "hpcc/ESPRequest",
     "hpcc/WsAccount",
+    "hpcc/ws_access",
     "hpcc/WsSMC",
     "hpcc/GraphWidget",
 
@@ -51,10 +53,10 @@ define([
     "hpcc/HPCCPlatformRoxieWidget",
     "hpcc/HPCCPlatformOpsWidget"
 
-], function (declare, lang, dom, domStyle,
+], function (declare, lang, arrayUtil, dom, domStyle,
                 registry, Tooltip,
                 UpgradeBar,
-                _TabContainerWidget, ESPRequest, WsAccount, WsSMC, GraphWidget,
+                _TabContainerWidget, ESPRequest, WsAccount, WsAccess, WsSMC, GraphWidget,
                 template) {
     return declare("HPCCPlatformWidget", [_TabContainerWidget], {
         templateString: template,
@@ -118,14 +120,16 @@ define([
         },
 
         init: function (params) {
-             if (this.inherited(arguments))
+            if (this.inherited(arguments))
                 return;
-
             var context = this;
+            registry.byId(context.id + "SetBanner").set("disabled", true);
+
             WsAccount.MyAccount({
             }).then(function (response) {
                 if (lang.exists("MyAccountResponse.username", response)) {
                     dom.byId(context.id + "UserID").innerHTML = response.MyAccountResponse.username;
+                    context.checkIfAdmin(response.MyAccountResponse.username);
                 }
             });
 
@@ -154,6 +158,28 @@ define([
             return "HPCC Platform";
         },
 
+        checkIfAdmin: function (user) {
+            var context = this;
+            if(user == null){
+                registry.byId(context.id + "SetBanner").set("disabled", false);
+            }else{
+                WsAccess.UserEdit({
+                    request: {
+                        username: user
+                    }
+                }).then(function (response) {
+                    if (lang.exists("UserEditResponse.Groups.Group", response)) {
+                        arrayUtil.forEach(response.UserEditResponse.Groups.Group, function (item, idx) {
+                            if(item.name == "Administrators"){
+                                registry.byId(context.id + "SetBanner").set("disabled", false);
+                                return true;
+                            }
+                        });
+                    }
+                });
+            }
+        },
+
         //  Hitched actions  ---
         _onFind: function (evt) {
             this.stackContainer.selectChild(this.mainPage);

esp/files/scripts/WUDetailsWidget.js

@@ -183,28 +183,28 @@ define([
 
         onZapReport: function (event) {
             var context = this;
-            WsWorkunits.WUGetBugReportInfo({
+            WsWorkunits.WUGetZAPInfo({
                 request: {
                     WUID: this.wu.Wuid
                 }
             }).then(function (response) {
                 registry.byId(context.id + "ZapDialog").show();
-                if (lang.exists("WUGetBugReportInfoResponse", response)) {
-                    context.updateInput("ZapWUID", null, response.WUGetBugReportInfoResponse.WUID);
-                    context.updateInput("BuildVersion", null, response.WUGetBugReportInfoResponse.BuildVersion);
-                    context.updateInput("ESPIPAddress", null, response.WUGetBugReportInfoResponse.ESPIPAddress);
-                    context.updateInput("ThorIPAddress", null, response.WUGetBugReportInfoResponse.ThorIPAddress);
+                if (lang.exists("WUGetZAPInfoResponse", response)) {
+                    context.updateInput("ZapWUID", null, response.WUGetZAPInfoResponse.WUID);
+                    context.updateInput("BuildVersion", null, response.WUGetZAPInfoResponse.BuildVersion);
+                    context.updateInput("ESPIPAddress", null, response.WUGetZAPInfoResponse.ESPIPAddress);
+                    context.updateInput("ThorIPAddress", null, response.WUGetZAPInfoResponse.ThorIPAddress);
                     
-                    context.buildVersion = response.WUGetBugReportInfoResponse.BuildVersion;
-                    context.espIPAddress = response.WUGetBugReportInfoResponse.ESPIPAddress;
-                    context.thorIPAddress = response.WUGetBugReportInfoResponse.ThorIPAddress;
+                    context.buildVersion = response.WUGetZAPInfoResponse.BuildVersion;
+                    context.espIPAddress = response.WUGetZAPInfoResponse.ESPIPAddress;
+                    context.thorIPAddress = response.WUGetZAPInfoResponse.ThorIPAddress;
                 }                
             });
         },
 
         onZapSubmit: function (event) {
             var frame = iframe.create("ZapDownload" + uniqueID++);
-            var url = ESPRequest.getBaseURL("WsWorkunits") + "/WUReportBug?WUID=" + this.wu.Wuid + "&ESPIPAddress=" + this.espIPAddress + "&ThorIPAddress=" + this.thorIPAddress + "&BuildVersion=" + encodeURIComponent(this.buildVersion);
+            var url = ESPRequest.getBaseURL("WsWorkunits") + "/WUCreateZAPInfo?WUID=" + this.wu.Wuid + "&ESPIPAddress=" + this.espIPAddress + "&ThorIPAddress=" + this.thorIPAddress + "&BuildVersion=" + encodeURIComponent(this.buildVersion);
             iframe.setSrc(frame, url, true);
             registry.byId(this.id + "ZapDialog").hide();
         },

esp/files/scripts/WsWorkunits.js

@@ -65,8 +65,8 @@ define([
             return ESPRequest.send("WsWorkunits", "WUQueryDetails", params);
         },
 
-        WUGetBugReportInfo: function (params) {
-            return ESPRequest.send("WsWorkunits", "WUGetBugReportInfo", params);
+        WUGetZAPInfo: function (params) {
+            return ESPRequest.send("WsWorkunits", "WUGetZAPInfo", params);
         },
 
         WUQuerysetAliasAction: function (selection, action) {

esp/files/scripts/_Widget.js

@@ -16,7 +16,7 @@ define([
         _onNewPage: function (event) {
             var baseUrl = document.URL.split("?")[0];
             var paramsString = ioQuery.objectToQuery(this.params);
-            var win = window.open(baseUrl + "?Widget=" + this.declaredClass + "&" + paramsString, "_blank");
+            var win = window.open(baseUrl + "?" + paramsString, "_blank");
             win.focus();
         },
 
@@ -25,6 +25,9 @@ define([
                 return true;
             this.initalized = true;
             this.params = params;
+            if (!this.params.Widget) {
+                this.params.Widget = this.declaredClass;
+            }
             
             return false;
         }

esp/files/scripts/ws_access.js

@@ -0,0 +1,26 @@
+/*##############################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+############################################################################## */
+define([
+    "hpcc/ESPRequest"
+], function (
+    ESPRequest) {
+    return {
+        UserEdit: function (params) {
+            return ESPRequest.send("ws_access", "UserEdit", params);
+        }
+    };
+});
+

esp/files/templates/HPCCPlatformWidget.html

@@ -61,7 +61,7 @@
     <div id="${id}SetBannerDialog" title="Set Banner" style="width: 480px;" data-dojo-type="dijit/Dialog">
         <div class="dijitDialogPaneContentArea">
             <div data-dojo-props="cols:2" data-dojo-type="dojox.layout.TableContainer">
-                <input id="${id}BannerText" title="Server:" name="ServerVersion" colspan="2" style="width:100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.Textarea" />
+                <input id="${id}BannerText" title="Banner&nbsp;Message:" name="ServerVersion" colspan="2" style="width:100%;" data-dojo-props="trim: true" data-dojo-type="dijit.form.Textarea" />
             </div>
         </div>
         <div class="dijitDialogPaneActionBar">

esp/scm/ws_workunits.ecm

@@ -1412,12 +1412,12 @@ ESPresponse [exceptions_inline] WUQuerySetCopyQueryResponse
     string QueryId;
 };
 
-ESPrequest [nil_remove] WUGetBugReportInfoRequest
+ESPrequest [nil_remove] WUGetZAPInfoRequest
 {
     string WUID;
 };
 
-ESPresponse [exceptions_inline] WUGetBugReportInfoResponse
+ESPresponse [exceptions_inline] WUGetZAPInfoResponse
 {
     string WUID;
     string ESPIPAddress;
@@ -1426,7 +1426,7 @@ ESPresponse [exceptions_inline] WUGetBugReportInfoResponse
     string Archive;
 };
 
-ESPrequest [nil_remove] WUReportBugRequest
+ESPrequest [nil_remove] WUCreateZAPInfoRequest
 {
     string WUID;
     string ESPIPAddress;
@@ -1437,7 +1437,7 @@ ESPrequest [nil_remove] WUReportBugRequest
     string WhereSlow;
 };
 
-ESPresponse [exceptions_inline] WUReportBugResponse
+ESPresponse [exceptions_inline] WUCreateZAPInfoResponse
 {
     [http_content("application/octet-stream")] binary thefile;
 };
@@ -1512,8 +1512,8 @@ ESPservice [
     ESPmethod [resp_xsl_default("/esp/xslt/WUCopyLogicalFiles.xslt")] WUCopyLogicalFiles(WUCopyLogicalFilesRequest, WUCopyLogicalFilesResponse);
     ESPmethod WUQueryConfig(WUQueryConfigRequest, WUQueryConfigResponse);
     ESPmethod WUListQueries(WUListQueriesRequest, WUListQueriesResponse);
-    ESPmethod WUReportBug(WUReportBugRequest, WUReportBugResponse);
-    ESPmethod [resp_xsl_default("/esp/xslt/WUBugReportForm.xslt")] WUGetBugReportInfo(WUGetBugReportInfoRequest, WUGetBugReportInfoResponse);
+    ESPmethod WUCreateZAPInfo(WUCreateZAPInfoRequest, WUCreateZAPInfoResponse);
+    ESPmethod [resp_xsl_default("/esp/xslt/WUZAPInfoForm.xslt")] WUGetZAPInfo(WUGetZAPInfoRequest, WUGetZAPInfoResponse);
 };
 
 

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3946,7 +3946,7 @@ void CWsWorkunitsEx::addProcessLogfile(IZZIPor* zipper, Owned<IConstWorkUnit> &c
 }
 
 
-bool CWsWorkunitsEx::onWUReportBug(IEspContext &context, IEspWUReportBugRequest &req, IEspWUReportBugResponse &resp)
+bool CWsWorkunitsEx::onWUCreateZAPInfo(IEspContext &context, IEspWUCreateZAPInfoRequest &req, IEspWUCreateZAPInfoResponse &resp)
 {
     try
     {
@@ -3963,7 +3963,7 @@ bool CWsWorkunitsEx::onWUReportBug(IEspContext &context, IEspWUReportBugRequest
         StringBuffer userName;
         if (context.queryUser())
             userName.append(context.queryUser()->getName());
-        zipFile.append("bugReport_").append(req.getWUID()).append('_').append(userName.str()).append(".zip");
+        zipFile.append("ZAPReport_").append(req.getWUID()).append('_').append(userName.str()).append(".zip");
         SCMStringBuffer temp;
         StringBuffer sb;
         sb.append("Workunit:     ").append(cwu->getWuid(temp)).append("\r\n");
@@ -4013,7 +4013,7 @@ bool CWsWorkunitsEx::onWUReportBug(IEspContext &context, IEspWUReportBugRequest
 #endif
             StringBuffer fs;
             //add report file to ZIP
-            fs.append("bugReport_").append(req.getWUID()).append('_').append(userName.str()).append(".txt");
+            fs.append("ZAPReport_").append(req.getWUID()).append('_').append(userName.str()).append(".txt");
             zipper->addContentToZIP(sb.length(), (void*)sb.str(), (char*)fs.str(), false);
 
             //add ECL query/archive to zip
@@ -4024,7 +4024,7 @@ bool CWsWorkunitsEx::onWUReportBug(IEspContext &context, IEspWUReportBugRequest
                 query->getQueryText(temp);
                 if (temp.length())
                 {
-                    fs.clear().append("bugReport_").append(req.getWUID()).append('_').append(userName.str()).append(".");
+                    fs.clear().append("ZAPReport_").append(req.getWUID()).append('_').append(userName.str()).append(".");
                     fs.append(isArchiveQuery(temp.str()) ? "archive" : "ecl");
                     ecl.append(temp.str());
                     zipper->addContentToZIP(ecl.length(), (void*)ecl.str(), (char*)fs.str(), true);
@@ -4042,7 +4042,7 @@ bool CWsWorkunitsEx::onWUReportBug(IEspContext &context, IEspWUReportBugRequest
             //Add Workunit XML file
             MemoryBuffer wuXmlMB;
             winfo.getWorkunitXml(NULL, wuXmlMB);
-            fs.clear().append("bugReport_").append(req.getWUID()).append('_').append(userName.str()).append(".xml");
+            fs.clear().append("ZAPReport_").append(req.getWUID()).append('_').append(userName.str()).append(".xml");
             zipper->addContentToZIP(wuXmlMB.length(), (void*)wuXmlMB.toByteArray(), (char*)fs.str(), true);
 
             //Write out ZIP file
@@ -4077,12 +4077,12 @@ bool CWsWorkunitsEx::onWUReportBug(IEspContext &context, IEspWUReportBugRequest
     return true;
 }
 
-bool CWsWorkunitsEx::onWUGetBugReportInfo(IEspContext &context, IEspWUGetBugReportInfoRequest &req, IEspWUGetBugReportInfoResponse &resp)
+bool CWsWorkunitsEx::onWUGetZAPInfo(IEspContext &context, IEspWUGetZAPInfoRequest &req, IEspWUGetZAPInfoResponse &resp)
 {
     try
     {
         StringBuffer wuid = req.getWUID();
-        checkAndTrimWorkunit("WUGetBugReportInfo", wuid);
+        checkAndTrimWorkunit("WUGetZAPInfo", wuid);
 
         Owned<IWorkUnitFactory> factory = getWorkUnitFactory(context.querySecManager(), context.queryUser());
         Owned<IConstWorkUnit> cw = factory->openWorkUnit(wuid, false);

esp/services/ws_workunits/ws_workunitsService.hpp

@@ -115,8 +115,8 @@ public:
 
     bool isQuerySuspended(const char* query, IConstWUClusterInfo *clusterInfo, unsigned wait, StringBuffer& errorMessage);
     bool onWUListQueries(IEspContext &context, IEspWUListQueriesRequest &req, IEspWUListQueriesResponse &resp);
-    bool onWUReportBug(IEspContext &context, IEspWUReportBugRequest &req, IEspWUReportBugResponse &resp);
-    bool onWUGetBugReportInfo(IEspContext &context, IEspWUGetBugReportInfoRequest &req, IEspWUGetBugReportInfoResponse &resp);
+    bool onWUCreateZAPInfo(IEspContext &context, IEspWUCreateZAPInfoRequest &req, IEspWUCreateZAPInfoResponse &resp);
+    bool onWUGetZAPInfo(IEspContext &context, IEspWUGetZAPInfoRequest &req, IEspWUGetZAPInfoResponse &resp);
 private:
     void addProcessLogfile(IZZIPor* zipper, Owned<IConstWorkUnit> &cwu, WsWuInfo &winfo, const char * process, PointerArray &mbArr);
 

plugins/Rembed/Rembed.cpp

@@ -69,7 +69,7 @@ public:
     RGlobalState()
     {
         const char *args[] = {"R", "--slave" };
-        R = new RInside(2, args, true, false, false);
+        R = new RInside(2, args, true, false, true);  // Setting interactive mode=true prevents R syntax errors from terminating the process
         // The R code for checking stack limits assumes that all calls are on the same thread
         // as the original context was created on - this will not always be true in ECL (and hardly
         // ever true in Roxie
@@ -164,29 +164,71 @@ public:
 
     virtual bool getBooleanResult()
     {
-        return ::Rcpp::as<bool>(result);
+        try
+        {
+            return ::Rcpp::as<bool>(result);
+        }
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
+        }
     }
     virtual void getDataResult(size32_t &__len, void * &__result)
     {
-        std::vector<byte> vval = ::Rcpp::as<std::vector<byte> >(result);
-        rtlStrToDataX(__len, __result, vval.size(), vval.data());
+        try
+        {
+            std::vector<byte> vval = ::Rcpp::as<std::vector<byte> >(result);
+            rtlStrToDataX(__len, __result, vval.size(), vval.data());
+        }
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
+        }
     }
     virtual double getRealResult()
     {
-        return ::Rcpp::as<double>(result);
+        try
+        {
+            return ::Rcpp::as<double>(result);
+        }
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
+        }
     }
     virtual __int64 getSignedResult()
     {
-        return ::Rcpp::as<long int>(result); // Should really be long long, but RInside does not support that
+        try
+        {
+            return ::Rcpp::as<long int>(result); // Should really be long long, but RInside does not support that
+        }
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
+        }
     }
     virtual unsigned __int64 getUnsignedResult()
     {
-        return ::Rcpp::as<unsigned long int>(result); // Should really be long long, but RInside does not support that
+        try
+        {
+            return ::Rcpp::as<unsigned long int>(result); // Should really be long long, but RInside does not support that
+        }
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
+        }
     }
     virtual void getStringResult(size32_t &__len, char * &__result)
     {
-        std::string str = ::Rcpp::as<std::string>(result);
-        rtlStrToStrX(__len, __result, str.length(), str.data());
+        try
+        {
+            std::string str = ::Rcpp::as<std::string>(result);
+            rtlStrToStrX(__len, __result, str.length(), str.data());
+        }
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
+        }
     }
     virtual void getUTF8Result(size32_t &chars, char * &result)
     {
@@ -198,10 +240,12 @@ public:
     }
     virtual void getSetResult(bool & __isAllResult, size32_t & __resultBytes, void * & __result, int _elemType, size32_t elemSize)
     {
-        type_t elemType = (type_t) _elemType;
-        __isAllResult = false;
-        switch(elemType)
+        try
         {
+            type_t elemType = (type_t) _elemType;
+            __isAllResult = false;
+            switch(elemType)
+            {
 
 #define FETCH_ARRAY(type) \
 {  \
@@ -209,111 +253,116 @@ public:
     rtlStrToDataX(__resultBytes, __result, vval.size()*elemSize, (const void *) vval.data()); \
 }
 
-        case type_boolean:
-        {
-            std::vector<bool> vval = ::Rcpp::as< std::vector<bool> >(result);
-            size32_t size = vval.size();
-            // Vector of bool is odd, and can't be retrieved via data()
-            // Instead we need to iterate, I guess
-            rtlDataAttr out(size);
-            bool *outData = (bool *) out.getdata();
-            for (std::vector<bool>::iterator iter = vval.begin(); iter < vval.end(); iter++)
+            case type_boolean:
             {
-                *outData++ = *iter;
-            }
-            __resultBytes = size;
-            __result = out.detachdata();
-            break;
-        }
-        case type_int:
-            /* if (elemSize == sizeof(signed char))  // rcpp does not seem to support...
-                FETCH_ARRAY(signed char)
-            else */ if (elemSize == sizeof(short))
-                FETCH_ARRAY(short)
-            else if (elemSize == sizeof(int))
-                FETCH_ARRAY(int)
-            else if (elemSize == sizeof(long))    // __int64 / long long does not work...
-                FETCH_ARRAY(long)
-            else
-                rtlFail(0, "Rembed: Unsupported result type");
-            break;
-        case type_unsigned:
-            if (elemSize == sizeof(byte))
-                FETCH_ARRAY(byte)
-            else if (elemSize == sizeof(unsigned short))
-                FETCH_ARRAY(unsigned short)
-            else if (elemSize == sizeof(unsigned int))
-                FETCH_ARRAY(unsigned int)
-            else if (elemSize == sizeof(unsigned long))    // __int64 / long long does not work...
-                FETCH_ARRAY(unsigned long)
-            else
-                rtlFail(0, "Rembed: Unsupported result type");
-            break;
-        case type_real:
-            if (elemSize == sizeof(float))
-                FETCH_ARRAY(float)
-            else if (elemSize == sizeof(double))
-                FETCH_ARRAY(double)
-            else
-                rtlFail(0, "Rembed: Unsupported result type");
-            break;
-        case type_string:
-        case type_varstring:
-        {
-            std::vector<std::string> vval = ::Rcpp::as< std::vector<std::string> >(result);
-            size32_t numResults = vval.size();
-            rtlRowBuilder out;
-            byte *outData = NULL;
-            size32_t outBytes = 0;
-            if (elemSize != UNKNOWN_LENGTH)
-            {
-                outBytes = numResults * elemSize;  // MORE - check for overflow?
-                out.ensureAvailable(outBytes);
-                outData = out.getbytes();
+                std::vector<bool> vval = ::Rcpp::as< std::vector<bool> >(result);
+                size32_t size = vval.size();
+                // Vector of bool is odd, and can't be retrieved via data()
+                // Instead we need to iterate, I guess
+                rtlDataAttr out(size);
+                bool *outData = (bool *) out.getdata();
+                for (std::vector<bool>::iterator iter = vval.begin(); iter < vval.end(); iter++)
+                {
+                    *outData++ = *iter;
+                }
+                __resultBytes = size;
+                __result = out.detachdata();
+                break;
             }
-            for (std::vector<std::string>::iterator iter = vval.begin(); iter < vval.end(); iter++)
+            case type_int:
+                /* if (elemSize == sizeof(signed char))  // rcpp does not seem to support...
+                    FETCH_ARRAY(signed char)
+                else */ if (elemSize == sizeof(short))
+                    FETCH_ARRAY(short)
+                else if (elemSize == sizeof(int))
+                    FETCH_ARRAY(int)
+                else if (elemSize == sizeof(long))    // __int64 / long long does not work...
+                    FETCH_ARRAY(long)
+                else
+                    rtlFail(0, "Rembed: Unsupported result type");
+                break;
+            case type_unsigned:
+                if (elemSize == sizeof(byte))
+                    FETCH_ARRAY(byte)
+                else if (elemSize == sizeof(unsigned short))
+                    FETCH_ARRAY(unsigned short)
+                else if (elemSize == sizeof(unsigned int))
+                    FETCH_ARRAY(unsigned int)
+                else if (elemSize == sizeof(unsigned long))    // __int64 / long long does not work...
+                    FETCH_ARRAY(unsigned long)
+                else
+                    rtlFail(0, "Rembed: Unsupported result type");
+                break;
+            case type_real:
+                if (elemSize == sizeof(float))
+                    FETCH_ARRAY(float)
+                else if (elemSize == sizeof(double))
+                    FETCH_ARRAY(double)
+                else
+                    rtlFail(0, "Rembed: Unsupported result type");
+                break;
+            case type_string:
+            case type_varstring:
             {
-                size32_t lenBytes = (*iter).size();
-                const char *text = (*iter).data();
-                if (elemType == type_string)
+                std::vector<std::string> vval = ::Rcpp::as< std::vector<std::string> >(result);
+                size32_t numResults = vval.size();
+                rtlRowBuilder out;
+                byte *outData = NULL;
+                size32_t outBytes = 0;
+                if (elemSize != UNKNOWN_LENGTH)
                 {
-                    if (elemSize == UNKNOWN_LENGTH)
-                    {
-                        out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
-                        outData = out.getbytes() + outBytes;
-                        * (size32_t *) outData = lenBytes;
-                        rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
-                        outBytes += lenBytes + sizeof(size32_t);
-                    }
-                    else
-                    {
-                        rtlStrToStr(elemSize, outData, lenBytes, text);
-                        outData += elemSize;
-                    }
+                    outBytes = numResults * elemSize;  // MORE - check for overflow?
+                    out.ensureAvailable(outBytes);
+                    outData = out.getbytes();
                 }
-                else
+                for (std::vector<std::string>::iterator iter = vval.begin(); iter < vval.end(); iter++)
                 {
-                    if (elemSize == UNKNOWN_LENGTH)
+                    size32_t lenBytes = (*iter).size();
+                    const char *text = (*iter).data();
+                    if (elemType == type_string)
                     {
-                        out.ensureAvailable(outBytes + lenBytes + 1);
-                        outData = out.getbytes() + outBytes;
-                        rtlStrToVStr(0, outData, lenBytes, text);
-                        outBytes += lenBytes + 1;
+                        if (elemSize == UNKNOWN_LENGTH)
+                        {
+                            out.ensureAvailable(outBytes + lenBytes + sizeof(size32_t));
+                            outData = out.getbytes() + outBytes;
+                            * (size32_t *) outData = lenBytes;
+                            rtlStrToStr(lenBytes, outData+sizeof(size32_t), lenBytes, text);
+                            outBytes += lenBytes + sizeof(size32_t);
+                        }
+                        else
+                        {
+                            rtlStrToStr(elemSize, outData, lenBytes, text);
+                            outData += elemSize;
+                        }
                     }
                     else
                     {
-                        rtlStrToVStr(elemSize, outData, lenBytes, text);  // Fixed size null terminated strings... weird.
-                        outData += elemSize;
+                        if (elemSize == UNKNOWN_LENGTH)
+                        {
+                            out.ensureAvailable(outBytes + lenBytes + 1);
+                            outData = out.getbytes() + outBytes;
+                            rtlStrToVStr(0, outData, lenBytes, text);
+                            outBytes += lenBytes + 1;
+                        }
+                        else
+                        {
+                            rtlStrToVStr(elemSize, outData, lenBytes, text);  // Fixed size null terminated strings... weird.
+                            outData += elemSize;
+                        }
                     }
                 }
+                __resultBytes = outBytes;
+                __result = out.detachdata();
+                break;
+            }
+            default:
+                rtlFail(0, "REmbed: Unsupported result type");
+                break;
             }
-            __resultBytes = outBytes;
-            __result = out.detachdata();
-            break;
         }
-        default:
-            rtlFail(0, "REmbed: Unsupported result type");
-            break;
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
         }
     }
 
@@ -451,12 +500,21 @@ public:
     }
     virtual void compileEmbeddedScript(size32_t lenChars, const char *utf)
     {
-        func.assign(utf, rtlUtf8Size(lenChars, utf));
+        StringBuffer text(rtlUtf8Size(lenChars, utf), utf);
+        text.stripChar('\r');
+        func.assign(text.str());
     }
 
     virtual void callFunction()
     {
-        result = R.parseEval(func);
+        try
+        {
+            result = R.parseEval(func);
+        }
+        catch (std::runtime_error &E)
+        {
+            rtlFail(0, E.what());
+        }
     }
 private:
     RInside &R;

plugins/fileservices/fileservices.cpp

@@ -86,6 +86,7 @@ static const char * EclDefinition =
 "  AddSuperFile(const varstring lsuperfn,const varstring lfn,unsigned4 atpos=0,boolean addcontents=false, boolean strict=false) : c,action,globalcontext,entrypoint='fsAddSuperFile'; \n"
 "  RemoveSuperFile(const varstring lsuperfn,const varstring lfn,boolean del=false,boolean remcontents=false) : c,action,globalcontext,entrypoint='fsRemoveSuperFile'; \n"
 "  ClearSuperFile(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsClearSuperFile'; \n"
+"  RemoveOwnedSubFiles(const varstring lsuperfn,boolean del=false) : c,action,globalcontext,entrypoint='fsRemoveOwnedSubFiles'; \n"
 "  SwapSuperFile(const varstring lsuperfn1,const varstring lsuperfn2) : c,action,globalcontext,entrypoint='fsSwapSuperFile'; \n"
 "  ReplaceSuperFile(const varstring lsuperfn,const varstring lfn,const varstring bylfn) : c,action,globalcontext,entrypoint='fsReplaceSuperFile'; \n"
 "  FinishSuperFileTransaction(boolean rollback=false) : c,action,globalcontext,entrypoint='fsFinishSuperFileTransaction'; \n"
@@ -1025,6 +1026,7 @@ StartSuperFileTransaction();
 AddSuperFile(const varstring lsuperfn,const varstring lfn,unsigned4 atpos=0);
 RemoveSuperFile(const varstring lsuperfn,const varstring lfn,boolean del=false);
 ClearSuperFile(const varstring lsuperfn,boolean del=false);
+RemoveOwnedSubFiles(const varstring lsuperfn,boolean del=false);
 SwapSuperFile(const varstring lsuperfn1,const varstring lsuperfn2);
 ReplaceSuperFile(const varstring lsuperfn,const varstring lfn,const varstring bylfn);
 FinishSuperFileTransaction(boolean rollback=false);
@@ -1278,6 +1280,31 @@ FILESERVICES_API void FILESERVICES_CALL fsClearSuperFile(IGlobalCodeContext *gct
     fsRemoveSuperFile(gctx,lsuperfn,NULL,del);
 }
 
+FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *gctx, const char *lsuperfn, bool del)
+{
+    fslRemoveOwnedSubFiles(gctx->queryCodeContext(), lsuperfn, del);
+}
+
+FILESERVICES_API void FILESERVICES_CALL fslRemoveOwnedSubFiles(ICodeContext *ctx, const char *lsuperfn, bool del)
+{
+    Owned<IDistributedSuperFile> file;
+    StringBuffer lsfn;
+    lookupSuperFile(ctx, lsuperfn, file, true, lsfn, false, true);
+    IDistributedFileTransaction *transaction = ctx->querySuperFileTransaction();
+    assertex(transaction);
+    file->removeOwnedSubFiles(del,transaction);
+    VStringBuffer s("RemoveOwnedSubFiles ('%s'", lsfn.str());
+    if (del)
+        s.append(", del");
+    s.append(") ");
+    if (transaction->active())
+        s.append("trans");
+    else
+        s.append("done");
+    WUmessage(ctx,ExceptionSeverityInformation,NULL,s.str());
+    AuditMessage(ctx,"RemoveOwnedSubFiles",lsfn.str());
+}
+
 FILESERVICES_API void FILESERVICES_CALL fslClearSuperFile(ICodeContext *ctx, const char *lsuperfn,bool del)
 {
     fslRemoveSuperFile(ctx,lsuperfn,NULL,del);

plugins/fileservices/fileservices.hpp

@@ -68,6 +68,7 @@ FILESERVICES_API void FILESERVICES_CALL fsStartSuperFileTransaction(IGlobalCodeC
 FILESERVICES_API void FILESERVICES_CALL fsAddSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *_lfn,unsigned atpos=0,bool addcontents=false, bool strict=false);
 FILESERVICES_API void FILESERVICES_CALL fsRemoveSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *_lfn,bool del=false,bool remcontents=false);
 FILESERVICES_API void FILESERVICES_CALL fsClearSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
+FILESERVICES_API void FILESERVICES_CALL fsRemoveOwnedSubFiles(IGlobalCodeContext *ctx, const char *lsuperfn,bool del=false);
 FILESERVICES_API void FILESERVICES_CALL fsSwapSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn1,const char *lsuperfn2);
 FILESERVICES_API void FILESERVICES_CALL fsReplaceSuperFile(IGlobalCodeContext *ctx, const char *lsuperfn,const char *lfn,const char *bylfn);
 FILESERVICES_API void FILESERVICES_CALL fsFinishSuperFileTransaction(IGlobalCodeContext *ctx, bool rollback=false);
@@ -103,6 +104,7 @@ FILESERVICES_API void FILESERVICES_CALL fslStartSuperFileTransaction(ICodeContex
 FILESERVICES_API void FILESERVICES_CALL fslAddSuperFile(ICodeContext *ctx, const char *lsuperfn,const char *_lfn,unsigned atpos=0,bool addcontents=false, bool strict=false);
 FILESERVICES_API void FILESERVICES_CALL fslRemoveSuperFile(ICodeContext *ctx, const char *lsuperfn,const char *_lfn,bool del=false,bool remcontents=false);
 FILESERVICES_API void FILESERVICES_CALL fslClearSuperFile(ICodeContext *ctx, const char *lsuperfn,bool del=false);
+FILESERVICES_API void FILESERVICES_CALL fslRemoveOwnedSubFiles(ICodeContext *ctx, const char *lsuperfn,bool del=false);
 FILESERVICES_API void FILESERVICES_CALL fslSwapSuperFile(ICodeContext *ctx, const char *lsuperfn1,const char *lsuperfn2);
 FILESERVICES_API void FILESERVICES_CALL fslReplaceSuperFile(ICodeContext *ctx, const char *lsuperfn,const char *lfn,const char *bylfn);
 FILESERVICES_API void FILESERVICES_CALL fslFinishSuperFileTransaction(ICodeContext *ctx, bool rollback=false);

plugins/javaembed/javaembed.cpp

@@ -113,6 +113,9 @@ public:
             optionStrings.appendList(conf->queryProp("jvmoptions"), ENVSEPSTR);
         }
 
+        // Options we know we always want set
+        optionStrings.append("-Xrs");
+
         // These may be useful for debugging
         // optionStrings.append("-Xcheck:jni");
         // optionStrings.append("-verbose:jni");
@@ -243,7 +246,7 @@ public:
             rtlFail(0, message.str());
         }
     }
-    
+
     void ensureContextClassLoaderAvailable ()
     {
         // JVMs that are created by native threads have a context class loader set to the
@@ -255,7 +258,7 @@ public:
         //
         // if (Thread.currentThread().getContextClassLoader == NULL)
         //     Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader());
-        
+
         if (!contextClassLoaderChecked)
         {
             JNIenv->ExceptionClear();
@@ -270,7 +273,7 @@ public:
             checkException();
             jobject contextClassLoaderObj = JNIenv->CallObjectMethod(threadObj, getContextClassLoaderMethod);
             checkException();
-            
+
             if (!contextClassLoaderObj)
             {
                 // No context class loader, so use the system class loader (hopefully it's present)
@@ -280,7 +283,7 @@ public:
                 checkException();
                 jobject systemClassLoaderObj = JNIenv->CallStaticObjectMethod(javaLangClassLoaderClass, getSystemClassLoaderMethod);
                 checkException();
-                
+
                 if (systemClassLoaderObj)
                 {
                     jmethodID setContextClassLoaderMethod = JNIenv->GetMethodID(javaLangThreadClass, "setContextClassLoader", "(Ljava/lang/ClassLoader;)V");
@@ -289,7 +292,7 @@ public:
                     checkException();
                 }
             }
-            
+
             contextClassLoaderChecked = true;
         }
     }
@@ -301,11 +304,11 @@ public:
         if (!prevtext || strcmp(text, prevtext) != 0)
         {
             prevtext.clear();
-            
+
             // Make sure there is a context class loader available; we need to
             // do this before calling FindClass() on the class we need
             ensureContextClassLoaderAvailable();
-            
+
             // Name should be in the form class.method:signature
             const char *funcname = strchr(text, '.');
             if (!funcname)

plugins/pyembed/pyembed.cpp

@@ -178,6 +178,7 @@ public:
         {
             prevtext.clear();
             // Try compiling as a eval first... if that fails, try as a script.
+            text.stripChar('\r');
             script.setown(Py_CompileString(text, "", Py_eval_input));
             if (!script)
             {
@@ -187,7 +188,7 @@ public:
                 script.setown(Py_CompileString(wrapped, "<embed>", Py_file_input));
             }
             checkPythonError();
-            prevtext.set(text);
+            prevtext.set(utf, bytes);
         }
         return script.getLink();
     }

system/jlib/jstring.cpp

@@ -839,6 +839,24 @@ StringBuffer & StringBuffer::replaceString(const char* oldStr, const char* newSt
     return *this;
 }
 
+StringBuffer & StringBuffer::stripChar(char oldChar)
+{
+    if (buffer)
+    {
+        size32_t delta = 0;
+        size32_t l = curLen;
+        for (size32_t i = 0; i < l; i++)
+        {
+            if (buffer[i] == oldChar)
+                delta++;
+            else if (delta)
+                buffer[i-delta] = buffer[i];
+        }
+        curLen = curLen - delta;
+    }
+    return *this;
+}
+
 const char * StringBuffer::toCharArray() const
 {
     if (buffer)

system/jlib/jstring.hpp

@@ -115,6 +115,7 @@ public:
     StringBuffer &  replaceString(const char* oldStr, const char* newStr);
     char *          reserve(size32_t size);
     char *          reserveTruncate(size32_t size);
+    StringBuffer &  stripChar(char oldChar);
     void            swapWith(StringBuffer &other);
     void setBuffer(size32_t buffLen, char * newBuff, size32_t strLen);
 

thorlcr/thorutil/thmem.hpp

@@ -200,19 +200,25 @@ enum {
 
 graph_decl StringBuffer &getRecordString(const void *key, IOutputRowSerializer *serializer, const char *prefix, StringBuffer &out);
 
-#define SPILL_PRIORITY_DEFAULT 50
+//NB: low priorities are spilt 1st
+#define SPILL_PRIORITY_LOW  100
+#define SPILL_PRIORITY_HIGH 1000000
+#define SPILL_PRIORITY_DEFAULT SPILL_PRIORITY_LOW
 #define SPILL_PRIORITY_DISABLE UINT_MAX
 
-#define SPILL_PRIORITY_JOIN 10
-#define SPILL_PRIORITY_SELFJOIN 10
-#define SPILL_PRIORITY_HASHJOIN 10
-#define SPILL_PRIORITY_LARGESORT 10
-#define SPILL_PRIORITY_GROUPSORT 20
-#define SPILL_PRIORITY_HASHDEDUP 30
-#define SPILL_PRIORITY_OVERFLOWABLE_BUFFER SPILL_PRIORITY_DEFAULT
-#define SPILL_PRIORITY_SPILLABLE_STREAM SPILL_PRIORITY_DEFAULT
-#define SPILL_PRIORITY_RESULT SPILL_PRIORITY_DEFAULT
-#define SPILL_PRIORITY_LOOKUPJOIN 10
+#define SPILL_PRIORITY_OVERFLOWABLE_BUFFER SPILL_PRIORITY_LOW
+#define SPILL_PRIORITY_SPILLABLE_STREAM SPILL_PRIORITY_LOW
+#define SPILL_PRIORITY_RESULT SPILL_PRIORITY_LOW
+
+#define SPILL_PRIORITY_GROUPSORT SPILL_PRIORITY_LOW+1000
+#define SPILL_PRIORITY_HASHDEDUP SPILL_PRIORITY_LOW+2000
+
+#define SPILL_PRIORITY_JOIN SPILL_PRIORITY_HIGH
+#define SPILL_PRIORITY_SELFJOIN SPILL_PRIORITY_HIGH
+#define SPILL_PRIORITY_HASHJOIN SPILL_PRIORITY_HIGH
+#define SPILL_PRIORITY_LARGESORT SPILL_PRIORITY_HIGH
+#define SPILL_PRIORITY_LOOKUPJOIN SPILL_PRIORITY_HIGH
+
 
 enum StableSortFlag { stableSort_none, stableSort_earlyAlloc, stableSort_lateAlloc };
 class CThorSpillableRowArray;