Merge remote-tracking branch 'origin/candidate-3.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/deftype/defvalue.cpp

@@ -39,7 +39,6 @@ BoolValue *BoolValue::falseconst;
 static _ATOM asciiAtom;
 static _ATOM ebcdicAtom;
 
-
 MODULE_INIT(INIT_PRIORITY_DEFVALUE)
 {
     asciiAtom = createLowerCaseAtom("ascii");
@@ -236,7 +235,7 @@ VarStringValue::VarStringValue(unsigned len, const char *v, ITypeInfo *_type) :
         val.set(v, typeLen);
     else
     {
-        char * temp = (char *)malloc(typeLen+1);
+        char * temp = (char *)checked_malloc(typeLen+1, DEFVALUE_MALLOC_FAILED);
         memcpy(temp, v, len);
         temp[len] = 0;
         val.set(temp, typeLen);
@@ -439,7 +438,7 @@ void MemoryValue::deserialize(MemoryBuffer &src)
 {
     size32_t size;
     src.read(size);
-    void *mem = malloc(size);
+    void *mem = checked_malloc(size, DEFVALUE_MALLOC_FAILED);
     assertex(mem);
     src.read(size, mem);
     val.set(size, mem);
@@ -583,7 +582,7 @@ IValue *createStringValue(const char *val, ITypeInfo *type, int srcLength, IChar
     }
     else if (tgtLength > srcLength)
     {
-        char * extended = (char *)malloc(tgtLength);
+        char * extended = (char *)checked_malloc(tgtLength, DEFVALUE_MALLOC_FAILED);
         memcpy(extended, val, srcLength);
         memset(extended+srcLength, type->queryCharset()->queryFillChar(), tgtLength-srcLength);
         IValue * ret = new StringValue(extended, type);
@@ -750,7 +749,7 @@ IValue *createUnicodeValue(char const * value, ITypeInfo * type, unsigned srclen
         type->Release();
         return createUnicodeValue(value, srclen, type->queryLocale()->str(), false);
     }
-    UChar * buff = (UChar *)malloc(type->getSize());
+    UChar * buff = (UChar *)checked_malloc(type->getSize(), DEFVALUE_MALLOC_FAILED);
     rtlCodepageToUnicode(type->getStringLen(), buff, srclen, value, "US-ASCII");
     IValue * ret = new UnicodeValue(buff, type);
     free(buff);
@@ -790,7 +789,7 @@ IValue * createUnicodeValue(size32_t len, const void * text, ITypeInfo * type)
 void UnicodeAttr::set(UChar const * _text, unsigned _len)
 {
     free(text);
-    text = (UChar *) malloc((_len+1)*2);
+    text = (UChar *) checked_malloc((_len+1)*2, DEFVALUE_MALLOC_FAILED);
     memcpy(text, _text, _len*2);
     text[_len] = 0x0000;
 }
@@ -803,7 +802,7 @@ VarUnicodeValue::VarUnicodeValue(unsigned len, const UChar * v, ITypeInfo * _typ
         val.set(v, typeLen);
     else
     {
-        UChar * temp = (UChar *)malloc((typeLen+1)*2);
+        UChar * temp = (UChar *)checked_malloc((typeLen+1)*2, DEFVALUE_MALLOC_FAILED);
         memcpy(temp, v, len*2);
         temp[len] = 0;
         val.set(temp, typeLen);
@@ -948,7 +947,7 @@ void VarUnicodeValue::deserialize(MemoryBuffer & src)
 {
     size32_t len;
     src.read(len);
-    UChar * buff = (UChar *) malloc(len*2);
+    UChar * buff = (UChar *) checked_malloc(len*2, DEFVALUE_MALLOC_FAILED);
     src.read(len*2, buff);
     val.set(buff, len);
 }
@@ -1206,7 +1205,7 @@ IValue *DataValue::castTo(ITypeInfo *t)
             return new DataValue(val.get(), LINK(t));
         else
         {
-            char *newstr = (char *) malloc(nsize);
+            char *newstr = (char *) checked_malloc(nsize, DEFVALUE_MALLOC_FAILED);
             memcpy(newstr, val.get(), osize);
             memset(newstr+osize, 0, nsize-osize);
             IValue * ret = new DataValue(newstr, LINK(t));
@@ -1226,7 +1225,7 @@ IValue *DataValue::castTo(ITypeInfo *t)
             return new StringValue((char *)val.get(), t);
         else
         {
-            char *newstr = (char *) malloc(nsize);
+            char *newstr = (char *) checked_malloc(nsize, DEFVALUE_MALLOC_FAILED);
             memcpy(newstr, val.get(), osize);
             memset(newstr+osize, t->queryCharset()->queryFillChar(), nsize-osize);
             IValue * ret = new StringValue(newstr, t);
@@ -1304,7 +1303,7 @@ QStringValue::QStringValue(const char *v, ITypeInfo *_type) : MemoryValue(_type)
 const char *QStringValue::generateECL(StringBuffer &out)
 {
     unsigned strLen = type->getStringLen();
-    char * strData = (char *)malloc(strLen);
+    char * strData = (char *)checked_malloc(strLen, DEFVALUE_MALLOC_FAILED);
     rtlQStrToStr(strLen, strData, strLen, (const char *)val.get());
     out.append('Q');
     appendStringAsQuotedECL(out, strLen, strData);
@@ -1341,7 +1340,7 @@ IValue *QStringValue::castTo(ITypeInfo *t)
     }
 
     unsigned strLen = type->getStringLen();
-    char * strData = (char *)malloc(strLen);
+    char * strData = (char *)checked_malloc(strLen, DEFVALUE_MALLOC_FAILED);
     rtlQStrToStr(strLen, strData, strLen, (const char *)val.get());
     IValue * ret = t->castFrom(strLen, strData);
     free(strData);
@@ -1370,7 +1369,7 @@ bool QStringValue::getBoolValue()
 __int64 QStringValue::getIntValue()
 {
     unsigned strLen = type->getStringLen();
-    char * strData = (char *)malloc(strLen);
+    char * strData = (char *)checked_malloc(strLen, DEFVALUE_MALLOC_FAILED);
     rtlQStrToStr(strLen, strData, strLen, (const char *)val.get());
     __int64 ret = rtlStrToInt8(strLen, strData);
     free(strData);
@@ -1380,7 +1379,7 @@ __int64 QStringValue::getIntValue()
 const char *QStringValue::getStringValue(StringBuffer &out)
 {
     unsigned strLen = type->getStringLen();
-    char * strData = (char *)malloc(strLen);
+    char * strData = (char *)checked_malloc(strLen, DEFVALUE_MALLOC_FAILED);
     rtlQStrToStr(strLen, strData, strLen, (const char *)val.get());
     out.append(strLen, strData);
     free(strData);
@@ -1390,7 +1389,7 @@ const char *QStringValue::getStringValue(StringBuffer &out)
 void QStringValue::pushDecimalValue()
 {
     unsigned strLen = type->getStringLen();
-    char * strData = (char *)malloc(strLen);
+    char * strData = (char *)checked_malloc(strLen, DEFVALUE_MALLOC_FAILED);
     rtlQStrToStr(strLen, strData, strLen, (const char *)val.get());
     DecPushString(strLen, strData);
     free(strData);
@@ -1571,7 +1570,7 @@ IValue *IntValue::castTo(ITypeInfo *t)
         if (nLen == UNKNOWN_LENGTH)
             return castViaString(t);
 
-        char *newstr = (char *) malloc(nLen);
+        char *newstr = (char *) checked_malloc(nLen, DEFVALUE_MALLOC_FAILED);
         if (type->isSigned())
             rtlInt8ToStr(nLen, newstr, val);
         else
@@ -2036,7 +2035,7 @@ IValue *createRealValue(double val, ITypeInfo * type)
 DecimalValue::DecimalValue(const void * v, ITypeInfo * _type) : CValue(_type)
 {
     unsigned len = _type->getSize();
-    val = (char *)malloc(len);
+    val = (char *)checked_malloc(len, DEFVALUE_MALLOC_FAILED);
     memcpy(val, v, len);
 }
 
@@ -2179,7 +2178,7 @@ void DecimalValue::deserialize(MemoryBuffer &src)
 {
     size32_t size;
     src.read(size);
-    val = malloc(size);
+    val = checked_malloc(size, DEFVALUE_MALLOC_FAILED);
     assertex(val);
 }
 
@@ -2404,7 +2403,7 @@ void appendValueToBuffer(MemoryBuffer & mem, IValue * value)
 {
     ITypeInfo * type = value->queryType();
     unsigned len = type->getSize();
-    void * temp = malloc(len);
+    void * temp = checked_malloc(len, DEFVALUE_MALLOC_FAILED);
     value->toMem(temp);
 
     if (type->isSwappedEndian() != mem.needSwapEndian())
@@ -3045,15 +3044,6 @@ void getStringFromIValue(unsigned & len, char* & str, IValue* val)
 }
 //---------------------------------------------------------------------
 
-unsigned extractUnicode(IValue * in, rtlDataAttr & out)
-{
-    ITypeInfo * type = in->queryType();
-    unsigned bufflen = type->getStringLen()+1;
-    out.setown(rtlMalloc(2*bufflen));
-    in->getUCharStringValue(bufflen, out.getdata());
-    return rtlUnicodeStrlen(out.getustr());
-}
-
 IValue * concatValues(IValue * left, IValue * right)
 {
     ITypeInfo * leftType = left->queryType();

common/deftype/defvalue.hpp

@@ -30,6 +30,9 @@
 #define DEFTYPE_API
 #endif
 
+#define DEFVALUE_MALLOC_FAILED 701  //Unable to allocate requested memory
+
+
 interface IValue : public serializable
 {
 public:

common/workunit/wujobq.cpp

@@ -474,6 +474,12 @@ public:
             EXCLOG(e, "~CJobQueue calling dounsubscribe");
             e->Release();
         }
+        while (qdata)
+        {
+            sQueueData * next = qdata->next;
+            delete qdata;
+            qdata = next;
+        }
     }
 
 

dali/base/dadfs.cpp

@@ -882,6 +882,7 @@ public:
     }
     IDistributedFile *createNew(IPropertyTree *tree,bool ignoregroup);
     IDistributedSuperFile *createSuperFile(const char *logicalname,IUserDescriptor *user,bool interleaved,bool ifdoesnotexist,IDistributedFileTransaction *transaction=NULL);
+    void removeSuperFile(const char *_logicalname, bool delSubs, IUserDescriptor *user, IDistributedFileTransaction *transaction);
 
     IDistributedFileIterator *getIterator(const char *wildname, bool includesuper,IUserDescriptor *user);
     IDFAttributesIterator *getDFAttributesIterator(const char *wildname, IUserDescriptor *user, bool recursive, bool includesuper,INode *foreigndali,unsigned foreigndalitimeout);
@@ -1025,6 +1026,12 @@ public:
         state = TAS_RETRY;
         unlock();
     }
+    // MORE: In the rare event of a commit failure, not all actions can be rolled back.
+    // Since all actions today occur during "run", and since commit phases does very little,
+    // this chance is minimal and will probably be caused by corrupted file descriptors.
+    // The problem is that the state of the sub removals and the order in which they occur might not
+    // be trivial on such a low level error, and there's no way to atomically do operations in SDS
+    // at present time. We need more thought about this.
     virtual void commit()
     {
         state = TAS_SUCCESS;
@@ -6555,7 +6562,7 @@ IDistributedFile *CDistributedFileDirectory::createNew(IFileDescriptor *fdesc,co
 /**
  * Creates a super-file within a transaction.
  */
-class cCreateSuperFileAction: public CDFAction
+class CCreateSuperFileAction: public CDFAction
 {
     CDfsLogicalFileName logicalname;
     CDistributedFileDirectory *parent;
@@ -6564,7 +6571,7 @@ class cCreateSuperFileAction: public CDFAction
     IPropertyTree *root;
     bool created;
 public:
-    cCreateSuperFileAction(IDistributedFileTransaction *_transaction,
+    CCreateSuperFileAction(IDistributedFileTransaction *_transaction,
                            CDistributedFileDirectory *_parent,
                            IUserDescriptor *_user,
                            const char *_flname,
@@ -6573,10 +6580,8 @@ public:
     {
         logicalname.set(_flname);
         // We *have* to make sure the file doesn't exist here
-        IDistributedSuperFile *sfile = parent->lookupSuperFile(logicalname.get(), user, transaction, SDS_SUB_LOCK_TIMEOUT);
-        if (sfile) {
-            super.setown(sfile);
-        } else {
+        super.setown(transaction->lookupSuperFile(logicalname.get(), SDS_SUB_LOCK_TIMEOUT));
+        if (!super) {
             // Create file and link to transaction, so subsequent lookups won't fail
             root = createPTree();
             root->setPropInt("@interleaved",interleaved?2:0); // this is ill placed
@@ -6585,7 +6590,7 @@ public:
         }
         addFileLock(super);
     }
-    virtual ~cCreateSuperFileAction() {}
+    virtual ~CCreateSuperFileAction() {}
     bool prepare()
     {
         // Attach the file to DFS, if wasn't there already
@@ -6616,6 +6621,52 @@ public:
     }
 };
 
+/**
+ * Removes a super-file within a transaction.
+ */
+class CRemoveSuperFileAction: public CDFAction
+{
+    CDfsLogicalFileName logicalname;
+    Linked<IDistributedSuperFile> super;
+    IUserDescriptor *user;
+    bool delSub;
+public:
+    CRemoveSuperFileAction(IDistributedFileTransaction *_transaction,
+                           IUserDescriptor *_user,
+                           const char *_flname,
+                           bool _delSub)
+        : CDFAction(_transaction), user(_user), delSub(_delSub)
+    {
+        logicalname.set(_flname);
+        // We *have* to make sure the file exists here
+        super.setown(transaction->lookupSuperFile(logicalname.get(), SDS_SUB_LOCK_TIMEOUT));
+        if (!super)
+            ThrowStringException(-1, "Super File %s doesn't exist in the file system", logicalname.get());
+        addFileLock(super);
+        // Adds actions to transactions before this one and gets executed only on commit
+        if (delSub)
+            super->removeSubFile(NULL, true, true, false, transaction);
+    }
+    virtual ~CRemoveSuperFileAction() {}
+    bool prepare()
+    {
+        if (lock())
+            return true;
+        unlock();
+        return false;
+    }
+    void run()
+    {
+        // Removing here would make it hard to re-attach the sub files on rollback (FIXME?)
+    }
+    void commit()
+    {
+        super->detach();
+        CDFAction::commit();
+    }
+};
+
+// MORE: This should be implemented in DFSAccess later on
 IDistributedSuperFile *CDistributedFileDirectory::createSuperFile(const char *_logicalname,IUserDescriptor *user, bool _interleaved,bool ifdoesnotexist,IDistributedFileTransaction *transaction)
 {
     CDfsLogicalFileName logicalname;
@@ -6643,13 +6694,35 @@ IDistributedSuperFile *CDistributedFileDirectory::createSuperFile(const char *_l
     }
 
     // action is owned by transaction (acquired on CDFAction's c-tor) so don't unlink or delete!
-    cCreateSuperFileAction *action = new cCreateSuperFileAction(localtrans,this,user,_logicalname,_interleaved);
+    CCreateSuperFileAction *action = new CCreateSuperFileAction(localtrans,this,user,_logicalname,_interleaved);
 
     localtrans->autoCommit();
 
     return localtrans->lookupSuperFile(_logicalname);
 }
 
+// MORE: This should be implemented in DFSAccess later on
+void CDistributedFileDirectory::removeSuperFile(const char *_logicalname, bool delSubs, IUserDescriptor *user, IDistributedFileTransaction *transaction)
+{
+    CDfsLogicalFileName logicalname;
+    logicalname.set(_logicalname);
+    checkLogicalName(logicalname,user,true,true,false,"have a superfile with");
+
+    // Create a local transaction that will be destroyed (but never touch the external transaction)
+    Linked<IDistributedFileTransaction> localtrans;
+    if (transaction) {
+        localtrans.set(transaction);
+    } else {
+        // TODO: Make it explicit in the API that a transaction is required
+        localtrans.setown(new CDistributedFileTransaction(user));
+    }
+
+    // action is owned by transaction (acquired on CDFAction's c-tor) so don't unlink or delete!
+    CRemoveSuperFileAction *action = new CRemoveSuperFileAction(localtrans, user, _logicalname, delSubs);
+
+    localtrans->autoCommit();
+}
+
 // MORE - this should go when remove file gets into transactions
 bool CDistributedFileDirectory::cannotRemove(CDfsLogicalFileName &dlfn,IUserDescriptor *user,StringBuffer &reason,bool ignoresub, unsigned timeoutms)
 {

dali/base/dadfs.hpp

@@ -472,9 +472,8 @@ interface IDistributedFileDirectory: extends IInterface
     virtual IDistributedSuperFile *createSuperFile(const char *logicalname,IUserDescriptor *user,bool interleaved,bool ifdoesnotexist=false,IDistributedFileTransaction *transaction=NULL) = 0;
     virtual IDistributedSuperFile *lookupSuperFile(const char *logicalname,IUserDescriptor *user,
                                                     IDistributedFileTransaction *transaction=NULL, // transaction only used for looking up sub files
-                                                    unsigned timeout=INFINITE
-
-                                                ) = 0;  // NB lookup will also return superfiles 
+                                                    unsigned timeout=INFINITE) = 0;  // NB lookup will also return superfiles
+    virtual void removeSuperFile(const char *_logicalname, bool delSubs=false, IUserDescriptor *user=NULL, IDistributedFileTransaction *transaction=NULL)=0;
 
     virtual int getFilePermissions(const char *lname,IUserDescriptor *user,unsigned auditflags=0)=0; // see dasess for auditflags values
     virtual void setDefaultUser(IUserDescriptor *user)=0;

docs/ECLLanguageReference/ECLR_mods/BltInFunc-ASSERT.xml

@@ -12,11 +12,13 @@
       <primary>FAIL</primary>
     </indexterm> ] [ </emphasis>, <emphasis role="bold">CONST<indexterm>
       <primary>CONST</primary>
-    </indexterm> ])</emphasis><emphasis role="bold">ASSERT(
-  </emphasis><emphasis>recset, condition </emphasis><emphasis role="bold">[
-  </emphasis><emphasis>, message </emphasis><emphasis role="bold">] [
-  </emphasis>, <emphasis role="bold">FAIL ] [ </emphasis>, <emphasis
-  role="bold">CONST ])</emphasis><emphasis role="bold"><indexterm>
+    </indexterm> ])</emphasis></para>
+
+  <para><emphasis role="bold">ASSERT( </emphasis><emphasis>recset, condition
+  </emphasis><emphasis role="bold">[ </emphasis><emphasis>, message
+  </emphasis><emphasis role="bold">] [ </emphasis>, <emphasis role="bold">FAIL
+  ] [ </emphasis>, <emphasis role="bold">CONST ])</emphasis><emphasis
+  role="bold"><indexterm>
       <primary>ASSERT function</primary>
     </indexterm></emphasis></para>
 
@@ -130,5 +132,6 @@ END;
 OUTPUT(PROJECT(ds, t(LEFT)));
 </programlisting>
 
-  <para>See Also: <link linkend="FAIL">FAIL</link>, <link linkend="ERROR">ERROR</link></para>
+  <para>See Also: <link linkend="FAIL">FAIL</link>, <link
+  linkend="ERROR">ERROR</link></para>
 </sect1>

docs/ECLLanguageReference/ECLR_mods/ParSppt-PARSPattrn.xml

@@ -153,7 +153,7 @@
 
     <informaltable colsep="0" frame="none" rowsep="0">
       <tgroup cols="2">
-        <colspec align="left" colwidth="122.40pt" />
+        <colspec align="left" colwidth="125.50pt" />
 
         <colspec />
 
@@ -397,27 +397,29 @@
             <entry><emphasis role="bold">PATTERN</emphasis>('<emphasis>regular
             expression</emphasis>')</entry>
 
-            <entry><programlisting role="tab">Define a pattern using a <emphasis>regular expression </emphasis>built from the following 
-supported syntax elements: 
- (x)                        Grouping (not used for matching) 
- x|y                        Alteratives x or y 
- xy                         Concatenation of x and y. 
- x* x*?                     Zero or more. Greedy and minimal versions. 
- x+ x+?                     One or more. Greedy and minimal versions. 
- x? x??                     Zero or one. Greedy and minimal versions. 
- x{m} x{m,} x{m,n}          Bounded repeats, also minimal versions 
- [0-9abcdef]                A set of characters (may use ^ for exclusion list) 
- (?=…) (?!...)             Look ahead assertion 
- (?&lt;=…) (?&lt;!...)           Look behind assertion</programlisting><!--*** Note this and the following row entries have been monospace optimized for PDF/HTML*** --></entry>
+            <entry><programlisting role="tab">Define a pattern using a <emphasis>regular expression </emphasis>built from
+the following supported syntax elements:
+ (x)                Grouping (not used for matching)
+ x|y                Alteratives x or y
+ xy                 Concatenation of x and y.
+ x* x*?           Zero or more. Greedy and minimal versions.
+ x+ x+?           One or more. Greedy and minimal versions.
+ x? x??            Zero or one. Greedy and minimal versions.
+ x{m} x{m,} x{m,n}   Bounded repeats, also minimal versions
+ [0-9abcdef]    A set of characters
+                        (may use ^ for exclusion list)
+ (?=…) (?!...)     Look ahead assertion
+ (?&lt;=…) (?&lt;!...)   Look behind assertion</programlisting><!--*** Note this and the following row entries have been monospace optimized for PDF/HTML*** --></entry>
           </row>
 
           <row>
             <entry></entry>
 
-            <entry><programlisting role="tab">The following character class expressions are supported (inside sets): 
-[:alnum:]     [:cntrl:]     [:lower:]     [:upper:]     [:space:] 
-[:alpha:]     [:digit:]     [:print:]     [:blank:]     [:graph:] 
-[:punct:]     [:xdigit:]</programlisting></entry>
+            <entry><programlisting role="tab">The following character class expressions are supported
+(inside sets):
+[:alnum:]  [:cntrl:]  [:lower:]  [:upper:]  [:space:]
+[:alpha:]  [:digit:]  [:print:]  [:blank:]  [:graph:]
+[:punct:]  [:xdigit:]</programlisting></entry>
           </row>
 
           <row>
@@ -429,7 +431,7 @@ supported syntax elements:
     Collating symbols      [.ch.] 
     Equivalence<indexterm>
                   <primary>Equivalence</primary>
-                </indexterm> class      [=e=]</programlisting></entry>
+                </indexterm> class       [=e=]</programlisting></entry>
           </row>
 
           <row>

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -292,8 +292,8 @@ ecl deploy --target=roxie --name=FindPersonService libW20120224-125557.so
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -513,8 +513,8 @@ ecl publish --target=roxie --name=FindPersonService --no-activate findperson.ecl
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -638,8 +638,8 @@ ecl unpublish roxie "FindpersonService*"
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -785,8 +785,8 @@ ecl run --target=thor --input="&lt;request&gt;&lt;LName&gt;JONES&lt;/LName&gt;&l
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -922,8 +922,8 @@ ecl run --target=thor --input="&lt;request&gt;&lt;LName&gt;JONES&lt;/LName&gt;&l
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1005,8 +1005,8 @@ ecl run --target=thor --input="&lt;request&gt;&lt;LName&gt;JONES&lt;/LName&gt;&l
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1137,8 +1137,8 @@ ecl queries list roxie --target=roxie --show=A </programlisting>
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1189,9 +1189,9 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
 
                   <entry>Copies a query from one queryset to another. A query
                   can be copied from one HPCC environment to another by using
-                  a path which begins with '//' followed by the IP and Port of
-                  the source EclWatch and then followed by the source queryset
-                  and query.</entry>
+                  a path which begins with '//' followed by the IP or hostname
+                  and Port of the source EclWatch and then followed by the
+                  source queryset and query.</entry>
                 </row>
 
                 <row>
@@ -1294,8 +1294,8 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1417,8 +1417,8 @@ ecl queries copy //192.168.1.10:8010/thor/findperson thor
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1509,7 +1509,7 @@ ecl packagemap add roxie mypackagemap.pkg --overwrite
                 </row>
 
                 <row>
-                  <entry> -A, --activate</entry>
+                  <entry>-A, --activate</entry>
 
                   <entry>Activates packagemap</entry>
                 </row>
@@ -1523,8 +1523,8 @@ ecl packagemap add roxie mypackagemap.pkg --overwrite
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1613,8 +1613,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>--daliip</entry>
 
-                  <entry>IP Address of of the source Dali to use for file
-                  lookups</entry>
+                  <entry>IP Address or hostname of of the source Dali to use
+                  for file lookups</entry>
                 </row>
 
                 <row>
@@ -1626,8 +1626,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1701,8 +1701,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1784,8 +1784,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1866,8 +1866,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -1951,8 +1951,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -2037,8 +2037,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -2102,8 +2102,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -2167,8 +2167,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -2232,8 +2232,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>
@@ -2297,8 +2297,8 @@ ecl packagemap copyFiles myroxie  mypackagemap.pkg --daliip=192.168.1.3
                 <row>
                   <entry>-s, --server</entry>
 
-                  <entry>The IP Address of ESP server running eclwatch
-                  services</entry>
+                  <entry>The IP Address or hostname of ESP server running
+                  eclwatch services</entry>
                 </row>
 
                 <row>

docs/IMDB/IMDB.xml

@@ -579,7 +579,7 @@ Blankline</programlisting></para>
 
               <mediaobject>
                 <imageobject>
-                  <imagedata fileref="images/IMDB_06.jpg" />
+                  <imagedata fileref="images/IMDB_06_new.jpg" />
                 </imageobject>
               </mediaobject>
             </figure>
@@ -604,7 +604,7 @@ END;
           </listitem>
         </itemizedlist>
 
-        <sect3>
+        <sect3 role="brk">
           <title>Examine the Data</title>
 
           <para>In this section, we will look at the data and determine if
@@ -612,6 +612,34 @@ END;
           the development process where we convert the raw data into a form we
           can actually use.</para>
 
+          <variablelist>
+            <varlistentry>
+              <term>Note:</term>
+
+              <listitem>
+                <para>The IMDB.FileActors.ecl file specifies the size of the
+                header in the files (actors.list and actresses.list.) The
+                HEADING() value in the example code was accurate at the time
+                we downloaded the IMDB data, but could change at any time. We
+                suggest opening in a text editor and checking the line number
+                where the header ends and actual data begins (as shown
+                below).</para>
+              </listitem>
+            </varlistentry>
+          </variablelist>
+
+          <figure>
+            <title>actors.list in text editor</title>
+
+            <mediaobject>
+              <imageobject>
+                <imagedata fileref="images/IMDB_fileheading.jpg" />
+              </imageobject>
+            </mediaobject>
+          </figure>
+
+          <para></para>
+
           <itemizedlist mark="bullet">
             <listitem>
               <para>Open a new Builder window (CTRL+N) and write the following

docs/Installing_and_RunningTheHPCCPlatform/Installing_and_RunningTheHPCCPlatform.xml

@@ -2711,7 +2711,7 @@ OUTPUT(ValidWords)
 
 sudo /sbin/service dafilesrv start</programlisting><property><!--*** not for publication
 cd /var/lib/HPCCSystems/mythor
-source /opt/HPCCSystems/sbin/hpcc_setenv 
+...
 init_stop_thor
 init_start_thor--></property></para>
         </listitem>

docs/RuningHPCCinAmazonWebServicesEC2/RuningHPCCinAmazonWebServicesEC2.xml

@@ -464,10 +464,9 @@
             </listitem>
 
             <listitem>
-              <para>Start Configuration Manager using these commands:</para>
+              <para>Start Configuration Manager using this command:</para>
 
-              <programlisting>source /opt/HPCCSystems/sbin/hpcc_setenv
-sudo /opt/HPCCSystems/sbin/configmgr</programlisting>
+              <programlisting>sudo /opt/HPCCSystems/sbin/configmgr</programlisting>
             </listitem>
 
             <listitem>
@@ -1784,11 +1783,10 @@ sudo /opt/HPCCSystems/sbin/configmgr</programlisting>
           </listitem>
 
           <listitem>
-            <para>At the command prompt, enter the following commands to start
+            <para>At the command prompt, enter the following command to start
             Configuration Manager.</para>
 
-            <para><emphasis></emphasis><programlisting>source /opt/HPCCSystems/sbin/hpcc_setenv
-sudo /opt/HPCCSystems/sbin/configmgr</programlisting><emphasis></emphasis></para>
+            <para><emphasis></emphasis><programlisting>sudo /opt/HPCCSystems/sbin/configmgr</programlisting><emphasis></emphasis></para>
           </listitem>
 
           <listitem>

docs/UsingConfigManager/UsingConfigManager.xml

@@ -415,8 +415,7 @@ sudo -u hpcc cp /etc/HPCCSystems/source/NewEnvironment.xml /etc/HPCCSystems/envi
             the first node is considered the head node and is used for this
             task, but this is up to you).</para>
 
-            <programlisting>source /opt/HPCCSystems/sbin/hpcc_setenv
-sudo configmgr</programlisting>
+            <programlisting>sudo configmgr</programlisting>
 
             <para><graphic
             fileref="images/gs_img_configmgrStart.jpg" /></para>

ecl/ecl-package/ecl-package.cpp

@@ -599,114 +599,6 @@ private:
 };
 
 
-
-class EclCmdPackageCopyFiles : public EclCmdCommon
-{
-public:
-    EclCmdPackageCopyFiles() :optOverWrite (false)
-    {
-    }
-    virtual bool parseCommandLineOptions(ArgvIterator &iter)
-    {
-        if (iter.done())
-        {
-            usage();
-            return false;
-        }
-
-        for (; !iter.done(); iter.next())
-        {
-            const char *arg = iter.query();
-            if (*arg!='-')
-            {
-                if (optFileName.isEmpty())
-                    optFileName.set(arg);
-                else
-                {
-                    fprintf(stderr, "\nargument is already defined %s\n", arg);
-                    return false;
-                }
-                continue;
-            }
-            if (iter.matchOption(optDaliIp, ECLOPT_DALIIP))
-                continue;
-            if (iter.matchFlag(optOverWrite, ECLOPT_OVERWRITE))
-                continue;
-            if (EclCmdCommon::matchCommandLineOption(iter, true)!=EclCmdOptionMatch)
-                return false;
-        }
-        return true;
-    }
-    virtual bool finalizeOptions(IProperties *globals)
-    {
-        if (!EclCmdCommon::finalizeOptions(globals))
-        {
-            usage();
-            return false;
-        }
-        StringBuffer err;
-        if (optFileName.isEmpty())
-            err.append("\n ... Missing package file name\n\n");
-        else if (optTarget.isEmpty())
-            err.append("\n ... Specify a process name\n\n");
-
-        if (err.length())
-        {
-            fprintf(stdout, "%s", err.str());
-            usage();
-            return false;
-        }
-        return true;
-    }
-    virtual int processCMD()
-    {
-        Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
-        StringBuffer pkgInfo;
-        pkgInfo.loadFile(optFileName);
-
-        fprintf(stdout, "\n ... looking up files in packagemap to see what needs copying\n\n");
-
-        Owned<IClientCopyFilesRequest> request = packageProcessClient->createCopyFilesRequest();
-        request->setInfo(pkgInfo);
-        request->setTarget(optTarget);
-        request->setPackageName(optFileName);
-        request->setOverWrite(optOverWrite);
-        if (!optDaliIp.isEmpty())
-            request->setDaliIp(optDaliIp.get());
-
-        Owned<IClientCopyFilesResponse> resp = packageProcessClient->CopyFiles(request);
-        if (resp->getExceptions().ordinality())
-            outputMultiExceptions(resp->getExceptions());
-
-        return 0;
-    }
-
-    virtual void usage()
-    {
-        fputs("\nUsage:\n"
-                    "\n"
-                    "The 'copyFiles' command will copy any file listed in the packages contained \n"
-                    "in the packagemap file that are not currently known to the cluster.\n"
-                    "This will NOT load the package information \n"
-                    "\n"
-                    "ecl packagemap copyFiles [options] <target> <filename>\n"
-                    " Options:\n"
-                    "   -O, --overwrite             overwrite existing information\n"
-                    "  --daliip=<daliip>            ip of the source dali to use for file lookups\n"
-                    "   <target>                    name of target to use when adding package information\n"
-                    "   <filename>                  name of file containing package information\n",
-                    stdout);
-
-        EclCmdCommon::usage();
-    }
-private:
-    StringAttr optFileName;
-    StringAttr optTarget;
-    StringAttr optDaliIp;
-    StringBuffer pkgInfo;
-    bool optOverWrite;
-};
-
 IEclCommand *createPackageSubCommand(const char *cmdname)
 {
     if (!cmdname || !*cmdname)
@@ -723,8 +615,6 @@ IEclCommand *createPackageSubCommand(const char *cmdname)
         return new EclCmdPackageInfo();
     if (strieq(cmdname, "list"))
         return new EclCmdPackageList();
-    if (strieq(cmdname, "copyFiles"))
-        return new EclCmdPackageCopyFiles();
     return NULL;
 }
 

ecl/hql/hqlexpr.cpp

@@ -7161,6 +7161,9 @@ void CFileContents::ensureLoaded()
         throw MakeStringException(1, "File %s could not be opened", file->queryFilename());
 
     offset_t size = io->size();
+    if (size == (offset_t)-1)
+        throw MakeStringException(1, "File %s could not be read", file->queryFilename());
+
     size32_t sizeToRead = (size32_t)size;
     if (sizeToRead != size)
         throw MakeStringException(1, "File %s is larger than 4Gb", file->queryFilename());

ecl/hql/hqlgram.y

@@ -852,7 +852,8 @@ paramType
                         }
     | _LINKCOUNTED_ ROW {
                             IHqlExpression* record = queryNullRecord();
-                            $$.setType(setLinkCountedAttr(makeRowType(record->getType()), true));
+                            Owned<ITypeInfo> rowType = makeRowType(record->getType());
+                            $$.setType(setLinkCountedAttr(rowType, true));
                             $$.setPosition($1);
                         }
     | abstractModule

ecl/hql/hqlgram2.cpp

@@ -2222,8 +2222,10 @@ void HqlGram::addToActiveRecord(IHqlExpression * newField)
     //it means fields from ifblocks are inserted twice into the symbol table - and we still need to expose this internal function
     OwnedHqlExpr self = getSelfScope();
     assertex(self);
-    CHqlRecord *currentRecord = QUERYINTERFACE(self.get(), CHqlRecord);
-    if (currentRecord != &topRecord)
+
+    CHqlRecord *currentRecord = QUERYINTERFACE(self->queryRecord(), CHqlRecord);
+    //Protect against adding fields to closed records (can only occur after errors).
+    if ((currentRecord != &topRecord) && !currentRecord->isExprClosed())
         currentRecord->insertSymbols(newField);
 }
 

ecllibrary/std/system/Thorlib.ecl

@@ -18,6 +18,7 @@ varstring daliServer() : once, ctxmethod, entrypoint='getDaliServers';
 varstring cluster() : once, ctxmethod, entrypoint='getClusterName';
 varstring getExpandLogicalName(const varstring name) : pure, ctxmethod, entrypoint='getExpandLogicalName';
 varstring group() : once, ctxmethod, entrypoint='getGroupName';
+varstring platform() : pure ,ctxmethod, entrypoint='getPlatform';
     END;
 
 RETURN MODULE
@@ -68,6 +69,12 @@ export getExpandLogicalName(const varstring name) := externals.getExpandLogicalN
 export cluster() := externals.cluster();
 
 /*
+ * Returns the platform the query is currently executing on.
+ */
+
+export platform() := externals.platform();
+
+/*
  * The following are either unused, or should be replaced with a different syntax.
  
 export getenv(const varstring name, const varstring defaultValue) := externals.getenv(name, defaultValue);

esp/eclwatch/ws_XSLT/wuidcommon.xslt

@@ -1075,7 +1075,7 @@
         <xsl:if test="number(IsSupplied)"> supplied</xsl:if>
       </td>
      <xsl:choose>
-       <xsl:when test="number(ShowFileContent) and string-length(Link)">
+       <xsl:when test="((string-length(FileName) &lt; 1) or number(ShowFileContent)) and string-length(Link)">
           <td>
             <a href="javascript:void(0);" onclick="getLink(document.getElementById('ECL_Result_{position()}'), '/WsWorkunits/WUResult?Wuid={$wuid}&amp;Sequence={Link}');return false;">
               <xsl:value-of select="Value"/>
@@ -1254,25 +1254,25 @@
       </xsl:if>
       <xsl:if test="starts-with(Type, 'ThorLog')">
         <td>
-          <a href="/WsWorkunits/WUFile/ThorLog?Wuid={$wuid}&amp;Process={Description}&amp;Type={Type}"
+          <a href="/WsWorkunits/WUFile/ThorLog?Wuid={$wuid}&amp;Name={Name}&amp;Type={Type}"
                         >
             thormaster.log: <xsl:value-of select="Name"/>
           </a>
         </td>
         <td>
-          <a href="javascript:void(0)" onclick="getOptions('thormaster.log', '/WsWorkunits/WUFile/ThorLog?Wuid={$wuid}&amp;Process={Description}&amp;Type={Type}', false); return false;">
+          <a href="javascript:void(0)" onclick="getOptions('thormaster.log', '/WsWorkunits/WUFile/ThorLog?Wuid={$wuid}&amp;Name={Name}&amp;Type={Type}', false); return false;">
             download
           </a>
         </td>
       </xsl:if>
       <xsl:if test="Type = 'EclAgentLog'">
         <td>
-          <a href="/WsWorkunits/WUFile/EclAgentLog?Wuid={$wuid}&amp;Process={Description}&amp;Type=EclAgentLog">
+          <a href="/WsWorkunits/WUFile/EclAgentLog?Wuid={$wuid}&amp;Name={Name}&amp;Type=EclAgentLog">
               eclagent.log: <xsl:value-of select="Name"/>
           </a>
         </td>
         <td>
-          <a href="javascript:void(0)" onclick="getOptions('eclagent.log', '/WsWorkunits/WUFile/EclAgentLog?Wuid={$wuid}&amp;Process={Description}&amp;Type=EclAgentLog', false); return false;">
+          <a href="javascript:void(0)" onclick="getOptions('eclagent.log', '/WsWorkunits/WUFile/EclAgentLog?Wuid={$wuid}&amp;Name={Name}&amp;Type=EclAgentLog', false); return false;">
             download
           </a>
         </td>

esp/scm/ws_packageprocess.ecm

@@ -103,21 +103,6 @@ ESPresponse [exceptions_inline] ListPackageResponse
     ESParray<ESPstruct PackageListMapData> PkgListMapData;
 };
 
-ESPrequest CopyFilesRequest
-{
-    string Target;
-    string PackageName;
-    string DaliIp;
-    string Info;
-    bool OverWrite;
-};
-
-
-ESPresponse [exceptions_inline] CopyFilesResponse
-{
-    ESPstruct BasePackageStatus status;
-};
-
 ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
@@ -127,7 +112,6 @@ ESPservice [version("1.00"), default_client_version("1.00"), exceptions_inline("
     ESPmethod DeActivatePackage(DeActivatePackageRequest, DeActivatePackageResponse);
     ESPmethod ListPackage(ListPackageRequest, ListPackageResponse);
     ESPmethod GetPackage(GetPackageRequest, GetPackageResponse);
-    ESPmethod CopyFiles(CopyFilesRequest, CopyFilesResponse);
 };
 
 SCMexportdef(WsPackageProcess);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -167,65 +167,6 @@ bool cloneFileInfoToDali(StringArray &fileNames, const char *lookupDaliIp, const
     return cloneFileInfoToDali(fileNames, lookupDaliIp, clusterInfo, overWrite, userdesc);
 }
 
-bool addFileInfoToDali(const char *logicalname, const char *lookupDaliIp, const char *target, bool overwrite, IUserDescriptor* userdesc, StringBuffer &host, short port, StringBuffer &msg)
-{
-    bool retval = true;
-    try
-    {
-        if (!overwrite)
-        {
-            if (isFileKnownOnCluster(logicalname, lookupDaliIp, target, userdesc))
-                return true;
-        }
-
-        StringBuffer user;
-        StringBuffer password;
-
-        if (userdesc)
-        {
-            userdesc->getUserName(user);
-            userdesc->getPassword(password);
-        }
-
-        Owned<IClientFileSpray> fs;
-        fs.setown(createFileSprayClient());
-        fs->setUsernameToken(user.str(), password.str(), NULL);
-
-        VStringBuffer url("http://%s:%d/FileSpray", host.str(), port);
-        fs->addServiceUrl(url.str());
-
-        bool isRoxie = isRoxieProcess(target);
-
-        Owned<IClientCopy> req = fs->createCopyRequest();
-        req->setSourceLogicalName(logicalname);
-        req->setDestLogicalName(logicalname);
-        req->setDestGroup(target);
-        req->setSuperCopy(false);
-        if (isRoxie)
-            req->setDestGroupRoxie("Yes");
-
-        req->setSourceDali(lookupDaliIp);
-
-        req->setSrcusername(user);
-        req->setSrcpassword(password);
-        req->setOverwrite(overwrite);
-
-        Owned<IClientCopyResponse> resp = fs->Copy(req);
-    }
-    catch(IException *e)
-    {
-        e->errorMessage(msg);
-        DBGLOG("ERROR = %s", msg.str());
-        e->Release();  // report the error later if needed
-        retval = false;
-    }
-    catch(...)
-    {
-        retval = false;
-    }
-
-    return retval;
-}
 
 void makePackageActive(IPropertyTree *pkgSetRegistry, IPropertyTree *pkgSetTree, const char *setName)
 {
@@ -324,40 +265,6 @@ void addPackageMapInfo(IPropertyTree *pkgSetRegistry, const char *target, const
         pkgSetTree->setPropBool("@active", false);
 }
 
-void copyPackageSubFiles(IPropertyTree *packageInfo, const char *target, const char *defaultLookupDaliIp, bool overwrite, IUserDescriptor* userdesc, StringBuffer &host, short port)
-{
-    Owned<IPropertyTreeIterator> iter = packageInfo->getElements("Package");
-    ForEach(*iter)
-    {
-        IPropertyTree &item = iter->query();
-        StringBuffer lookupDaliIp;
-        lookupDaliIp.append(item.queryProp("@daliip"));
-        if (lookupDaliIp.length() == 0)
-            lookupDaliIp.append(defaultLookupDaliIp);
-        if (lookupDaliIp.length() == 0)
-        {
-            StringAttr superfile(item.queryProp("@id"));
-            throw MakeStringException(PKG_MISSING_DALI_LOOKUP_IP, "Could not lookup SubFiles in package %s because no remote dali ip was specified", superfile.get());
-        }
-        Owned<IPropertyTreeIterator> super_iter = item.getElements("SuperFile");
-        ForEach(*super_iter)
-        {
-            IPropertyTree &supertree = super_iter->query();
-            Owned<IPropertyTreeIterator> sub_iter = supertree.getElements("SubFile");
-            ForEach(*sub_iter)
-            {
-                IPropertyTree &subtree = sub_iter->query();
-                StringAttr subid = subtree.queryProp("@value");
-                if (subid.length())
-                {
-                    StringBuffer msg;
-                    addFileInfoToDali(subid.get(), lookupDaliIp, target, overwrite, userdesc, host, port, msg);
-                }
-            }
-        }
-    }
-}
-
 void getPackageListInfo(IPropertyTree *mapTree, IEspPackageListMapData *pkgList)
 {
     pkgList->setId(mapTree->queryProp("@id"));
@@ -624,36 +531,3 @@ bool CWsPackageProcessEx::onGetPackage(IEspContext &context, IEspGetPackageReque
     resp.setInfo(info);
     return true;
 }
-
-bool CWsPackageProcessEx::onCopyFiles(IEspContext &context, IEspCopyFilesRequest &req, IEspCopyFilesResponse &resp)
-{
-    resp.updateStatus().setCode(0);
-    StringBuffer info(req.getInfo());
-    StringAttr target(req.getTarget());
-    StringAttr pkgName(req.getPackageName());
-    StringAttr lookupDaliIp(req.getDaliIp());
-
-    if (target.length() == 0)
-        throw MakeStringException(PKG_MISSING_PARAM, "CWsPackageProcessEx::onCopyFiles process parameter not set.");
-
-    Owned<IUserDescriptor> userdesc;
-    const char *user = context.queryUserId();
-    const char *password = context.queryPassword();
-    if (user && *user && *password && *password)
-    {
-        userdesc.setown(createUserDescriptor());
-        userdesc->set(user, password);
-    }
-
-    StringBuffer host;
-    short port;
-    context.getServAddress(host, port);
-
-    Owned<IPropertyTree> packageTree = createPTreeFromXMLString(info.str());
-    copyPackageSubFiles(LINK(packageTree), target, lookupDaliIp.get(), req.getOverWrite(), userdesc, host, port);
-
-    StringBuffer msg;
-    msg.append("Successfully loaded ").append(pkgName.get());
-    resp.updateStatus().setDescription(msg.str());
-    return true;
-}

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -51,7 +51,6 @@ public:
     virtual bool onDeActivatePackage(IEspContext &context, IEspDeActivatePackageRequest &req, IEspDeActivatePackageResponse &resp);
     virtual bool onListPackage(IEspContext &context, IEspListPackageRequest &req, IEspListPackageResponse &resp);
     virtual bool onGetPackage(IEspContext &context, IEspGetPackageRequest &req, IEspGetPackageResponse &resp);
-    virtual bool onCopyFiles(IEspContext &context, IEspCopyFilesRequest &req, IEspCopyFilesResponse &resp);
 };
 
 #endif //_ESPWIZ_ws_packageprocess_HPP__

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -486,28 +486,18 @@ void WsWuInfo::getHelpers(IEspECLWorkunit &info, unsigned flags)
 
         if (cw->getWuidVersion() > 0)
         {
-            Owned<IStringIterator> eclAgentInstances = cw->getProcesses("EclAgent");
-            ForEach (*eclAgentInstances)
+            Owned<IStringIterator> eclAgentLogs = cw->getLogs("EclAgent");
+            ForEach (*eclAgentLogs)
             {
-                SCMStringBuffer processName;
-                eclAgentInstances->str(processName);
-                if (processName.length() < 1)
+                SCMStringBuffer logName;
+                eclAgentLogs->str(logName);
+                if (logName.length() < 1)
                     continue;
 
-                Owned<IStringIterator> eclAgentLogs = cw->getLogs("EclAgent", processName.str());
-                ForEach (*eclAgentLogs)
-                {
-                    SCMStringBuffer logName;
-                    eclAgentLogs->str(logName);
-                    if (logName.length() < 1)
-                        continue;
-
-                    Owned<IEspECLHelpFile> h= createECLHelpFile("","");
-                    h->setName(logName.str());
-                    h->setDescription(processName.str());
-                    h->setType(File_EclAgentLog);
-                    helpers.append(*h.getLink());
-                }
+                Owned<IEspECLHelpFile> h= createECLHelpFile("","");
+                h->setName(logName.str());
+                h->setType(File_EclAgentLog);
+                helpers.append(*h.getLink());
             }
         }
         else // legacy wuid
@@ -915,14 +905,17 @@ unsigned WsWuInfo::getWorkunitThorLogInfo(IArrayOf<IEspECLHelpFile>& helpers, IE
 
         unsigned numberOfSlaves = clusterInfo->getSize();
 
+        BoolHash uniqueProcesses;
         Owned<IStringIterator> thorInstances = cw->getProcesses("Thor");
         ForEach (*thorInstances)
         {
             SCMStringBuffer processName;
             thorInstances->str(processName);
-            if (processName.length() < 1)
+            if ((processName.length() < 1) || uniqueProcesses.getValue(processName.str()))
                 continue;
 
+            uniqueProcesses.setValue(processName.str(), true);
+
             StringBuffer groupName;
             getClusterThorGroupName(groupName, processName.str());
 
@@ -1608,32 +1601,23 @@ void appendIOStreamContent(MemoryBuffer &mb, IFileIOStream *ios, bool forDownloa
     }
 }
 
-void WsWuInfo::getWorkunitEclAgentLog(const char* eclAgentInstance, MemoryBuffer& buf)
+void WsWuInfo::getWorkunitEclAgentLog(const char* fileName, MemoryBuffer& buf)
 {
-    SCMStringBuffer logname;
-    Owned<IStringIterator> eclAgentLogs = cw->getLogs("EclAgent", eclAgentInstance);
-    ForEach (*eclAgentLogs)
-    {
-        eclAgentLogs->str(logname);
-        if (logname.length() > 0)
-            break;
-    }
-
-    unsigned pid = cw->getAgentPID();
-    if(logname.length() == 0)
-        throw MakeStringException(ECLWATCH_ECLAGENT_LOG_NOT_FOUND,"EclAgent log file not available for workunit %s.", wuid.str());
-    Owned<IFile> rFile = createIFile(logname.str());
+    if(!fileName || !*fileName)
+        throw MakeStringException(ECLWATCH_ECLAGENT_LOG_NOT_FOUND,"Log file not specified");
+    Owned<IFile> rFile = createIFile(fileName);
     if(!rFile)
-        throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE, "Cannot open file %s.", logname.str());
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE, "Cannot open file %s.", fileName);
     OwnedIFileIO rIO = rFile->openShared(IFOread,IFSHfull);
     if(!rIO)
-        throw MakeStringException(ECLWATCH_CANNOT_READ_FILE, "Cannot read file %s.", logname.str());
+        throw MakeStringException(ECLWATCH_CANNOT_READ_FILE, "Cannot read file %s.", fileName);
     OwnedIFileIOStream ios = createBufferedIOStream(rIO);
 
     StringBuffer line;
     bool eof = false;
     bool wuidFound = false;
 
+    unsigned pid = cw->getAgentPID();
     VStringBuffer pidstr(" %5d ", pid);
     char const * pidchars = pidstr.str();
     while(!eof)
@@ -1673,25 +1657,21 @@ void WsWuInfo::getWorkunitEclAgentLog(const char* eclAgentInstance, MemoryBuffer
             buf.append(line.length(), line.str());
         }
     }
+
+    if (buf.length() < 1)
+        buf.append(47, "(Not found a log line related to this workunit)");
 }
 
-void WsWuInfo::getWorkunitThorLog(const char* processName, MemoryBuffer& buf)
+void WsWuInfo::getWorkunitThorLog(const char* fileName, MemoryBuffer& buf)
 {
-    SCMStringBuffer logname;
-    Owned<IStringIterator> thorLogs = cw->getLogs("Thor", processName);
-    ForEach (*thorLogs)
-    {
-        thorLogs->str(logname);
-        if (logname.length() > 0)
-            break;
-    }
-
-    Owned<IFile> rFile = createIFile(logname.str());
+    if(!fileName || !*fileName)
+        throw MakeStringException(ECLWATCH_ECLAGENT_LOG_NOT_FOUND,"Log file not specified");
+    Owned<IFile> rFile = createIFile(fileName);
     if (!rFile)
-        throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE,"Cannot open file %s.",logname.str());
+        throw MakeStringException(ECLWATCH_CANNOT_OPEN_FILE,"Cannot open file %s.",fileName);
     OwnedIFileIO rIO = rFile->openShared(IFOread,IFSHfull);
     if (!rIO)
-        throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read file %s.",logname.str());
+        throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read file %s.",fileName);
     OwnedIFileIOStream ios = createBufferedIOStream(rIO);
 
     StringBuffer line;

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2633,7 +2633,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             }
             else if (strncmp(req.getType(), File_ThorLog, 7) == 0)
             {
-                winfo.getWorkunitThorLog(req.getProcess(), mb);
+                winfo.getWorkunitThorLog(req.getName(), mb);
                 openSaveFile(context, opt, "thormaster.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_ThorSlaveLog,req.getType()))
@@ -2646,7 +2646,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             }
             else if (strieq(File_EclAgentLog,req.getType()))
             {
-                winfo.getWorkunitEclAgentLog(req.getProcess(), mb);
+                winfo.getWorkunitEclAgentLog(req.getName(), mb);
                 openSaveFile(context, opt, "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_XML,req.getType()))

initfiles/sbin/hpcc_setenv.in

@@ -46,6 +46,11 @@ HPCC_CONFIG=${HPCC_CONFIG:-${CONFIG_DIR}/${ENV_CONF_FILE}}
 #SECTION=${1:-DEFAULT}
 SECTION="DEFAULT"
 
+OIFS="${IFS}"
+unset IFS
+source /etc/profile
+IFS="${OIFS}"
+
 PATH_PREFIX=`cat ${HPCC_CONFIG} | sed -n "/\[${SECTION}\]/,/\[/p" | grep "^path *= *" | sed -e 's/^path *= *//'`
 
 export PID=`cat ${HPCC_CONFIG} | sed -n "/\[${SECTION}\]/,/\[/p" | grep "^pid *= *" | sed -e 's/^pid *= *//'`
@@ -156,5 +161,3 @@ else
         fi
     fi
 fi
-
-source /etc/profile

plugins/fileservices/fileservices.cpp

@@ -1041,18 +1041,23 @@ FILESERVICES_API bool FILESERVICES_CALL fsSuperFileExists(ICodeContext *ctx, con
 
 FILESERVICES_API void FILESERVICES_CALL fsDeleteSuperFile(ICodeContext *ctx, const char *lsuperfn,bool deletesub)
 {
-    // Note because deleting a superfile, not within transaction (currently)
+    IDistributedFileTransaction *transaction = ctx->querySuperFileTransaction();
+    Linked<IUserDescriptor> udesc = ctx->queryUserDescriptor();
     Owned<IDistributedSuperFile> file;
     StringBuffer lsfn;
     bool found = lookupSuperFile(ctx, lsuperfn, file, false, lsfn, false);
+    file.clear(); // MORE: this should really be exists(file)
+    StringBuffer s("DeleteSuperFile ('");
+    s.append(lsfn).appendf("')");
     if (found) {
-        CheckNotInTransaction(ctx,"DeleteSuperFile");
-        if (deletesub)
-            file->removeSubFile(NULL,true,true,false);
-        file->detach();
+        queryDistributedFileDirectory().removeSuperFile(lsfn.str(), deletesub, udesc, transaction);
+        if (transaction->active())
+            s.append(" action added to transaction");
+        else
+            s.append(" done");
+    } else {
+        s.append(" file not found");
     }
-    StringBuffer s("DeleteSuperFile ('");
-    s.append(lsfn).appendf("') %s",found?"done":"not found");
     WUmessage(ctx,ExceptionSeverityInformation,NULL,s.str());
     if (found)
         AuditMessage(ctx,"DeleteSuperFile",lsfn.str());

system/jlib/jarray.hpp

@@ -135,11 +135,7 @@ public:
     CopyArrayOf() { SELF::_init(); }
     ~CopyArrayOf();
     
-#ifdef __clang__
-    PARAM item(aindex_t pos) const;  // Clang's stricter template checking is not working with inline case. Should revisit for efficiency sometime
-#else
-    inline PARAM item(aindex_t pos) const             { assertex(SELF::isItem(pos)); return Array__Member2Param(((MEMBER *)AllocatorOf<sizeof(MEMBER)>::_head)[pos]);}
-#endif
+    inline PARAM item(aindex_t pos) const;
     PARAM tos(void) const;
     PARAM tos(aindex_t) const;
 
@@ -181,11 +177,7 @@ class ArrayOf : public OwningArrayOf<MEMBER, PARAM>
     typedef ArrayOf<MEMBER,PARAM> SELF;
 
 public:
-#ifdef __clang__
-    PARAM item(aindex_t pos) const;  // Clang's stricter template checking is not working with inline case. Should revisit for efficiency sometime
-#else
-    inline PARAM item(aindex_t pos) const             { assertex(SELF::isItem(pos)); return Array__Member2Param(((MEMBER *)AllocatorOf<sizeof(MEMBER)>::_head)[pos]);}
-#endif
+    inline PARAM item(aindex_t pos) const; 
     PARAM popGet();
     PARAM tos(void) const;
     PARAM tos(aindex_t) const;

system/jlib/jarray.tpp

@@ -123,13 +123,11 @@ void BaseArrayOf<MEMBER, PARAM>::sort(CompareFunc cf)
  *                            Master CopyArrays                         *
  ************************************************************************/
 
-#ifdef __clang__
 template <class MEMBER, class PARAM>
 PARAM CopyArrayOf<MEMBER, PARAM>::item(aindex_t pos) const
 {
    assertex(SELF::isItem(pos)); return Array__Member2Param(((MEMBER *)AllocatorOf<sizeof(MEMBER)>::_head)[pos]);
 }
-#endif
 
 template <class MEMBER, class PARAM>
 void CopyArrayOf<MEMBER, PARAM>::replace(PARAM it, aindex_t pos)
@@ -334,13 +332,11 @@ bool OwningArrayOf<MEMBER, PARAM>::zap(PARAM sought, bool nodel)
  *                            Master Ref Array                          *
  ************************************************************************/
 
-#ifdef __clang__
 template <class MEMBER, class PARAM>
 PARAM ArrayOf<MEMBER, PARAM>::item(aindex_t pos) const
 {
    assertex(SELF::isItem(pos)); return Array__Member2Param(((MEMBER *)AllocatorOf<sizeof(MEMBER)>::_head)[pos]);
 }
-#endif
 
 template <class MEMBER, class PARAM>
 PARAM ArrayOf<MEMBER, PARAM>::popGet()

testing/ecl/hthor/key/platform.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>hthor</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><x>hthor1</x></Row>
+ <Row><x>hthor2</x></Row>
+ <Row><x>hthor3</x></Row>
+ <Row><x>hthor4</x></Row>
+ <Row><x>hthor5</x></Row>
+</Dataset>

testing/ecl/key/normalize4.xml

@@ -0,0 +1,36 @@
+<Dataset name='Result 1'>
+ <Row><id>91824</id></Row>
+ <Row><id>91825</id></Row>
+ <Row><id>91826</id></Row>
+ <Row><id>91827</id></Row>
+ <Row><id>91828</id></Row>
+ <Row><id>91829</id></Row>
+ <Row><id>91830</id></Row>
+ <Row><id>91831</id></Row>
+ <Row><id>91832</id></Row>
+ <Row><id>91833</id></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>91824.0</id></Row>
+ <Row><id>91825.0</id></Row>
+ <Row><id>91826.0</id></Row>
+ <Row><id>91827.0</id></Row>
+ <Row><id>91828.0</id></Row>
+ <Row><id>91829.0</id></Row>
+ <Row><id>91830.0</id></Row>
+ <Row><id>91831.0</id></Row>
+ <Row><id>91832.0</id></Row>
+ <Row><id>91833.0</id></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>91824</id></Row>
+ <Row><id>91825</id></Row>
+ <Row><id>91826</id></Row>
+ <Row><id>91827</id></Row>
+ <Row><id>91828</id></Row>
+ <Row><id>91829</id></Row>
+ <Row><id>91830</id></Row>
+ <Row><id>91831</id></Row>
+ <Row><id>91832</id></Row>
+ <Row><id>91833</id></Row>
+</Dataset>

testing/ecl/key/platform.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>thor</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><x>thor1</x></Row>
+ <Row><x>thor2</x></Row>
+ <Row><x>thor3</x></Row>
+ <Row><x>thor4</x></Row>
+ <Row><x>thor5</x></Row>
+</Dataset>

testing/ecl/key/superfile7.xml

@@ -0,0 +1,83 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>true</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>4</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><i>1</i><id>A</id></Row>
+ <Row><i>1</i><id>B</id></Row>
+ <Row><i>1</i><id>C</id></Row>
+ <Row><i>2</i><id>D</id></Row>
+ <Row><i>2</i><id>E</id></Row>
+ <Row><i>3</i><id>F</id></Row>
+ <Row><i>3</i><id>G</id></Row>
+ <Row><i>3</i><id>H</id></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>true</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>4</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><i>1</i><id>A</id></Row>
+ <Row><i>1</i><id>B</id></Row>
+ <Row><i>1</i><id>C</id></Row>
+ <Row><i>2</i><id>D</id></Row>
+ <Row><i>2</i><id>E</id></Row>
+ <Row><i>3</i><id>F</id></Row>
+ <Row><i>3</i><id>G</id></Row>
+ <Row><i>3</i><id>H</id></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>true</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>true</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>true</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>true</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>true</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>4</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><i>1</i><id>A</id></Row>
+ <Row><i>1</i><id>B</id></Row>
+ <Row><i>1</i><id>C</id></Row>
+ <Row><i>2</i><id>D</id></Row>
+ <Row><i>2</i><id>E</id></Row>
+ <Row><i>3</i><id>F</id></Row>
+ <Row><i>3</i><id>G</id></Row>
+ <Row><i>3</i><id>H</id></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>false</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>false</Result_19></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><Result_20>false</Result_20></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>false</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><Result_22>false</Result_22></Row>
+</Dataset>

testing/ecl/key/superfile8.xml

@@ -0,0 +1,53 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+</Dataset>
+<Dataset name='Result 3'>
+</Dataset>
+<Dataset name='Result 4'>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>false</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>true</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>true</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>true</Result_8></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><Result_9>true</Result_9></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><Result_10>false</Result_10></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>true</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><Result_12>true</Result_12></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><Result_13>true</Result_13></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><Result_14>true</Result_14></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><Result_15>false</Result_15></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>false</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><Result_17>false</Result_17></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><Result_18>false</Result_18></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><Result_19>false</Result_19></Row>
+</Dataset>

testing/ecl/normalize4.ecl

@@ -0,0 +1,92 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+inRec := { unsigned id };
+doneRec := { unsigned4 execid };
+out1rec := { unsigned id; };
+out2rec := { real id; };
+
+dataset(doneRec) doSomethingNasty(DATASET(inRec) input) := BEGINC++
+  __lenResult = 4;
+  __result = rtlMalloc(8);
+  *(unsigned *)__result = 91823;
+ENDC++;
+
+dataset(out1Rec) extractResult1(doneRec done) := BEGINC++
+   const unsigned id = *(unsigned *)done;
+   const unsigned cnt = 10;
+   __lenResult = cnt * sizeof(unsigned __int64);
+   __result = rtlMalloc(__lenResult);
+   for (unsigned i=0; i < cnt; i++)
+       ((unsigned __int64 *)__result)[i] = id + i + 1;
+ENDC++;
+
+_LINKCOUNTED_ dataset(out2Rec) extractResult2(doneRec done) := BEGINC++
+   const unsigned id = *(unsigned *)done;
+   const unsigned cnt = 10;
+   __countResult = cnt;
+   __result = _resultAllocator->createRowset(cnt);
+   for (unsigned i=0; i < cnt; i++)
+   {
+       size32_t allocSize;
+        void * row = _resultAllocator->createRow(allocSize);
+        *(double *)row = id + i + 1;
+        __result[i] =  (byte *)_resultAllocator->finalizeRow(allocSize, row, allocSize);
+   }
+ENDC++;
+
+streamed dataset(out1Rec) extractResult3(doneRec done) := BEGINC++
+   class myStream : public IRowStream, public RtlCInterface
+   {
+    public:
+        myStream(IEngineRowAllocator * _allocator, unsigned _id) : allocator(_allocator), id(_id), idx(0) {}
+        RTLIMPLEMENT_IINTERFACE
+
+        virtual const void *nextRow()
+        {
+            if (idx >= 10)
+               return NULL;
+            size32_t allocSize;
+            void * row = allocator->createRow(allocSize);
+            *(unsigned __int64 *)row = id + ++idx;
+            return allocator->finalizeRow(allocSize, row, allocSize);
+        }
+        virtual void stop() {}
+    private:
+        unsigned id;
+        unsigned idx;
+        Linked<IEngineRowAllocator> allocator;
+    };
+    #body
+    const unsigned id = *(unsigned *)done;
+    return new myStream(_resultAllocator, id);
+ENDC++;
+
+ds := dataset([1,2,3,4], inRec);
+
+processed := doSomethingNasty(ds);
+
+out1 := NORMALIZE(processed, extractResult1(LEFT), transform(RIGHT));
+out2 := NORMALIZE(processed, extractResult2(LEFT), transform(RIGHT));
+out3 := NORMALIZE(processed, extractResult3(LEFT), transform(RIGHT));
+
+SEQUENTIAL(
+output(out1);
+output(out2);
+output(out3);
+);
+

testing/ecl/platform.ecl

@@ -0,0 +1,27 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import std.system.thorlib;
+
+output(thorlib.platform());
+
+pl := thorlib.platform() : independent;
+
+ds := NOFOLD(dataset([1,2,3,4,5],{ unsigned id; }));
+p := table(ds, { string x := thorlib.platform() + (string)id });
+output(p);
+

testing/ecl/roxie/key/platform.xml

@@ -0,0 +1,10 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>roxie</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><x>roxie1</x></Row>
+ <Row><x>roxie2</x></Row>
+ <Row><x>roxie3</x></Row>
+ <Row><x>roxie4</x></Row>
+ <Row><x>roxie5</x></Row>
+</Dataset>

testing/ecl/stepping.ecl

@@ -18,6 +18,9 @@
 //UseStandardFiles
 //nothor
 
+//Stepped global joins unsupported, see issue HPCC-8148
+//skip type==thorlcr TBD
+
 import lib_stringLib;
 
 MaxTerms            := TS_MaxTerms;

testing/ecl/stepping2.ecl

@@ -18,6 +18,8 @@
 //UseStandardFiles
 //nothor
 
+//Stepped global joins unsupported, see issue HPCC-8148
+//skip type==thorlcr TBD
 
 OUTPUT(SORTED(STEPPED(TS_WordIndex(keyed(kind = TS_kindType.TextEntry and word in ['boy', 'sheep'])), doc, segment, wpos), doc, segment, wpos, assert)) : independent;
 OUTPUT(SORTED(STEPPED(TS_WordIndex(keyed(kind = TS_kindType.TextEntry and word in ['b%%%', 'sheep'])), doc, segment, wpos), doc, segment, wpos, assert)) : independent;

testing/ecl/stepping3.ecl

@@ -22,5 +22,8 @@
 //varskip trans
 //nothor
 
+//Stepped global joins unsupported, see issue HPCC-8148
+//skip type==thorlcr TBD
+
 // should be equivalent to OUTPUT(SORT(DG_IndexFile(DG_firstname = 'DAVID'), DG_Prange));
 OUTPUT(STEPPED(DG_IndexFile(KEYED(DG_firstname = 'DAVID')), DG_Prange));

testing/ecl/stepping4.ecl

@@ -22,6 +22,9 @@
 //varskip trans
 //nothor
 
+//Stepped global joins unsupported, see issue HPCC-8148
+//skip type==thorlcr TBD
+
 boy := STEPPED(TS_WordIndex(keyed(kind = TS_kindType.TextEntry and word = 'boy')), doc);
 
 sheep := STEPPED(TS_WordIndex(keyed(kind = TS_kindType.TextEntry and word = 'sheep')), doc);

testing/ecl/superfile7.ecl

@@ -0,0 +1,87 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import Std.System.Thorlib;
+import Std.File AS FileServices;
+import Std.Str;
+// Super File regression test
+//noRoxie
+
+rec :=
+RECORD
+        integer i;
+    string1 id;
+END;
+
+ds1 := DATASET([{1,'A'}, {1,'B'}, {1,'C'}], rec);
+ds2 := DATASET([{2,'D'}, {2,'E'}], rec);
+ds3 := DATASET([{3,'F'}, {3,'G'}, {3,'H'}], rec);
+ds4 := DATASET([],rec);
+
+clusterLFNPrefix := thorlib.getExpandLogicalName('regress::');
+
+string stripPrefix(string qlfn) := IF (Str.Find(qlfn, clusterLFNprefix, 1) = 1, Str.FindReplace(qlfn, clusterLFNPrefix, ''), qlfn);
+
+
+SEQUENTIAL(
+  // Prepare
+  FileServices.DeleteSuperFile('regress::superfile7'),
+  OUTPUT(ds1,,'regress::subfile1',overwrite),
+  OUTPUT(ds2,,'regress::subfile2',overwrite),
+  OUTPUT(ds3,,'regress::subfile3',overwrite),
+  OUTPUT(ds4,,'regress::subfile4',overwrite),
+  FileServices.StartSuperFileTransaction(),
+  FileServices.CreateSuperFile('regress::superfile7'),
+  FileServices.AddSuperFile('regress::superfile7','regress::subfile1'),
+  FileServices.AddSuperFile('regress::superfile7','regress::subfile2'),
+  FileServices.AddSuperFile('regress::superfile7','regress::subfile3'),
+  FileServices.AddSuperFile('regress::superfile7','regress::subfile4'),
+  FileServices.FinishSuperFileTransaction(),
+  OUTPUT(FileServices.SuperFileExists('regress::superfile7')), // true
+  OUTPUT(FileServices.GetSuperFileSubCount('regress::superfile7')), // 4
+  OUTPUT(dataset ('regress::superfile7', rec, flat)),
+
+  // Delete Super + Rollback (keep subs)
+  FileServices.StartSuperFileTransaction(),
+  FileServices.DeleteSuperFile('regress::superfile7'),
+  FileServices.FinishSuperFileTransaction(true),    // rollback
+  OUTPUT(FileServices.SuperFileExists('regress::superfile7')), // true
+  OUTPUT(FileServices.GetSuperFileSubCount('regress::superfile7')), // 4
+  OUTPUT(dataset ('regress::superfile7', rec, flat)),
+
+  // Delete Super + Rollback (del subs, not really)
+  FileServices.StartSuperFileTransaction(),
+  FileServices.DeleteSuperFile('regress::superfile7'),
+  FileServices.FinishSuperFileTransaction(true),    // rollback
+  OUTPUT(FileServices.SuperFileExists('regress::superfile7')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile1')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile2')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile3')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile4')), // true
+  OUTPUT(FileServices.GetSuperFileSubCount('regress::superfile7')), // 4
+  OUTPUT(dataset ('regress::superfile7', rec, flat)),
+
+  // Delete Super + Commit (del subs, yes really)
+  FileServices.StartSuperFileTransaction(),
+  FileServices.DeleteSuperFile('regress::superfile7', true), // del subs
+  FileServices.FinishSuperFileTransaction(),        // commit
+  OUTPUT(FileServices.SuperFileExists('regress::superfile7')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile1')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile2')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile3')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile4')), // false
+);

testing/ecl/superfile8.ecl

@@ -0,0 +1,92 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import Std.System.Thorlib;
+import Std.File AS FileServices;
+import Std.Str;
+// Super File regression test
+//noRoxie
+
+rec :=
+RECORD
+        integer i;
+    string1 id;
+END;
+
+ds1 := DATASET([{1,'A'}, {1,'B'}, {1,'C'}], rec);
+ds2 := DATASET([{2,'D'}, {2,'E'}], rec);
+ds3 := DATASET([{3,'F'}, {3,'G'}, {3,'H'}], rec);
+ds4 := DATASET([],rec);
+
+clusterLFNPrefix := thorlib.getExpandLogicalName('regress::');
+
+string stripPrefix(string qlfn) := IF (Str.Find(qlfn, clusterLFNprefix, 1) = 1, Str.FindReplace(qlfn, clusterLFNPrefix, ''), qlfn);
+
+
+SEQUENTIAL(
+  // Prepare
+  FileServices.DeleteSuperFile('regress::superfile8'),
+  OUTPUT(ds1,,'regress::subfile5',overwrite),
+  OUTPUT(ds2,,'regress::subfile6',overwrite),
+  OUTPUT(ds3,,'regress::subfile7',overwrite),
+  OUTPUT(ds4,,'regress::subfile8',overwrite),
+
+  // Delete Super + Rollback (keep subs)
+  FileServices.StartSuperFileTransaction(),
+  FileServices.CreateSuperFile('regress::superfile8'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile5'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile6'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile7'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile8'),
+  FileServices.DeleteSuperFile('regress::superfile8'),
+  FileServices.FinishSuperFileTransaction(true),    // rollback
+  OUTPUT(FileServices.SuperFileExists('regress::superfile8')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile5')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile6')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile7')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile8')), // true
+
+  // Delete Super + Rollback (del subs, not really)
+  FileServices.StartSuperFileTransaction(),
+  FileServices.CreateSuperFile('regress::superfile8'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile5'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile6'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile7'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile8'),
+  FileServices.DeleteSuperFile('regress::superfile8'),
+  FileServices.FinishSuperFileTransaction(true),    // rollback
+  OUTPUT(FileServices.SuperFileExists('regress::superfile8')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile5')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile6')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile7')), // true
+  OUTPUT(FileServices.FileExists('regress::subfile8')), // true
+
+  // Delete Super + Commit (del subs, yes really)
+  FileServices.StartSuperFileTransaction(),
+  FileServices.CreateSuperFile('regress::superfile8'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile5'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile6'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile7'),
+  FileServices.AddSuperFile('regress::superfile8','regress::subfile8'),
+  FileServices.DeleteSuperFile('regress::superfile8', true), // del subs
+  FileServices.FinishSuperFileTransaction(),        // commit
+  OUTPUT(FileServices.SuperFileExists('regress::superfile8')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile5')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile6')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile7')), // false
+  OUTPUT(FileServices.FileExists('regress::subfile8')), // false
+);

thorlcr/activities/hashdistrib/thhashdistribslave.cpp

@@ -332,7 +332,7 @@ class CDistributorBase : public CSimpleInterface, implements IHashDistributor, i
                 SpinBlock b(doDedupLock);
                 if (dedupSamples<10)
                 {
-                    if (postCount<preCount*9/10);
+                    if (postCount<preCount*9/10)
                         dedupSuccesses++;
                     dedupSamples++;
                     ActPrintLog(owner.activity, "pre-dedup sample %d : %d unique out of %d, took: %d ms", dedupSamples, postCount, preCount, tookMs);
@@ -742,11 +742,11 @@ public:
         fixedEstSize = meta->querySerializedMeta()->getFixedSize();
         rowManager = activity->queryJob().queryRowManager();
 
-        unsigned defaultAllowSpill = activity->queryJob().getWorkUnitValueBool("allowSpillHashDist", globals->getPropBool("@allowSpillHashDist", true));
+        bool defaultAllowSpill = activity->queryJob().getWorkUnitValueBool("allowSpillHashDist", globals->getPropBool("@allowSpillHashDist", true));
         allowSpill = activity->queryContainer().queryXGMML().getPropBool("hint[@name=\"allow_spill\"]/@value", defaultAllowSpill);
         if (allowSpill)
             ActPrintLog(activity, "Using spilling buffer (will spill if overflows)");
-        writerPoolSize = activity->queryJob().getWorkUnitValueInt("hashDistWritePoolSize", globals->getPropInt("@hashDistWritePoolSize", DEFAULT_WRITEPOOLSIZE));
+        writerPoolSize = (unsigned)activity->queryJob().getWorkUnitValueInt("hashDistWritePoolSize", globals->getPropInt("@hashDistWritePoolSize", DEFAULT_WRITEPOOLSIZE));
         if (writerPoolSize>numnodes)
             writerPoolSize = numnodes; // no point in more
         ActPrintLog(activity, "Writer thread pool size : %d", writerPoolSize);

thorlcr/activities/loop/thloop.cpp

@@ -149,6 +149,8 @@ class CLoopActivityMaster : public CLoopActivityMasterBase
     IHThorLoopArg *helper;
     IThorBoundLoopGraph *boundGraph;
     unsigned flags;
+    Owned<IBarrier> barrier;
+
     void checkEmpty()
     {
         // similar to sync, but continiously listens for messages from slaves
@@ -184,7 +186,7 @@ public:
     CLoopActivityMaster(CMasterGraphElement *info) : CLoopActivityMasterBase(info)
     {
         if (!container.queryLocalOrGrouped())
-            mpTag = container.queryJob().allocateMPTag();
+            barrier.setown(container.queryJob().createBarrier(mpTag));
     }
     void init()
     {
@@ -237,11 +239,19 @@ public:
                     initLoopResults(loopCounter);
                 boundGraph->execute(*this, (flags & IHThorLoopArg::LFcounter)?loopCounter:0, ownedResults, (IRowWriterMultiReader *)NULL, 0, extractBuilder.size(), extractBuilder.getbytes());
                 ++loopCounter;
+                if (!barrier->wait(false))
+                    break;
             }
         }
         else
             checkEmpty();
     }
+    virtual void abort()
+    {
+        CLoopActivityMasterBase::abort();
+        if (barrier)
+            barrier->cancel();
+    }
 };
 
 CActivityBase *createLoopActivityMaster(CMasterGraphElement *container)

thorlcr/activities/loop/thloopslave.cpp

@@ -222,6 +222,7 @@ class CLoopSlaveActivity : public CLoopSlaveActivityBase
     unsigned flags, lastMs;
     IHThorLoopArg *helper;
     bool eof, finishedLooping;
+    Owned<IBarrier> barrier;
 
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
@@ -239,6 +240,8 @@ public:
             if (container.queryOwner().isGlobal())
                 global = true;
         }
+        if (!container.queryLocalOrGrouped())
+            barrier.setown(container.queryJob().createBarrier(mpTag));
     }
     virtual void kill()
     {
@@ -246,6 +249,12 @@ public:
         loopPending.clear();
         curInput.clear();
     }
+    virtual void abort()
+    {
+        CLoopSlaveActivityBase::abort();
+        if (barrier)
+            barrier->cancel();
+    }
 // IThorDataLink
     virtual void start()
     {
@@ -364,7 +373,7 @@ public:
                 unsigned doLoopAgain = (flags & IHThorLoopArg::LFnewloopagain) ? helper->loopAgainResult() : 0;
                 ownedResults.setown(queryGraph().createThorGraphResults(3));
                 // ensures remote results are available, via owning activity (i.e. this loop act)
-                // so that when aggreagate result is fetched from the master, it will retreive from the act, not the (already cleaned) graph localresults
+                // so that when aggregate result is fetched from the master, it will retrieve from the act, not the (already cleaned) graph localresults
                 ownedResults->setOwner(container.queryId());
 
                 boundGraph->prepareLoopResults(*this, ownedResults);
@@ -380,6 +389,11 @@ public:
 
                 if (flags & IHThorLoopArg::LFnewloopagain)
                 {
+                    if (!container.queryLocalOrGrouped())
+                    {
+                        if (!barrier->wait(false))
+                            return NULL; // aborted
+                    }
                     Owned<IThorResult> loopAgainResult = ownedResults->getResult(helper->loopAgainResult(), !queryGraph().isLocalChild());
                     assertex(loopAgainResult);
                     Owned<IRowStream> loopAgainRows = loopAgainResult->getRowStream();

thorlcr/thorutil/thbuf.cpp

@@ -1385,7 +1385,9 @@ class CSharedWriteAheadDisk : public CSharedWriteAheadBase
     }
     virtual size32_t rowSize(const void *row)
     {
-        if (meta == serializeMeta)
+        if (!row)
+            return 1; // eog;
+        else if (meta == serializeMeta)
             return meta->getRecordSize(row)+1; // space on disk, +1 = eog marker
         CSizingSerializer ssz;
         serializer->serialize(ssz,(const byte *)row);