Merge remote-tracking branch 'origin/closedown-4.0.x'

Conflicts:
	roxie/ccd/ccdfile.cpp
	roxie/ccd/ccdfile.hpp

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -268,7 +268,7 @@ if (NOT "${CMAKE_VERSION}" VERSION_LESS "2.8.1")
 
             set ( CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/sbin/prerm" )
             set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_CURRENT_BINARY_DIR}/initfiles/bash/sbin/deb/postrm" )
-                else()
+        else()
             message("WARNING: Unsupported package ${packageManagement}.")
         endif ()
 

common/remote/sockfile.cpp

@@ -1547,6 +1547,17 @@ public:
         }
         return clientCrit.getClear();
     }
+    unsigned getHashFromElement(const void *e) const
+    {
+        const CEndpointCS &elem=*(const CEndpointCS *)e;
+        return getHashFromFindParam(elem.queryFindParam());
+    }
+
+    unsigned getHashFromFindParam(const void *fp) const
+    {
+        return ((const SocketEndpoint *)fp)->hash(0);
+    }
+
     void removeExact(CEndpointCS *clientCrit)
     {
         CriticalBlock b(crit);

common/thorhelper/roxierow.cpp

@@ -49,6 +49,8 @@ public:
     {
         byte * ptr = static_cast<byte *>(_ptr);
         memsize_t capacity = RoxieRowCapacity(ptr);
+        if (capacity < size + extraSize)
+            throw MakeStringException(0, "Data was written past the end of the row - allocated %d, written %d", capacity - extraSize, size);
         memset(ptr+size, 0, capacity - size - extraSize);
         unsigned short * check = reinterpret_cast<unsigned short *>(ptr + capacity - extraSize);
         *check = crc16(ptr, capacity-extraSize, 0);
@@ -78,6 +80,8 @@ public:
     {
         byte * ptr = static_cast<byte *>(_ptr);
         memsize_t capacity = RoxieRowCapacity(ptr);
+        if (capacity < size + extraSize)
+            throw MakeStringException(0, "Data was written past the end of the row - allocated %d, written %d", capacity - extraSize, size);
         memset(ptr+size, 0, capacity - size - extraSize);
         unsigned short * check = reinterpret_cast<unsigned short *>(ptr + capacity - extraSize);
         *check = chksum16(ptr, capacity-extraSize);

dali/base/dadfs.cpp

@@ -6315,13 +6315,25 @@ public:
 
 #define GROUP_CACHE_INTERVAL (1000*60)
 
+class CNamedGroupCacheEntry: public CInterface
+{
+public:
+    Linked<IGroup> group;
+    StringAttr name;
+    StringAttr groupdir;
+    unsigned cachedtime;
+
+    CNamedGroupCacheEntry(IGroup *_group, const char *_name, const char *_dir)
+    : group(_group), name(_name), groupdir(_dir)
+    {
+        cachedtime = msTick();
+    }
+};
+
 class CNamedGroupStore: public CInterface, implements INamedGroupStore
 {
     CriticalSection cachesect;
-    Owned<IGroup> cachedgroup;
-    StringAttr cachedname;
-    StringAttr cachedgroupdir;
-    unsigned cachedtime;
+    CIArrayOf<CNamedGroupCacheEntry> cache;
     unsigned defaultTimeout;
 
 public:
@@ -6330,7 +6342,6 @@ public:
     CNamedGroupStore()
     {
         defaultTimeout = INFINITE;
-        cachedtime = 0;
     }
 
     IGroup *dolookup(const char *logicalgroupname,IRemoteConnection *conn, StringBuffer *dirret)
@@ -6342,20 +6353,6 @@ public:
             return NULL;
         gname.toLowerCase();
         logicalgroupname = gname.str();
-        if ((gname.length()>9)&&(memcmp(logicalgroupname,"foreign::",9)==0)) {
-            StringBuffer eps;
-            const char *s = logicalgroupname+9;
-            while (*s&&((*s!=':')||(s[1]!=':')))
-                eps.append(*(s++));
-            if (*s) {
-                s+=2;
-                if (*s) {
-                    Owned<INode> dali = createINode(eps.str());
-                    if (dali) 
-                        return getRemoteGroup(dali,s,FOREIGN_DALI_TIMEOUT,dirret);
-                }
-            }
-        }
         bool isiprange = (*logicalgroupname!=0);
         for (const char *s1=logicalgroupname;*s1;s1++)
             if (isalpha(*s1)) {
@@ -6398,28 +6395,48 @@ public:
             logicalgroupname = gname.str();
         }
         StringAttr groupdir;
+        bool cached = false;
+        unsigned timeNow = msTick();
         {
             CriticalBlock block(cachesect);
-            if (cachedgroup.get()) {
-                if (msTick()-cachedtime>GROUP_CACHE_INTERVAL) {
-                    cachedgroup.clear();
-                    cachedname.clear();
-                    cachedgroupdir.clear();
+            ForEachItemInRev(idx, cache)
+            {
+                CNamedGroupCacheEntry &entry = cache.item(idx);
+                if (timeNow-entry.cachedtime > GROUP_CACHE_INTERVAL)
+                {
+                    cache.remove(idx);
                 }
-                else if (strcmp(gname.str(),cachedname.get())==0) {
-                    cachedtime = msTick();
-                    if (range.length()==0) {
+                else if (strcmp(gname.str(),entry.name.get())==0)
+                {
+                    cached = true;
+                    if (range.length()==0)
+                    {
                         if (dirret)
-                            dirret->append(cachedgroupdir);
-                        return cachedgroup.getLink();
+                            dirret->append(entry.groupdir);
+                        return entry.group.getLink();
                     }
                     // there is a range so copy to epa
-                    cachedgroup->getSocketEndpoints(epa);
-                    groupdir.set(cachedgroupdir);
+                    entry.group->getSocketEndpoints(epa);
+                    groupdir.set(entry.groupdir);
+                    break;
                 }
             }
         }
-        if (epa.ordinality()==0) {
+        if ((gname.length()>9)&&(memcmp(logicalgroupname,"foreign::",9)==0)) {
+            StringBuffer eps;
+            const char *s = logicalgroupname+9;
+            while (*s&&((*s!=':')||(s[1]!=':')))
+                eps.append(*(s++));
+            if (*s) {
+                s+=2;
+                if (*s) {
+                    Owned<INode> dali = createINode(eps.str());
+                    if (!dali || !getRemoteGroup(dali, s, FOREIGN_DALI_TIMEOUT, groupdir, epa))
+                        return NULL;
+                }
+            }
+        }
+        else if (epa.ordinality()==0) {
             struct sLock
             {
                 sLock()  { lock = NULL; };
@@ -6442,30 +6459,33 @@ public:
                 epa.append(ep);
             }
         }
-        IGroup *ret = createIGroup(epa);
+        Owned<IGroup> ret = createIGroup(epa);
+        if (!cached)
         {
             CriticalBlock block(cachesect);
-            cachedgroup.set(ret);
-            cachedname.set(gname);
-            cachedgroupdir.set(groupdir);
-            cachedtime = msTick();
+            cache.append(*new CNamedGroupCacheEntry(ret, gname, groupdir));
         }
-        if (range.length()) {
+        if (range.length())
+        {
             SocketEndpointArray epar;
             const char *s = range.str();
-            while (*s) {
+            while (*s)
+            {
                 unsigned start = 0;
-                while (isdigit(*s)) {
+                while (isdigit(*s))
+                {
                     start = start*10+*s-'0';
                     s++;
                 }
                 if (!start)
                     break;
                 unsigned end;
-                if (*s=='-') {
+                if (*s=='-')
+                {
                     s++;
                     end = 0;
-                    while (isdigit(*s)) {
+                    while (isdigit(*s))
+                    {
                         end = end*10+*s-'0';
                         s++;
                     }
@@ -6474,7 +6494,8 @@ public:
                 }
                 else 
                     end = start;
-                if ((start>epa.ordinality())||(end>epa.ordinality())) {
+                if ((start>epa.ordinality())||(end>epa.ordinality()))
+                {
                     s = range.str();
                     break;
                 }
@@ -6490,12 +6511,11 @@ public:
             }
             if (*s) 
                 throw MakeStringException(-1,"Invalid group range %s",range.str());
-            ::Release(ret);
-            ret = createIGroup(epar);
+            ret.setown(createIGroup(epar));
         }
         if (dirret)
             dirret->append(groupdir);
-        return ret;
+        return ret.getClear();
     }
 
     IGroup *lookup(const char *logicalgroupname)
@@ -6583,11 +6603,10 @@ public:
         connlock.conn->queryRoot()->removeProp(prop.str()); 
         doadd(connlock,name.str(),group,cluster,dir);
         {                                                           
-            CriticalBlock block(cachesect);                     
-            cachedgroup.set(group); // may be NULL
-            cachedname.set(name.str());
-            cachedgroupdir.set(dir);
-            cachedtime = msTick();
+            CriticalBlock block(cachesect);
+            cache.kill();
+            if (group)
+                cache.append(*new CNamedGroupCacheEntry(group, name.str(), dir));
         }
     }
 
@@ -6654,12 +6673,18 @@ public:
             }
         }
         CriticalBlock block(cachesect);
-        cachedgroup.clear();
-        cachedname.clear();
-        cachedgroupdir.clear();
+        cache.kill();
     }
 
-    IGroup *getRemoteGroup(const INode *foreigndali, const char *gname, unsigned foreigndalitimeout, StringBuffer *dirret)
+    unsigned setDefaultTimeout(unsigned timems)
+    {
+        unsigned ret = defaultTimeout;
+        defaultTimeout = timems;
+        return ret;
+    }
+
+private:
+    bool getRemoteGroup(const INode *foreigndali, const char *gname, unsigned foreigndalitimeout, StringAttr &groupdir, SocketEndpointArray &epa)
     {
         StringBuffer lcname(gname);
         gname = lcname.trim().toLowerCase().str();
@@ -6669,7 +6694,7 @@ public:
         foreignDaliSendRecv(foreigndali,mb,foreigndalitimeout);
         checkDfsReplyException(mb);
         if (mb.length()==0)
-            return NULL;
+            return false;
         byte ok;
         mb.read(ok);
         if (ok!=1) {
@@ -6678,39 +6703,21 @@ public:
                 mb.skip(mbsz-1);
                 mb.read(ok);
                 if (ok!=1) 
-                    return NULL;
+                    return false;
             }
             else
-                return NULL;
+                return false;
         }
         Owned<IPropertyTree> pt = createPTree(mb);
         Owned<IPropertyTreeIterator> pe = pt->getElements("Node");
-        SocketEndpointArray epa;
+        groupdir.set(pt->queryProp("@dir"));
         ForEach(*pe) {
             SocketEndpoint ep(pe->query().queryProp("@ip"));
             epa.append(ep);
         }
-        IGroup *ret = createIGroup(epa);
-        {
-            CriticalBlock block(cachesect);
-            cachedgroup.set(ret);
-            cachedname.set(gname);
-            cachedgroupdir.set(pt->queryProp("@dir"));
-            if (dirret)
-                dirret->append(cachedgroupdir);
-            cachedtime = msTick();
-        }
-        return ret;
+        return epa.ordinality() > 0;
     }
 
-    unsigned setDefaultTimeout(unsigned timems)
-    {
-        unsigned ret = defaultTimeout;
-        defaultTimeout = timems;
-        return ret;
-    }
-
-
 };
 
 static CNamedGroupStore *groupStore = NULL;

dali/base/dadfs.hpp

@@ -584,7 +584,6 @@ interface INamedGroupStore: implements IGroupResolver
     virtual bool find(IGroup *grp, StringBuffer &lname, bool add=false) = 0;
     virtual void addUnique(IGroup *group,StringBuffer &lname,const char *dir=NULL) = 0;
     virtual void swapNode(const IpAddress &from, const IpAddress &to) = 0;
-    virtual IGroup *getRemoteGroup(const INode *foreigndali, const char *gname, unsigned foreigndalitimeout=FOREIGN_DALI_TIMEOUT, StringBuffer *dir=NULL) = 0;
     virtual IGroup *lookup(const char *logicalgroupname, StringBuffer &dir) = 0;
     virtual unsigned setDefaultTimeout(unsigned timems) = 0;                                    // sets default timeout for SDS connections and locking                                                                                         // returns previous value
 

dali/dfu/dfuutil.cpp

@@ -402,11 +402,9 @@ class CFileCloner
             throw afor2.exc.getClear();
     }
 
-
-
     void cloneSubFile(IPropertyTree *ftree,const char *destfilename, INode *srcdali)   // name already has prefix added
     {
-        Owned<IFileDescriptor> srcfdesc = deserializeFileDescriptorTree(ftree,&queryNamedGroupStore(),0);
+        Owned<IFileDescriptor> srcfdesc = deserializeFileDescriptorTree(ftree, NULL, 0);
         const char * kind = srcfdesc->queryProperties().queryProp("@kind");
         bool iskey = kind&&(strcmp(kind,"key")==0);
 
@@ -437,11 +435,6 @@ class CFileCloner
         dstfdesc->addCluster(cluster1,grp1,spec);
         if (iskey&&!cluster2.isEmpty())
             dstfdesc->addCluster(cluster2,grp2,spec2);
-#ifdef _TESTING
-//      LOGFDESC("createFileClone",dstfdesc);
-#endif
-
-
 
         for (unsigned pn=0;pn<srcfdesc->numParts();pn++) {
             offset_t sz = srcfdesc->queryPart(pn)->queryProperties().getPropInt64("@size",-1);
@@ -461,6 +454,17 @@ class CFileCloner
         {
             StringBuffer s;
             dstfdesc->queryProperties().setProp("@cloneFrom", srcdali->endpoint().getUrlStr(s).str());
+            unsigned numClusters = srcfdesc->numClusters();
+            for (unsigned clusterNum = 0; clusterNum < numClusters; clusterNum++)
+            {
+                StringBuffer sourceGroup;
+                srcfdesc->getClusterGroupName(clusterNum, sourceGroup, NULL);
+                Owned<IPropertyTree> groupInfo = createPTree("cloneFromGroup");
+                groupInfo->setProp("@groupName", sourceGroup);
+                ClusterPartDiskMapSpec &spec = srcfdesc->queryPartDiskMapping(clusterNum);
+                spec.toProp(groupInfo);
+                dstfdesc->queryProperties().addPropTree("cloneFromGroup", groupInfo.getClear());
+            }
         }
 
         Owned<IDistributedFile> dstfile = fdir->createNew(dstfdesc);
@@ -552,7 +556,6 @@ public:
         }
     }
 
-
     void cloneSuperFile(const char *filename, CDfsLogicalFileName &dlfn)
     {
         level++;

dali/dfu/dfuutil.hpp

@@ -75,7 +75,6 @@ interface IDFUhelper: extends IInterface
         IPropertyTree *relationships,   // if not NULL, tree will have all relationships filled in
         IUserDescriptor *user
     ) = 0;
-
 };
 
 IDFUhelper *createIDFUhelper();

docs/ECLProgrammersGuide/PRG_Mods/PrG_Creating_example_data.xml

@@ -4,14 +4,54 @@
 <sect1 id="Creating_Example_Data">
   <title><emphasis role="bold">Creating Example Data</emphasis></title>
 
+  <sect2 id="Getting_Code_Files">
+    <title>Getting Code Files</title>
+  <para>All the example code for the <emphasis>Programmer's Guide</emphasis>
+  is available for download from the HPCC Systems website from the same page
+  that the PDF is available (<ulink
+  url="http://hpccsystems.com/download/docs/learning-ecl">click here</ulink>).
+  To make this code available for use in the ECL IDE, you simply:</para>
+
+  <orderedlist>
+    <listitem>
+      <para>Download the ECL_Code_Files.ZIP file.</para>
+    </listitem>
+
+    <listitem>
+      <para>In the ECL IDE, highlight your "My Files" folder, right-click and
+      select "Insert Folder" from the popup menu.</para>
+    </listitem>
+
+    <listitem>
+      <para>Name your new folder "ProgrammersGuide" (please note -- spaces are
+      NOT allowed in your ECL repository folder names).</para>
+    </listitem>
+
+    <listitem>
+      <para>In the ECL IDE, highlight your "ProgrammersGuide" folder,
+      right-click and select "Locate File in Explorer" from the popup
+      menu.</para>
+    </listitem>
+
+    <listitem>
+      <para>Extract all the files from the ECL_Code_Files.ZIP file into your
+      new folder.</para>
+    </listitem>
+  </orderedlist>
+  </sect2>
+
+  <sect2 id="Generating_Files">
+    <title>Generating Files</title>
+
   <para>The code that generates the example data used by all the
   <emphasis>Programmer's Guide</emphasis> articles is contained in a file
-  named Gendata.ECL, which was installed by the ECL IDE installation program.
-  You simply need to open that file in the ECL IDE (select <emphasis
-  role="bold">File &gt; Open</emphasis> from the menu, select the Gendata.ECL
-  file, and it will open in a Builder window) then press the <emphasis
-  role="bold">Go</emphasis> button to generate the data files. The process
-  takes a couple of minutes to run. Here is the code, fully explained.</para>
+  named Gendata.ECL. You simply need to open that file in the ECL IDE (select
+  <emphasis role="bold">File &gt; Open</emphasis> from the menu, select the
+  Gendata.ECL file, and it will open in a Builder window) then press the
+  <emphasis role="bold">Submit</emphasis> button to generate the data files.
+  The process takes a couple of minutes to run. Here is the code, fully
+  explained.</para>
+  </sect2>
 
   <sect2 id="Some_Constants">
     <title>Some Constants</title>

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -465,14 +465,7 @@ ecl publish --target=roxie --name=FindPersonService --no-activate findperson.ecl
                 <row>
                   <entry>--no-reload</entry>
 
-                  <entry>Does not reload the queryset</entry>
-                </row>
-
-                <row>
-                  <entry>--no-reload</entry>
-
-                  <entry>Specifies to not request a reload of the Roxie
-                  cluster</entry>
+                  <entry>Specifies to not reload the target cluster</entry>
                 </row>
 
                 <row>
@@ -2332,7 +2325,7 @@ ecl packagemap validate roxie --active</programlisting>
                 <row>
                   <entry>ecl roxie check</entry>
 
-                  <entry>Checks the roxie process state</entry>
+                  <entry>Checks the state of the roxie process </entry>
                 </row>
 
                 <row>
@@ -2359,6 +2352,12 @@ ecl packagemap validate roxie --active</programlisting>
                 </row>
 
                 <row>
+                  <entry>--wait=<emphasis>&lt;ms&gt;</emphasis></entry>
+
+                  <entry>Max time to wait in milliseconds</entry>
+                </row>
+
+                <row>
                   <entry>-u, --username</entry>
 
                   <entry>The username (if necessary)</entry>

ecl/hqlcpp/hqlcerrors.hpp

@@ -208,6 +208,7 @@
 #define HQLERR_EmbeddedTypeNotSupported_X       4187
 #define HQLERR_MaximumSizeLessThanMinimum_XY    4188
 #define HQLERR_UnexpectedOptionValue_XY         4189
+#define HQLERR_VariableRowMustBeLinked          4190
 
 //Warnings....
 #define HQLWRN_PersistDataNotLikely             4500
@@ -491,6 +492,7 @@
 #define HQLERR_EmbeddedTypeNotSupported_X_Text  "Type %s not supported for embedded/external scripts"
 #define HQLERR_MaximumSizeLessThanMinimum_XY_Text "Maximum size (%u) for this record is lower than the minimum (%u)"
 #define HQLERR_UnexpectedOptionValue_XY_Text    "Unexpected value for option %s: %s"
+#define HQLERR_VariableRowMustBeLinked_Text     "External function '%s' cannot return a non-linked variable length row"
 
 //Warnings.
 #define HQLWRN_CannotRecreateDistribution_Text  "Cannot recreate the distribution for a persistent dataset"

ecl/hqlcpp/hqlcpp.cpp

@@ -5775,11 +5775,16 @@ void HqlCppTranslator::doBuildCall(BuildCtx & ctx, const CHqlBoundTarget * tgt,
                 }
                 else
                 {
+                    if (isVariableSizeRecord(expr->queryRecord()))
+                    {
+                        const char * name = expr->queryName()->str();
+                        throwError1(HQLERR_VariableRowMustBeLinked, name ? name : "");
+                    }
                     resultRow.setown(declareTempRow(ctx, ctx, expr));
                     resultRowBuilder.setown(createRowBuilder(ctx, resultRow));
                     IHqlExpression * bound = resultRowBuilder->queryBound();
                     args.append(*getPointer(bound));
-                    localBound.expr.set(bound);
+                    localBound.expr.setown(getPointer(resultRow->queryBound()));
                 }
                 returnByReference = true;
             }

ecl/hthor/hthor.ipp

@@ -1781,6 +1781,7 @@ public:
 
     virtual void ready();
     virtual bool needsAllocator() const { return true; }    
+    virtual bool isGrouped() { return false; }
 
     //interface IHThorInput
     virtual const void *nextInGroup();

ecl/regress/issue9482.ecl

@@ -0,0 +1,14 @@
+FO := SERVICE
+  {varstring512 mya, varstring512 myb} GetInfo(const varstring in_a, const varstring in_b):c,pure,entrypoint='GetInfo';
+END;
+
+{varstring512 mya, varstring512 myb} FOO(const varstring in_a, const varstring in_b):= BEGINC++
+    struct FO_ { char mya[513]; char myb[513]; };
+    FO_ * FOO = (FO_*)__result;
+    strncpy(FOO->mya, in_a, 512);
+    FOO->mya[512] = 0;
+    strncpy(FOO->myb, in_b, 512);
+    FOO->myb[512] = 0;
+ENDC++;
+
+FOO('a','b');

ecl/regress/issue9482b.ecl

@@ -0,0 +1,10 @@
+
+{varstring mya, varstring myb} FOO(const varstring in_a, const varstring in_b):= BEGINC++
+    size_t lena = strlen(in_a)+1;
+    size_t lenb = strlen(in_a)+1;
+    byte * target = (byte*)__result;
+    memcpy(target, in_a, lena);
+    memcpy(target+lena, in_b, lenb);
+ENDC++;
+
+FOO('abcbdeb','bdedeef');

esp/CMakeLists.txt

@@ -23,5 +23,5 @@ HPCC_ADD_SUBDIRECTORY (scm)
 HPCC_ADD_SUBDIRECTORY (services "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (smc "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (test "PLATFORM")
-HPCC_ADD_SUBDIRECTORY (tools "PLATFORM")
+HPCC_ADD_SUBDIRECTORY (tools "CLIENTTOOLS")
 HPCC_ADD_SUBDIRECTORY (xslt "PLATFORM")

initfiles/bash/etc/init.d/install-init.in

@@ -24,23 +24,29 @@ installConfs ()
 {
     fileName=$1
     configPath=$2
+    mkdir -p ${configPath}
+    mkdir -p ${configPath}/rpmnew
 
     printf "Installing %-44s ..." "${fileName}"
+
     if [ ! -e ${configPath}/${fileName} ]; then
-        if [ -e ${configPath}/rpmnew/${fileName} ]; then
-            cp -f ${configPath}/rpmnew/${fileName} ${configPath}/${fileName}
-            log_success_msg
+        # Always install new files without comment
+        cp -f ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} ${configPath}/${fileName}
+        cp -f ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} ${configPath}/rpmnew/${fileName}
+        log_success_msg
+    elif [ -e ${configPath}/rpmnew/${fileName} ] && ! `diff -q ${configPath}/rpmnew/${fileName} ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} >/dev/null` ; then
+        # There are changes in the default config since last installed
+        if ! `diff -q ${configPath}/rpmnew/${fileName} ${configPath}/${fileName} >/dev/null` ; then
+            # User has made their own changes too, so don't overwrite
+            log_failure_msg "Not overwriting modified configuration file ${fileName}"
         else
-            if [ -e ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} ]; then
-                cp -f ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} ${configPath}/${fileName}
-                cp -f ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} ${configPath}/rpmnew/${fileName}
-                log_success_msg
-            else
-                log_failure_msg "Fail: File doesn't exist in rpmnew"
-            fi
+            # User has NOT made their own changes - ok to update
+            cp -f ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} ${configPath}/${fileName}
+            cp -f ${INSTALL_DIR}/${configPath}/rpmnew/${fileName} ${configPath}/rpmnew/${fileName}
+            log_success_msg "Updated configuration file ${fileName}"
         fi
     else
-        log_success_msg
+        log_success_msg "No changes to configuration file ${fileName}"
     fi
 }
 
@@ -127,6 +133,9 @@ SECTION=${SECTION:-DEFAULT}
 confToUse="${INSTALL_DIR}${CONFIG_DIR}/${ENV_CONF_FILE}"
 
 if [ -d ${CONFIG_DIR} ]; then
+    if [ -f ${CONFIG_DIR}/installed ] ; then
+        exit 0
+    fi
     if [ -e ${CONFIG_DIR}/${ENV_CONF_FILE} ]; then
         confToUse="${CONFIG_DIR}/${ENV_CONF_FILE}"
     fi
@@ -265,3 +274,7 @@ fi
 chown root:$group ${configs}
 chown -R $user:$group ${configs}/*
 chmod 775 ${configs}
+
+if [ -d ${CONFIG_DIR} ]; then
+    date > ${CONFIG_DIR}/installed
+fi

initfiles/bash/sbin/deb/postrm.in

@@ -17,7 +17,12 @@
 
 ###<REPLACE>###
 
+# On some systems, when running an update (as opposed to a remove) the postinstall script does
+# not get run. To work around that issue, we run the post-install actions in the post-remove script
+# (which IS run)
+#
+# If the install dir is still present post-removal, must be an update
+
 if [ -f "${INSTALL_DIR}/etc/init.d/install-init" ]; then
-        ${INSTALL_DIR}/etc/init.d/install-init
+    ${INSTALL_DIR}/etc/init.d/install-init
 fi
-

initfiles/etc/DIR_NAME/CMakeLists.txt

@@ -25,7 +25,7 @@ FOREACH( iFILES
 ENDFOREACH ( iFILES)
 
 Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.xml DESTINATION .${CONFIG_DIR}/rpmnew COMPONENT Runtime ) 	#  Don't delete the "."
-Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.conf DESTINATION ${CONFIG_DIR}/rpmnew COMPONENT Runtime )
-Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/genenvrules.conf DESTINATION ${CONFIG_DIR}/rpmnew COMPONENT Runtime )
+Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.conf DESTINATION .${CONFIG_DIR}/rpmnew COMPONENT Runtime )
+Install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/genenvrules.conf DESTINATION .${CONFIG_DIR}/rpmnew COMPONENT Runtime )
 
 ADD_SUBDIRECTORY(configmgr)

initfiles/etc/DIR_NAME/environment.conf.in

@@ -1,4 +1,4 @@
-## Default environment configuration file
+## Default environment configuration file for OpenHPCC
 
 [DEFAULT]
 configs=${CONFIG_DIR}

initfiles/etc/DIR_NAME/environment.xml.in

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!--
-## Default environment file single-node hpcc installation
+## Default environment file single-node OpenHPCC installation
 -->
 
 <Environment>

initfiles/sbin/prerm.in

@@ -25,20 +25,27 @@ if [ $configmgr_status -gt 0 ]; then
     exit 1
 fi
 
-
 # Stop all services before cleaning up
 ######################################
 /etc/init.d/hpcc-init stop
 if [ -f "${PID_DIR}/mydafilesrv.pid" ]; then
     /etc/init.d/dafilesrv stop
 fi
-# Removing edits from /etc files
+
+# Remove edits from /etc files
+##############################
 ${INSTALL_DIR}/sbin/rm_conf_settings.sh
 
 
-# Removing symlinks.
+# Remove symlinks
+#################
 ${INSTALL_DIR}/etc/init.d/uninstall-init
 
+# Remove installed flag
+#######################
+if [ -f ${CONFIG_DIR}/installed ] ; then
+    rm ${CONFIG_DIR}/installed
+fi
 
 exit 0
  

plugins/Rembed/Rembed.cpp

@@ -46,8 +46,8 @@ extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
         return false;
     pb->magicVersion = PLUGIN_VERSION;
     pb->version = version;
-    pb->moduleName = "R";
-    pb->ECL = NULL;
+    pb->moduleName = "+R+"; // Hack - we don't want to export any ECL, but if we don't export something,
+    pb->ECL = "";           // Hack - the dll is unloaded at startup when compiling, and the R runtime closes stdin when unloaded
     pb->flags = PLUGIN_MULTIPLE_VERSIONS;
     pb->description = "R Embed Helper";
     return true;

roxie/ccd/ccddali.cpp

@@ -274,6 +274,53 @@ private:
         return localTree.getClear();
     }
 
+    IFileDescriptor *recreateCloneSource(IFileDescriptor *srcfdesc, const char *destfilename)
+    {
+        Owned<IFileDescriptor> dstfdesc = createFileDescriptor(srcfdesc->getProperties());
+        // calculate dest dir
+
+        CDfsLogicalFileName dstlfn;
+        if (!dstlfn.setValidate(destfilename,true))
+            throw MakeStringException(-1,"Logical name %s invalid",destfilename);
+
+        StringBuffer dstpartmask;
+        getPartMask(dstpartmask,destfilename,srcfdesc->numParts());
+        dstfdesc->setPartMask(dstpartmask.str());
+        unsigned np = srcfdesc->numParts();
+        dstfdesc->setNumParts(srcfdesc->numParts());
+        DFD_OS os = (getPathSepChar(srcfdesc->queryDefaultDir())=='\\')?DFD_OSwindows:DFD_OSunix;
+        StringBuffer dir;
+        StringBuffer dstdir;
+        makePhysicalPartName(dstlfn.get(),0,0,dstdir,false,os,NULL);
+        dstfdesc->setDefaultDir(dstdir.str());
+
+        for (unsigned pn=0;pn<srcfdesc->numParts();pn++) {
+            offset_t sz = srcfdesc->queryPart(pn)->queryProperties().getPropInt64("@size",-1);
+            if (sz!=(offset_t)-1)
+                dstfdesc->queryPart(pn)->queryProperties().setPropInt64("@size",sz);
+            StringBuffer dates;
+            if (srcfdesc->queryPart(pn)->queryProperties().getProp("@modified",dates))
+                dstfdesc->queryPart(pn)->queryProperties().setProp("@modified",dates.str());
+        }
+
+        const char *cloneFrom = srcfdesc->queryProperties().queryProp("@cloneFrom");
+        Owned<IPropertyTreeIterator> groups = srcfdesc->queryProperties().getElements("cloneFromGroup");
+        ForEach(*groups)
+        {
+            IPropertyTree &elem = groups->query();
+            const char *groupName = elem.queryProp("@groupName");
+            StringBuffer foreignGroup("foreign::");
+            foreignGroup.append(cloneFrom).append("::").append(groupName);
+            Owned<IGroup> group = queryNamedGroupStore().lookup(foreignGroup);  // NOTE - this is cached by the named group store
+            ClusterPartDiskMapSpec dmSpec;
+            dmSpec.fromProp(&elem);
+            dstfdesc->addCluster(groupName, group, dmSpec);
+        }
+
+        return dstfdesc.getClear();
+    }
+
+
 public:
 
     IMPLEMENT_IINTERFACE;
@@ -344,30 +391,38 @@ public:
         return ret.getClear();
     }
 
-    IFileDescriptor *checkClonedFromRemote(const char *_lfn, IFileDescriptor *fdesc, bool cacheIt, bool writeAccess)
+    IFileDescriptor *checkClonedFromRemote(const char *_lfn, IFileDescriptor *fdesc, bool cacheIt)
     {
+        // NOTE - we rely on the fact that  queryNamedGroupStore().lookup caches results,to avoid excessive load on remote dali
         if (_lfn && !strnicmp(_lfn, "foreign", 7)) //if need to support dali hopping should add each remote location
             return NULL;
         if (!fdesc || !fdesc->queryProperties().hasProp("@cloneFrom"))
             return NULL;
-        SocketEndpoint cloneFrom;
-        cloneFrom.set(fdesc->queryProperties().queryProp("@cloneFrom"));
-        if (cloneFrom.isNull())
-            return NULL;
-        CDfsLogicalFileName lfn;
-        lfn.set(_lfn);
-        lfn.setForeign(cloneFrom, false);
-        if (!connected())
-            return resolveCachedLFN(lfn.get());
-        Owned<IDistributedFile> cloneFile = resolveLFN(lfn.get(), cacheIt, writeAccess);
-        if (cloneFile)
+        if (fdesc->queryProperties().hasProp("cloneFromGroup"))
+        {
+            return recreateCloneSource(fdesc, _lfn);
+        }
+        else // Legacy mode - recently cloned files should have the extra info
         {
-            Owned<IFileDescriptor> cloneFDesc = cloneFile->getFileDescriptor();
-            if (cloneFDesc->numParts()==fdesc->numParts())
-                return cloneFDesc.getClear();
+            SocketEndpoint cloneFrom;
+            cloneFrom.set(fdesc->queryProperties().queryProp("@cloneFrom"));
+            if (cloneFrom.isNull())
+                return NULL;
+            CDfsLogicalFileName lfn;
+            lfn.set(_lfn);
+            lfn.setForeign(cloneFrom, false);
+            if (!connected())
+                return resolveCachedLFN(lfn.get());
+            Owned<IDistributedFile> cloneFile = resolveLFN(lfn.get(), cacheIt, false);
+            if (cloneFile)
+            {
+                Owned<IFileDescriptor> cloneFDesc = cloneFile->getFileDescriptor();
+                if (cloneFDesc->numParts()==fdesc->numParts())
+                    return cloneFDesc.getClear();
 
-            StringBuffer s;
-            DBGLOG(ROXIE_MISMATCH, "File %s cloneFrom(%s) mismatch", _lfn, cloneFrom.getIpText(s).str());
+                StringBuffer s;
+                DBGLOG(ROXIE_MISMATCH, "File %s cloneFrom(%s) mismatch", _lfn, cloneFrom.getIpText(s).str());
+            }
         }
         return NULL;
     }

roxie/ccd/ccddali.hpp

@@ -39,7 +39,7 @@ interface IRoxieDaliHelper : extends IInterface
 {
     virtual void commitCache() = 0;
     virtual bool connected() const = 0;
-    virtual IFileDescriptor *checkClonedFromRemote(const char *id, IFileDescriptor *fdesc, bool cacheIt, bool writeAccess) = 0;
+    virtual IFileDescriptor *checkClonedFromRemote(const char *id, IFileDescriptor *fdesc, bool cacheIt) = 0;
     virtual IDistributedFile *resolveLFN(const char *filename, bool cacheIt, bool writeAccess) = 0;
     virtual IFileDescriptor *resolveCachedLFN(const char *filename) = 0;
     virtual IConstWorkUnit *attachWorkunit(const char *wuid, ILoadedDllEntry *source) = 0;

roxie/ccd/ccdfile.cpp

@@ -82,15 +82,10 @@ protected:
     Owned<IFileIO> current;
     Owned<IMemoryMappedFile> mmapped;
     mutable CriticalSection crit;
-    bool memFileRequested;
-    StringAttr id;
     bool remote;
     offset_t fileSize;
     CDateTime fileDate;
     unsigned crc;
-    Owned<ILazyFileIO> patchFile;
-    StringBuffer baseIndexFileName;
-    RoxieFileType fileType;
     unsigned lastAccess;
     bool copying;
     bool isCompressed;
@@ -103,9 +98,8 @@ protected:
 public:
     IMPLEMENT_IINTERFACE;
 
-    CLazyFileIO(const char *_id, RoxieFileType _fileType, IFile *_logical, offset_t size, const CDateTime &_date, bool _memFileRequested, unsigned _crc, bool _isCompressed)
-        : id(_id), 
-          fileType(_fileType), logical(_logical), fileSize(size), crc(_crc), isCompressed(_isCompressed)
+    CLazyFileIO(IFile *_logical, offset_t size, const CDateTime &_date, unsigned _crc, bool _isCompressed)
+        : logical(_logical), fileSize(size), crc(_crc), isCompressed(_isCompressed)
     {
         fileDate.set(_date);
         currentIdx = 0;
@@ -114,7 +108,6 @@ public:
 #ifdef FAIL_20_READ
         readCount = 0;
 #endif
-        memFileRequested = _memFileRequested;
         lastAccess = msTick();
         copying = false;
         cached = NULL;
@@ -239,39 +232,32 @@ public:
                     if ((openCount % 5) == 0)
                         throw MakeStringException(ROXIE_FILE_OPEN_FAIL, "Pretending to fail on an open");
 #endif
-#if 0
-                    if (memFileRequested && &sources.item(currentIdx)==logical)
-                        current.setown(createMemoryFile(logical));
+                    IFile *f = &sources.item(currentIdx);
+                    if (firstTime)
+                        cacheFileConnect(f, dafilesrvLookupTimeout);  // set timeout to 10 seconds
                     else
-#endif
                     {
-                        IFile *f = &sources.item(currentIdx);
-                        if (firstTime)
-                            cacheFileConnect(f, dafilesrvLookupTimeout);  // set timeout to 10 seconds
-                        else
-                        {
-                            if (traceLevel > 10)
-                                DBGLOG("Looking for file using non-cached file open");
-                        }
+                        if (traceLevel > 10)
+                            DBGLOG("Looking for file using non-cached file open");
+                    }
 
-                        fileStatus = queryFileCache().fileUpToDate(f, fileType, fileSize, fileDate, crc, sourceName, isCompressed);
-                        if (fileStatus == FileIsValid)
+                    fileStatus = queryFileCache().fileUpToDate(f, fileSize, fileDate, crc, isCompressed);
+                    if (fileStatus == FileIsValid)
+                    {
+                        if (isCompressed)
+                            current.setown(createCompressedFileReader(f));
+                        else
+                            current.setown(f->open(IFOread));
+                        if (current)
                         {
-                            if (isCompressed)
-                                current.setown(createCompressedFileReader(f));
-                            else
-                                current.setown(f->open(IFOread));
-                            if (current)
-                            {
-                                if (traceLevel > 5)
-                                    DBGLOG("Opening %s", sourceName);
-                                disconnectRemoteIoOnExit(current);
-                                break;
-                            }
-        //                  throwUnexpected();  - try another location if this one has the wrong version of the file
+                            if (traceLevel > 5)
+                                DBGLOG("Opening %s", sourceName);
+                            disconnectRemoteIoOnExit(current);
+                            break;
                         }
-                        disconnectRemoteFile(f);
+    //                  throwUnexpected();  - try another location if this one has the wrong version of the file
                     }
+                    disconnectRemoteFile(f);
                 }
                 catch (IException *E)
                 {
@@ -368,32 +354,6 @@ public:
         return current->size();
     }
 
-    virtual void setBaseIndexFileName(const char *val)
-    {
-        CriticalBlock b(crit);
-        baseIndexFileName.append(val);
-    }
-
-    virtual const char *queryBaseIndexFileName()
-    {
-        CriticalBlock b(crit);
-        if (baseIndexFileName.length())
-            return baseIndexFileName.str();
-        return NULL;
-    }
-
-    virtual void setPatchFile(ILazyFileIO *val)
-    {
-        CriticalBlock b(crit);
-        patchFile.setown(LINK(val));
-    }
-
-    virtual ILazyFileIO *queryPatchFile()
-    {
-        CriticalBlock b(crit);
-        return patchFile;
-    }
-
     virtual size32_t write(offset_t pos, size32_t len, const void * data) { throwUnexpected(); }
     virtual void setSize(offset_t size) { throwUnexpected(); }
     virtual offset_t appendFile(IFile *file,offset_t pos,offset_t len) { throwUnexpected(); return 0; }
@@ -401,7 +361,6 @@ public:
     virtual const char *queryFilename() { return logical->queryFilename(); }
     virtual bool isAlive() const { return CInterface::isAlive(); }
     virtual int getLinkCount() const { return CInterface::getLinkCount(); }
-    virtual RoxieFileType getFileType() { return fileType; }
 
     virtual IMemoryMappedFile *queryMappedFile()
     {
@@ -436,7 +395,7 @@ public:
             filesTried.appendf(" %s", sourceName);
             try
             {
-                if (queryFileCache().fileUpToDate(&sources.item(currentIdx), fileType, fileSize, fileDate, crc, sourceName, isCompressed) == FileIsValid)
+                if (queryFileCache().fileUpToDate(&sources.item(currentIdx), fileSize, fileDate, crc, isCompressed) == FileIsValid)
                 {
                     StringBuffer source_drive;
                     splitFilename(sourceName, &source_drive, NULL, NULL, NULL);
@@ -511,8 +470,59 @@ public:
 
 //----------------------------------------------------------------------------------------------
 
+static IPartDescriptor *queryMatchingRemotePart(IPartDescriptor *pdesc, IFileDescriptor *remoteFDesc, unsigned int partNum)
+{
+    if (!remoteFDesc)
+        return NULL;
+    IPartDescriptor *remotePDesc = remoteFDesc->queryPart(partNum);
+    if (!remotePDesc)
+        return NULL;
+    unsigned int crc, remoteCrc;
+    if (!pdesc || !pdesc->getCrc(crc)) //local crc not available, never DFS copied?
+        return remotePDesc;
+    if (remotePDesc->getCrc(remoteCrc) && remoteCrc==crc)
+        return remotePDesc;
+    return NULL;
+}
+
+static bool isCopyFromCluster(IPartDescriptor *pdesc, unsigned clusterNo, const char *process)
+{
+    StringBuffer s;
+    return strieq(process, pdesc->queryOwner().getClusterGroupName(clusterNo, s));
+}
+
+static bool checkClusterCount(UnsignedArray &counts, unsigned clusterNo, unsigned max)
+{
+    while (!counts.isItem(clusterNo))
+        counts.append(0);
+    unsigned count = counts.item(clusterNo);
+    if (count>=max)
+        return false;
+    counts.replace(++count, clusterNo);
+    return true;
+}
+
+static void appendRemoteLocations(IPartDescriptor *pdesc, StringArray &locations, bool checkSelf)
+{
+    UnsignedArray clusterCounts;
+    unsigned numCopies = pdesc->numCopies();
+    for (unsigned copy = 0; copy < numCopies; copy++)
+    {
+        unsigned clusterNo = pdesc->copyClusterNum(copy);
+        if (!checkClusterCount(clusterCounts, clusterNo, 2))
+            continue;
+        if (checkSelf && isCopyFromCluster(pdesc, clusterNo, roxieName.str())) //don't add ourself
+            continue;
+        RemoteFilename r;
+        pdesc->getFilename(copy,r);
+        StringBuffer path;
+        locations.append(r.getRemotePath(path).str());
+    }
+}
+
+//----------------------------------------------------------------------------------------------
+
 typedef StringArray *StringArrayPtr;
-typedef MapStringTo<StringArrayPtr> MapStringToDiffFileUsage;
 
 class CRoxieFileCache : public CInterface, implements ICopyFileProgress, implements IRoxieFileCache
 {
@@ -532,8 +542,7 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
     Semaphore bctStarted;
     Semaphore hctStarted;
 
-
-    RoxieFileStatus fileUpToDate(IFile *f, RoxieFileType fileType, offset_t size, const CDateTime &modified, unsigned crc, const char* id, bool isCompressed)
+    RoxieFileStatus fileUpToDate(IFile *f, offset_t size, const CDateTime &modified, unsigned crc, bool isCompressed)
     {
         if (f->exists())
         {
@@ -543,8 +552,8 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
 
             if (crc > 0)
             {
-                // if a crc is specified lets check it
-                unsigned file_crc = getFileCRC(id);
+                // if a crc is specified let's check it
+                unsigned file_crc = f->getCRC();
                 if (file_crc && crc != file_crc)  // for remote files crc_file can fail, even if the file is valid
                 {
                     DBGLOG("FAILED CRC Check");
@@ -558,52 +567,42 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
             return FileNotFound;
     }
 
-    ILazyFileIO *openFile(const char *id, unsigned partNo, RoxieFileType fileType, const char *localLocation, const StringArray &peerRoxieCopiedLocationInfo, const StringArray &remoteLocationInfo, offset_t size, const CDateTime &modified, bool memFile, unsigned crc, bool isCompressed)
+    ILazyFileIO *openFile(const char *lfn, unsigned partNo, const char *localLocation,
+                           IPartDescriptor *pdesc,
+                           const StringArray &remoteLocationInfo,
+                           offset_t size, const CDateTime &modified, unsigned crc)
     {
         Owned<IFile> local = createIFile(localLocation);
-
-        Owned<CLazyFileIO> ret = new CLazyFileIO(id, fileType, local.getLink(), size, modified, memFile, crc, isCompressed);
-        RoxieFileStatus fileStatus = fileUpToDate(local, fileType, size, modified, crc, localLocation, isCompressed);
+        bool isCompressed = pdesc->queryOwner().isCompressed();
+        Owned<CLazyFileIO> ret = new CLazyFileIO(local.getLink(), size, modified, crc, isCompressed);
+        RoxieFileStatus fileStatus = fileUpToDate(local, size, modified, crc, isCompressed);
         if (fileStatus == FileIsValid)
         {
             ret->addSource(local.getLink());
             ret->setRemote(false);
         }
-        else if (copyResources || useRemoteResources)
+        else if (local->exists())  // Implies local dali and local file out of sync
+            throw MakeStringException(ROXIE_FILE_ERROR, "File does not match DFS information");
+        else
         {
-            if (local->exists())
-            {
-                StringBuffer errStatus;
-                switch (fileStatus)
-                {
-                    case FileSizeMismatch:
-                        errStatus.append("FileSizeMismatch");
-                        break;
-
-                    case FileCRCMismatch:
-                        errStatus.append("FileCRCMismatch");
-                        break;
-
-                    case FileDateMismatch:
-                        errStatus.append("FileDateMismatch");
-                        break;
-                }
-                DBGLOG("Removing local file - %s because %s", localLocation, errStatus.str());
-                local->remove();
-            }
             bool addedOne = false;
 
             // put the peerRoxieLocations next in the list
-            ForEachItemIn(roxie_idx, peerRoxieCopiedLocationInfo)
+            StringArray localLocations;
+            appendRemoteLocations(pdesc, localLocations, true);
+            ForEachItemIn(roxie_idx, localLocations)
             {
                 try
                 {
-                    const char *remoteName = peerRoxieCopiedLocationInfo.item(roxie_idx);
-                    if (miscDebugTraceLevel > 10)
-                        DBGLOG("adding peer roxie location %s", remoteName);
-        
-                    ret->addSource(createIFile(remoteName));
-                    addedOne = true;
+                    const char *remoteName = localLocations.item(roxie_idx);
+                    Owned<IFile> remote = createIFile(remoteName);
+                    if (fileUpToDate(remote, size, modified, crc, isCompressed)==FileIsValid)
+                    {
+                        if (miscDebugTraceLevel > 10)
+                            DBGLOG("adding peer roxie location %s", remoteName);
+                        ret->addSource(remote.getClear());
+                        addedOne = true;
+                    }
                 }
                 catch (IException *E)
                 {
@@ -612,110 +611,39 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                 }
             }
 
-            ForEachItemIn(idx, remoteLocationInfo)
+            if (!addedOne && (copyResources || useRemoteResources))  // If no peer locations available, go to remote
             {
-                try
-                {
-                    const char *remoteName = remoteLocationInfo.item(idx);
-                    if (miscDebugTraceLevel > 10)
-                        DBGLOG("adding remote location %s", remoteName);
-        
-                    ret->addSource(createIFile(remoteName));
-                    addedOne = true;
-                }
-                catch (IException *E)
+                ForEachItemIn(idx, remoteLocationInfo)
                 {
-                    EXCLOG(MCoperatorError, E, "While creating remote file reference");
-                    E->Release();
+                    try
+                    {
+                        const char *remoteName = remoteLocationInfo.item(idx);
+                        Owned<IFile> remote = createIFile(remoteName);
+                        if (fileUpToDate(remote, size, modified, crc, isCompressed)==FileIsValid)
+                        {
+                            if (miscDebugTraceLevel > 10)
+                                DBGLOG("adding remote location %s", remoteName);
+                            ret->addSource(remote.getClear());
+                            addedOne = true;
+                        }
+                    }
+                    catch (IException *E)
+                    {
+                        EXCLOG(MCoperatorError, E, "While creating remote file reference");
+                        E->Release();
+                    }
                 }
             }
-            if (!addedOne)
-            {
-                StringBuffer remoteLocs;
-
-//              ForEachItemIn(roxie_idx, peerRoxieCopiedLocationInfo)
-//                  remoteLocs.appendf("%s ", peerRoxieCopiedLocationInfo.item(roxie_idx));  // all remote locations that were checked
 
-                ForEachItemIn(idx2, remoteLocationInfo)
-                    remoteLocs.appendf("%s ", remoteLocationInfo.item(idx2));  // all remote locations that were checked
-                throw MakeStringException(ROXIE_FILE_OPEN_FAIL, "Could not open file %s at any remote location - %s", localLocation, remoteLocs.str());
-            }
+            if (!addedOne)
+                throw MakeStringException(ROXIE_FILE_OPEN_FAIL, "Could not open file %s", localLocation);
             ret->setRemote(true);
         }
-        else
-            throw MakeStringException(ROXIE_FILE_OPEN_FAIL, "Could not open file %s", localLocation);
         ret->setCache(this);
         files.setValue(localLocation, (ILazyFileIO *)ret);
         return ret.getClear();
     }
 
-    bool doCreateFromPatch(ILazyFileIO *targetFile, const char *baseIndexFilename, ILazyFileIO *patchFile, const char *targetFilename, const char *destPath)
-    {
-        if (!enableKeyDiff)
-            return false;  // feature disabled in roxietopology
-
-        bool fileCopied = false;
-        IFile *patch_sourceFile;
-        try
-        {
-            // MORE - sort out when to disallow closes and of what
-            patch_sourceFile = patchFile->querySource();
-        }
-        catch (IException *E)
-        {
-            EXCLOG(MCoperatorError, E, "While trying to open patch file");
-            throw;
-        }
-
-        unsigned __int64 freeDiskSpace = getFreeSpace(destPath);
-        if ( (targetFile->size() + minFreeDiskSpace) > freeDiskSpace)
-        {
-            StringBuffer err;
-            err.appendf("Insufficient disk space.  File %s needs %"I64F"d bytes, but only %"I64F"d remains, and %"I64F"d is needed as a reserve", targetFilename, targetFile->size(), freeDiskSpace, minFreeDiskSpace   );
-            IException *E = MakeStringException(ROXIE_DISKSPACE_ERROR, "%s", err.str());
-            EXCLOG(MCoperatorError, E);
-            E->Release();
-        }
-        else
-        {
-            try
-            {
-                MTimeSection timing(NULL, "createKeyDiff");
-                Owned<IKeyDiffApplicator> patchApplicator;
-                const char *patchFilename = patch_sourceFile->queryFilename();
-                    
-                DBGLOG("***** Using KeyDiff to create %s", targetFilename);
-                patchApplicator.setown(createKeyDiffApplicator(patchFilename, baseIndexFilename, targetFilename, NULL, true, true));
-                patchApplicator->run();
-                patchApplicator.clear();
-
-                // need to update time stamp in roxiestate file
-                Owned<IFile> tmp_file = createIFile(targetFilename);
-                IFile* remote_sourceFile = targetFile->querySource();
-                CDateTime dt1, dt2, dt3;
-                remote_sourceFile->getTime(&dt1, &dt2, &dt3);
-                tmp_file->setTime(&dt1, &dt2, &dt3);
-            }
-            catch(IException *E)
-            {
-                EXCLOG(E, "Create PatchFile exception");
-                E->Release();
-                return false;
-                //throw; - do not treat as fatal
-            }
-            if (needToDeleteFile)
-            {
-                DBGLOG("creating of data file %s stopped since query has been deleted", targetFilename);
-            }
-            else
-            {
-                targetFile->copyComplete();
-                fileCopied = true;
-            }
-        }
-        return fileCopied;
-    }
-
     void deleteTempFiles(const char *targetFilename)
     {
         try
@@ -887,13 +815,6 @@ class CRoxieFileCache : public CInterface, implements ICopyFileProgress, impleme
                 return false;
             }
             
-            const char *baseIndexFilename = f->queryBaseIndexFileName();
-            ILazyFileIO* patchFile = f->queryPatchFile();
-
-            if (baseIndexFilename && patchFile)
-                if (doCreateFromPatch(f, baseIndexFilename, patchFile, targetFilename, destPath.str()))
-                    return true;
-
             tempFile.append(".$$$");
             const char *msg = background ? "Background copy" : "Copy";
             return doCopyFile(f, tempFile.str(), targetFilename, destPath.str(), msg);
@@ -1120,8 +1041,34 @@ public:
         }
     }
 
-    virtual ILazyFileIO *lookupFile(const char *id, unsigned partNo, RoxieFileType fileType, const char *localLocation, const char *baseIndexFileName,  ILazyFileIO *patchFile, const StringArray &peerRoxieCopiedLocationInfo, const StringArray &deployedLocationInfo, offset_t size, const CDateTime &modified, bool memFile, bool isRemote, bool startFileCopy, bool doForegroundCopy, unsigned crc, bool isCompressed, const char *lookupDali)
+    virtual ILazyFileIO *lookupFile(const char *lfn, RoxieFileType fileType,
+                                     IPartDescriptor *pdesc, unsigned numParts,
+                                     const StringArray &deployedLocationInfo, bool startFileCopy)
     {
+        IPropertyTree &partProps = pdesc->queryProperties();
+        offset_t dfsSize = partProps.getPropInt64("@size", -1);
+        unsigned crc;
+        if (!pdesc->getCrc(crc))
+            crc = 0;
+        CDateTime dfsDate;
+        if (checkFileDate)
+        {
+            const char *dateStr = partProps.queryProp("@modified");
+            dfsDate.setString(dateStr);
+        }
+
+        unsigned partNo = pdesc->queryPartIndex() + 1;
+        StringBuffer localLocation;
+
+        // MORE - not at all sure about this. Foreign files should stay foreign ?
+        CDfsLogicalFileName dlfn;
+        dlfn.set(lfn);
+        if (dlfn.isForeign())
+            dlfn.clearForeign();
+        const char *logicalname = dlfn.get();
+
+        makePhysicalPartName(logicalname, partNo, numParts, localLocation, false, DFD_OSdefault, baseDataDirectory);  // MORE - if we get the dataDirectory we can pass it in and possibly reuse an existing file
+
         Owned<ILazyFileIO> ret;
         try
         {
@@ -1129,48 +1076,43 @@ public:
             Linked<ILazyFileIO> f = files.getValue(localLocation);
             if (f && f->isAlive())
             {
-                if ((size != -1 && size != f->getSize()) ||
-                    (!modified.isNull() && !modified.equals(*f->queryDateTime(), false)))
+                if ((dfsSize != (offset_t) -1 && dfsSize != f->getSize()) ||
+                    (!dfsDate.isNull() && !dfsDate.equals(*f->queryDateTime(), false)))
                 {
                     StringBuffer modifiedDt;
-                    if (!modified.isNull())
-                        modified.getString(modifiedDt);
+                    if (!dfsDate.isNull())
+                        dfsDate.getString(modifiedDt);
                     StringBuffer fileDt;
                     f->queryDateTime()->getString(fileDt);
-                    if (fileErrorList.find(id) == 0)
+                    if (fileErrorList.find(lfn) == 0)
                     {
                         switch (fileType)
                         {
                             case ROXIE_KEY:
-                                fileErrorList.setValue(id, "Key");
+                                fileErrorList.setValue(lfn, "Key");
                                 break;
 
                             case ROXIE_FILE:
-                                fileErrorList.setValue(id, "File");
+                                fileErrorList.setValue(lfn, "File");
                                 break;
                         }
                     }
-                    throw MakeStringException(ROXIE_MISMATCH, "Different version of %s already loaded: sizes = %"I64F"d %"I64F"d  Date = %s  %s", id, size, f->getSize(), modifiedDt.str(), fileDt.str());
+                    throw MakeStringException(ROXIE_MISMATCH, "Different version of %s already loaded: sizes = %"I64F"d %"I64F"d  Date = %s  %s", lfn, dfsSize, f->getSize(), modifiedDt.str(), fileDt.str());
                 }
                 else
                     return f.getClear();
             }
 
-            ret.setown(openFile(id, partNo, fileType, localLocation, peerRoxieCopiedLocationInfo, deployedLocationInfo, size, modified, memFile, crc, isCompressed));  // for now don't check crcs
-
-            if (baseIndexFileName)
-                ret->setBaseIndexFileName(baseIndexFileName);
-            if (patchFile)
-                ret->setPatchFile(patchFile);
+            ret.setown(openFile(lfn, partNo, localLocation, pdesc, deployedLocationInfo, dfsSize, dfsDate, crc));
 
             if (startFileCopy)
             {
                 if (ret->isRemote())
                 {
-                    if (copyResources || memFile)
+                    if (copyResources) // MORE - should always copy peer files
                     {
                         needToDeleteFile = false;
-                        if (doForegroundCopy)
+                        if (numParts==1 || (partNo==numParts && fileType==ROXIE_KEY))
                         {
                             ret->checkOpen();
                             doCopy(ret, false, false);
@@ -1183,31 +1125,25 @@ public:
 
                     }
                 }
-                else if (isRemote)
-                {
-//                  todo.append(*ret);  // don't really need to copy. But do need to send message that copy happened (async)
-//                  atomic_inc(&numFilesToProcess);  // must increment counter for SNMP accuracy
-//                  toCopy.signal();
-                }
             }
 
-            if (!lazyOpen || fileType == ROXIE_PATCH)  // patch file MUST be open at this point - make sure we open it
+            if (!lazyOpen)
                 ret->checkOpen();
         }
         catch(IException *e)
         {
             if (e->errorCode() == ROXIE_FILE_OPEN_FAIL)
             {
-                if (fileErrorList.find(id) == 0)
+                if (fileErrorList.find(lfn) == 0)
                 {
                     switch (fileType)
                     {
                         case ROXIE_KEY:
-                            fileErrorList.setValue(id, "Key");
+                            fileErrorList.setValue(lfn, "Key");
                             break;
                         
                         case ROXIE_FILE:
-                            fileErrorList.setValue(id, "File");
+                            fileErrorList.setValue(lfn, "File");
                             break;
                     }
                 }
@@ -1342,91 +1278,13 @@ public:
     }
 };
 
-IPartDescriptor *queryMatchingRemotePart(IPartDescriptor *pdesc, IFileDescriptor *remoteFDesc, unsigned int partNum)
-{
-    if (!remoteFDesc)
-        return NULL;
-    IPartDescriptor *remotePDesc = remoteFDesc->queryPart(partNum);
-    if (!remotePDesc)
-        return NULL;
-    unsigned int crc, remoteCrc;
-    if (!pdesc || !pdesc->getCrc(crc)) //local crc not available, never DFS copied?
-        return remotePDesc;
-    if (remotePDesc->getCrc(remoteCrc) && remoteCrc==crc)
-        return remotePDesc;
-    return NULL;
-}
-
-inline bool isCopyFromCluster(IPartDescriptor *pdesc, unsigned clusterNo, const char *process)
-{
-    StringBuffer s;
-    return strieq(process, pdesc->queryOwner().getClusterGroupName(clusterNo, s));
-}
-
-inline bool checkClusterCount(UnsignedArray &counts, unsigned clusterNo, unsigned max)
-{
-    while (!counts.isItem(clusterNo))
-        counts.append(0);
-    unsigned count = counts.item(clusterNo);
-    if (count>=max)
-        return false;
-    counts.replace(++count, clusterNo);
-    return true;
-}
-
-inline void appendRemoteLocations(IPartDescriptor *pdesc, StringArray &locations, bool checkSelf)
-{
-    UnsignedArray clusterCounts;
-    unsigned numCopies = pdesc->numCopies();
-    for (unsigned copy = 0; copy < numCopies; copy++)
-    {
-        unsigned clusterNo = pdesc->copyClusterNum(copy);
-        if (!checkClusterCount(clusterCounts, clusterNo, 2))
-            continue;
-        if (checkSelf && isCopyFromCluster(pdesc, clusterNo, roxieName.str())) //don't add ourself
-            continue;
-        RemoteFilename r;
-        pdesc->getFilename(copy,r);
-        StringBuffer path;
-        locations.append(r.getRemotePath(path).str());
-    }
-}
-
 ILazyFileIO *createDynamicFile(const char *id, IPartDescriptor *pdesc, IPartDescriptor *remotePDesc, RoxieFileType fileType, int numParts, bool startCopy)
 {
-    IPropertyTree &partProps = pdesc->queryProperties();
-    offset_t dfsSize = partProps.getPropInt64("@size");
-    unsigned crc;
-    if (!pdesc->getCrc(crc))
-        crc = 0;
-    CDateTime fileDate;
-    if (checkFileDate)
-    {
-        const char *dateStr = partProps.queryProp("@modified");
-        fileDate.setString(dateStr);
-    }
-
-    StringArray localLocations;
     StringArray remoteLocations;
-
-    unsigned partNo = pdesc->queryPartIndex() + 1;
-    StringBuffer localFileName;
-
-    CDfsLogicalFileName dlfn;   
-    dlfn.set(id);
-    if (dlfn.isForeign())
-        dlfn.clearForeign();
-
-    const char *logicalname = dlfn.get();
-
-    makePhysicalPartName(logicalname, partNo, numParts, localFileName, false, DFD_OSdefault, baseDataDirectory);  // MORE - if we get the dataDirectory we can pass it in and possibly reuse an existing file
-
-    appendRemoteLocations(pdesc, remoteLocations, true);
     if (remotePDesc)
         appendRemoteLocations(remotePDesc, remoteLocations, false);
 
-    bool foregroundCopy = numParts==1 || (partNo==numParts && fileType==ROXIE_KEY);
-    return queryFileCache().lookupFile(id, partNo, fileType, localFileName, NULL, NULL, localLocations, remoteLocations, dfsSize, fileDate, false, true, startCopy, foregroundCopy, crcResources ? crc : 0, pdesc->queryOwner().isCompressed(), NULL);
+    return queryFileCache().lookupFile(id, fileType, pdesc, numParts, remoteLocations, startCopy);
 }
 
 //====================================================================================================
@@ -1779,7 +1637,7 @@ public:
                     Owned<IFileDescriptor> fDesc = sub.getFileDescriptor();
                     Owned<IFileDescriptor> remoteFDesc;
                     if (daliHelper)
-                        remoteFDesc.setown(daliHelper->checkClonedFromRemote(sub.queryLogicalName(), fDesc, cacheIt, writeAccess));
+                        remoteFDesc.setown(daliHelper->checkClonedFromRemote(sub.queryLogicalName(), fDesc, cacheIt));
                     addFile(sub.queryLogicalName(), fDesc.getClear(), remoteFDesc.getClear());
                 }
             }
@@ -1788,7 +1646,7 @@ public:
                 Owned<IFileDescriptor> fDesc = dFile->getFileDescriptor();
                 Owned<IFileDescriptor> remoteFDesc;
                 if (daliHelper)
-                    remoteFDesc.setown(daliHelper->checkClonedFromRemote(_lfn, fDesc, cacheIt, writeAccess));
+                    remoteFDesc.setown(daliHelper->checkClonedFromRemote(_lfn, fDesc, cacheIt));
                 addFile(dFile->queryLogicalName(), fDesc.getClear(), remoteFDesc.getClear());
             }
             bool tsSet = dFile->getModificationTime(fileTimeStamp);
@@ -2400,151 +2258,6 @@ extern void releaseSlaveDynamicFileCache()
     slaveDynamicFileCache.clear();
 }
 
-class CDiffFileInfoCache : public CInterface, implements IDiffFileInfoCache
-{
-    CriticalSection crit;
-    MapStringToDiffFileUsage diffFileInfoMap;  // store all diff / patch file location info - even if not used
-
-public:
-    IMPLEMENT_IINTERFACE;
-
-    CDiffFileInfoCache()
-    {
-    }
-
-    ~CDiffFileInfoCache()
-    {
-        HashIterator info(diffFileInfoMap);
-        for(info.first();info.isValid();info.next())
-        {
-            StringArray *a = *diffFileInfoMap.mapToValue(&info.query());
-            delete a;
-        }
-    }
-
-    virtual void saveDiffFileLocationInfo(const char *id, const StringArray &locations)
-    {
-        CriticalBlock b(crit);
-
-        StringArray *diffNames = 0;
-        StringArrayPtr *diffs = diffFileInfoMap.getValue(id);
-        if (diffs)
-            diffNames = *diffs;
-        else
-        {
-            diffNames = new StringArray;
-            diffFileInfoMap.setValue(id, diffNames);
-        }
-
-        ForEachItemIn(idx, locations)
-            diffNames->append(locations.item(idx));
-    }
-
-    virtual void saveDiffFileLocationInfo(const char *id, const char *location)
-    {
-        CriticalBlock b(crit);
-
-        StringArray *diffNames = 0;
-        StringArrayPtr *diffs = diffFileInfoMap.getValue(id);
-        if (diffs)
-        {
-            diffNames = *diffs;
-        }
-        else
-        {
-            diffNames = new StringArray;
-            diffFileInfoMap.setValue(id, diffNames);
-        }
-
-        diffNames->append(location);
-    }
-
-    virtual const char *queryDiffFileNames(StringBuffer &names)
-    {
-        names.append("<DiffFileNames>");
-        HashIterator diffs_iter(diffFileInfoMap);
-        for(diffs_iter.first();diffs_iter.isValid();diffs_iter.next())
-        {
-            IMapping &cur = diffs_iter.query();
-            const char *name = (const char *) cur.getKey();
-            names.appendf("<name>%s</name>", name);
-        }
-
-        names.append("</DiffFileNames>");
-        return names.str();
-    }
-
-    virtual void deleteDiffFiles(IPropertyTree *tree, IPropertyTree *goers)
-    {
-        Owned<IPropertyTreeIterator> diffFiles = tree->getElements("Patch");
-
-        ForEach(*diffFiles)
-        {
-            IPropertyTree &item = diffFiles->query();
-            StringBuffer id(item.queryProp("@id"));
-            StringArray **a = diffFileInfoMap.getValue(id.str());
-
-            if (!a)
-            {
-                if (id[0] == '~')
-                    id.remove(0,1);
-                else
-                    id.insert(0,'~');
-
-                a = diffFileInfoMap.getValue(id.str());
-            }
-
-            if (a)
-            {
-                ForEachItemIn(idx, **a)
-                {
-                    const char *name = (*a)->item(idx);
-                    try
-                    {
-                        OwnedIFile unneededFile = createIFile(name);
-                        unneededFile->remove();
-                        DBGLOG("deleted key diff file %s", name);
-                    }
-                    catch (IException *E)
-                    {
-                        // we don't care if there was an error - the file may not exist
-                        E->Release();
-                    }
-                }
-                // add Patch name to delete delta state file info
-                IPropertyTree *goer = createPTree("Patch");
-                goer->setProp("@id", id);
-                goer->setProp("@mode", "delete");
-                goers->addPropTree("Patch", goer);
-
-                item.setProp("@mode", "delete");
-            }
-        }
-    }
-
-};
-
-
-static CriticalSection diffFileInfoCacheCrit;
-static Owned<IDiffFileInfoCache> diffFileInfoCache;
-
-extern IDiffFileInfoCache *queryDiffFileInfoCache()
-{
-    if (!diffFileInfoCache)
-    {
-        CriticalBlock b(diffFileInfoCacheCrit);
-        if (!diffFileInfoCache)
-            diffFileInfoCache.setown(new CDiffFileInfoCache());
-    }
-    return diffFileInfoCache;
-}
-
-extern void releaseDiffFileInfoCache()
-{
-    CriticalBlock b(diffFileInfoCacheCrit);
-    diffFileInfoCache.clear();
-}
-
 
 // Initialization/termination
 MODULE_INIT(INIT_PRIORITY_STANDARD)
@@ -2732,90 +2445,3 @@ extern IRoxieWriteHandler *createRoxieWriteHandler(IRoxieDaliHelper *_daliHelper
 {
     return new CRoxieWriteHandler(_daliHelper, _dFile, _clusters);
 }
-
-#ifdef _USE_CPPUNIT
-#include <cppunit/extensions/HelperMacros.h>
-
-class LazyIOTest: public CppUnit::TestFixture  
-{
-    CPPUNIT_TEST_SUITE( LazyIOTest );
-        CPPUNIT_TEST(testAllCases);
-    CPPUNIT_TEST_SUITE_END();
-
-    void testIt(bool localPresent, bool localCorrect)
-    {
-        DBGLOG("Testing localPresent=%d localCorrect=%d", localPresent, localCorrect);
-        remove("cppfile_localfile1");
-        FILE *x = fopen("cppfile_localfile2", "wb");
-        assertex(x);
-        fputs("Test", x);
-        fclose(x);
-        if (localPresent)
-        {
-            if (localCorrect)
-                copyFile("cppfile_localfile1", "cppfile_localfile2");
-            else
-            {
-                FILE *x = fopen("cppfile_localfile1", "wb");
-                assertex(x);
-                fputs("Pink1", x);
-                fclose(x);
-            }
-        }
-        Owned<CRoxieFileCache> cache = new CRoxieFileCache(true);
-        StringArray remoteNames;
-        StringArray peerNames;
-        remoteNames.append("cppfile_localfile2");
-        CDateTime nullDT;
-        Owned<IFileIO> l1 = cache->lookupFile("cppfile_localfile1", 0, ROXIE_FILE, "cppfile_localfile1", NULL, NULL, peerNames, remoteNames, 4, nullDT, false, false, true, false, 0, false, NULL);
-        Owned<IFileIO> l2 = cache->lookupFile("cppfile_localfile1", 0, ROXIE_FILE, "cppfile_localfile1", NULL, NULL, peerNames, remoteNames, 4, nullDT, false, false, true, false, 0, false, NULL);
-        CPPUNIT_ASSERT(l1 == l2);
-        char buf[4];
-        l1->read(0, 4, buf);
-        if (memcmp(buf, "Test", 4)!=0)
-            DBGLOG("huh");
-        CPPUNIT_ASSERT(memcmp(buf, "Test", 4)==0);
-        cache->start();
-        cache->wait();
-        memset(buf, 0, 4);
-        l1->read(0, 4, buf);
-        CPPUNIT_ASSERT(memcmp(buf, "Test", 4)==0);
-        l1.clear();
-        l2.clear();
-        cache.clear();
-        DBGLOG("Tested localPresent=%d localCorrect=%d", localPresent, localCorrect);
-    }
-
-protected:
-    void testAllCases()
-    {
-        for (unsigned i1 = 0; i1 < 2; i1++)
-            for (unsigned i2 = 0; i2 < 2; i2++)
-                for (unsigned i3 = 0; i3 < 2; i3++)
-                    for (unsigned i4 = 0; i4 < 2; i4++)
-                        for (unsigned i5 = 0; i5 < 2; i5++)
-                        {
-                            useRemoteResources = i1==0;
-                            copyResources = i2==0;
-                            lazyOpen = i3==0;
-                            bool localPresent = i4==0;
-                            bool localCorrect = i5==0;
-                            try
-                            {
-                                testIt(localPresent, localCorrect);
-                            }
-                            catch (IException *E)
-                            {
-                                E->Release();
-                                CPPUNIT_ASSERT(!(localPresent && localCorrect) && !(useRemoteResources || copyResources));
-                            }
-                        }
-    }
-
-};
-
-CPPUNIT_TEST_SUITE_REGISTRATION( LazyIOTest );
-CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( LazyIOTest, "LazyIOTest" );
-
-#endif
-

roxie/ccd/ccdfile.hpp

@@ -43,15 +43,9 @@ interface ILazyFileIO : extends IFileIO
     virtual int getLinkCount() const = 0;
     virtual bool createHardFileLink() = 0;
 
-    virtual void setBaseIndexFileName(const char *val) =0;
-    virtual const char *queryBaseIndexFileName() = 0;
-    virtual void setPatchFile(ILazyFileIO *val) = 0;
-    virtual ILazyFileIO *queryPatchFile() = 0;
-
     virtual unsigned getLastAccessed() const = 0;
     virtual bool isOpen() const = 0;
     virtual void close() = 0;
-    virtual RoxieFileType getFileType() = 0;
     virtual void setCopying(bool copying) = 0;
     virtual bool isCopying() const = 0;
     virtual IMemoryMappedFile *queryMappedFile() = 0;
@@ -64,8 +58,8 @@ extern ILazyFileIO *createDynamicFile(const char *id, IPartDescriptor *pdesc, Ro
 
 interface IRoxieFileCache : extends IInterface
 {
-    virtual ILazyFileIO *lookupFile(const char *id, unsigned partNo, RoxieFileType fileType, const char *localLocation, const char *baseIndexFileName, ILazyFileIO *patchFile, const StringArray &peerRoxieCopiedLocationInfo, const StringArray &deployedLocationInfo, offset_t size, const CDateTime &modified, bool memFile, bool isRemote, bool startFileCopy, bool doForegroundCopy, unsigned crc, bool isCompressed, const char *lookupDali) = 0;
-    virtual RoxieFileStatus fileUpToDate(IFile *f, RoxieFileType fileType, offset_t size, const CDateTime &modified, unsigned crc, const char* id, bool isCompressed) = 0;
+    virtual ILazyFileIO *lookupFile(const char *lfn, RoxieFileType fileType, IPartDescriptor *pdesc, unsigned numParts, const StringArray &deployedLocationInfo, bool startFileCopy) = 0;
+    virtual RoxieFileStatus fileUpToDate(IFile *f, offset_t size, const CDateTime &modified, unsigned crc, bool isCompressed) = 0;
     virtual int numFilesToCopy() = 0;
     virtual void closeExpired(bool remote) = 0;
     virtual StringAttrMapping *queryFileErrorList() = 0;  // returns list of files that could not be open
@@ -145,7 +139,5 @@ extern IRoxieWriteHandler *createRoxieWriteHandler(IRoxieDaliHelper *_daliHelper
 
 extern IRoxieFileCache &queryFileCache();
 extern IMemoryFile *createMemoryFile(const char *fileName);
-extern IDiffFileInfoCache *queryDiffFileInfoCache();
-extern void releaseDiffFileInfoCache();
 
 #endif

roxie/ccd/ccdmain.cpp

@@ -1082,7 +1082,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
     cleanupPlugins();
     closeMulticastSockets();
     releaseSlaveDynamicFileCache();
-    releaseDiffFileInfoCache();
     releaseRoxieStateCache();
     setDaliServixSocketCaching(false);  // make sure it cleans up or you get bogus memleak reports
     setNodeCaching(false); // ditto

roxie/ccd/ccdstate.cpp

@@ -264,7 +264,7 @@ protected:
                 if (fd)
                 {
                     Owned <IResolvedFileCreator> result = createResolvedFile(fileName, NULL);
-                    Owned<IFileDescriptor> remoteFDesc = daliHelper->checkClonedFromRemote(fileName, fd, cacheIt, writeAccess);
+                    Owned<IFileDescriptor> remoteFDesc = daliHelper->checkClonedFromRemote(fileName, fd, cacheIt);
                     result->addSubFile(fd.getClear(), remoteFDesc.getClear());
                     return result.getClear();
                 }
@@ -987,6 +987,11 @@ public:
         return serverManager.getLink();
     }
 
+    void getInfo(StringBuffer &reply, const IRoxieContextLogger &logctx) const
+    {
+        reply.appendf(" <PackageSet id=\"%s\" querySet=\"%s\"/>\n", queryPackageId(), querySet.get());
+    }
+
     void resetStats(const char *queryId, const IRoxieContextLogger &logctx)
     {
         CriticalBlock b(updateCrit);
@@ -1297,6 +1302,16 @@ public:
         }
     }
 
+    void getInfo(StringBuffer &reply, const IRoxieContextLogger &logctx) const
+    {
+        reply.append("<PackageSets>\n");
+        ForEachItemIn(idx, allQueryPackages)
+        {
+            allQueryPackages.item(idx).getInfo(reply, logctx);
+        }
+        reply.append("</PackageSets>\n");
+    }
+
     void getStats(StringBuffer &reply, const char *id, const char *action, const char *graphName, const IRoxieContextLogger &logctx) const
     {
         ForEachItemIn(idx, allQueryPackages)
@@ -1667,10 +1682,6 @@ private:
                 defaultWarnTimeLimit[2] = control->getPropInt("@limit", 0);
                 topology->setPropInt("@defaultSLAPriorityTimeWarning", defaultWarnTimeLimit[2]);
             }
-            else if (stricmp(queryName, "control:deleteKeyDiffFiles")==0)
-            {
-                UNIMPLEMENTED;
-            }
             else if (stricmp(queryName, "control:deleteUnneededPhysicalFiles")==0)
             {
                 UNIMPLEMENTED;
@@ -1971,13 +1982,10 @@ private:
                     toXML(stats, reply);
                 }
             }
-            else if (stricmp(queryName, "control:queryDiffFileInfoCache")==0)
-            {
-                queryDiffFileInfoCache()->queryDiffFileNames(reply);
-            }
             else if (stricmp(queryName, "control:queryPackageInfo")==0)
             {
-                UNIMPLEMENTED;
+                ReadLockBlock readBlock(packageCrit);
+                allQueryPackages->getInfo(reply, logctx);
             }
             else if (stricmp(queryName, "control:queryStats")==0)
             {

testing/ecl/countgrouprollup.ecl

@@ -0,0 +1,9 @@
+
+
+ds := DATASET([], { unsigned id; });
+
+g := GROUP(NOFOLD(ds), id);
+
+r := ROLLUP(g, GROUP, TRANSFORM({unsigned i}, SELF.i := COUNT(ROWS(LEFT))));
+
+output(count(r));

testing/ecl/key/countgrouprollup.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>0</Result_1></Row>
+</Dataset>

testing/ecl/key/keyed_join.xml

@@ -691,6 +691,72 @@
  <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
 <Dataset name='Result 25'>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>DAVID     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>CLAIRE    SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Full keyed: simple limit(3,skip), LEFT OUTER </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Full keyed: simple limit(3,skip), </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+</Dataset>
+<Dataset name='Result 26'>
  <Row><name>Full keyed: postfilter after fetch           </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: postfilter after fetch           </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: postfilter after fetch           </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -756,7 +822,7 @@
  <Row><name>Full keyed: postfilter after fetch           </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: postfilter after fetch           </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 26'>
+<Dataset name='Result 27'>
  <Row><name>Half keyed: grouped inner                    </name><leftrec>CLAIRE    BAYLISS   1                        </leftrec><rightrec>CLAIRE    BAYLISS   1                        </rightrec></Row>
  <Row><name>Half keyed: grouped inner                    </name><leftrec>CLAIRE    BAYLISS   2                        </leftrec><rightrec>CLAIRE    BAYLISS   2                        </rightrec></Row>
  <Row><name>Half keyed: grouped inner                    </name><leftrec>CLAIRE    BAYLISS   3                        </leftrec><rightrec>CLAIRE    BAYLISS   3                        </rightrec></Row>
@@ -790,7 +856,7 @@
  <Row><name>Half keyed: grouped inner                    </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>KIMBERLY  SMITH     3                        </rightrec></Row>
  <Row><name>Half keyed: grouped inner                    </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>KIMBERLY  SMITH     4                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 27'>
+<Dataset name='Result 28'>
  <Row><name>Half keyed: grouped left only                </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped left only                </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped left only                </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -824,7 +890,7 @@
  <Row><name>Half keyed: grouped left only                </name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped left only                </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 28'>
+<Dataset name='Result 29'>
  <Row><name>Half keyed: grouped left outer               </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped left outer               </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped left outer               </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -890,7 +956,7 @@
  <Row><name>Half keyed: grouped left outer               </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>KIMBERLY  SMITH     3                        </rightrec></Row>
  <Row><name>Half keyed: grouped left outer               </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>KIMBERLY  SMITH     4                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 29'>
+<Dataset name='Result 30'>
  <Row><name>Half keyed: grouped skip                     </name><leftrec>CLAIRE    BAYLISS   1                        </leftrec><rightrec>CLAIRE    BAYLISS   1                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip                     </name><leftrec>CLAIRE    BAYLISS   2                        </leftrec><rightrec>CLAIRE    BAYLISS   2                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip                     </name><leftrec>CLAIRE    BAYLISS   3                        </leftrec><rightrec>CLAIRE    BAYLISS   3                        </rightrec></Row>
@@ -908,7 +974,7 @@
  <Row><name>Half keyed: grouped skip                     </name><leftrec>CLAIRE    SMITH     3                        </leftrec><rightrec>CLAIRE    SMITH     3                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip                     </name><leftrec>CLAIRE    SMITH     4                        </leftrec><rightrec>CLAIRE    SMITH     4                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 30'>
+<Dataset name='Result 31'>
  <Row><name>Half keyed: grouped skip, left only          </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip, left only          </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip, left only          </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -942,7 +1008,7 @@
  <Row><name>Half keyed: grouped skip, left only          </name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip, left only          </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 31'>
+<Dataset name='Result 32'>
  <Row><name>Half keyed: grouped skip, left outer         </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip, left outer         </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip, left outer         </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -992,17 +1058,17 @@
  <Row><name>Half keyed: grouped skip, left outer         </name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped skip, left outer         </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 32'>
- <Row><Result_32>64</Result_32></Row>
-</Dataset>
 <Dataset name='Result 33'>
+ <Row><Result_33>64</Result_33></Row>
 </Dataset>
 <Dataset name='Result 34'>
- <Row><Result_34>1</Result_34></Row>
 </Dataset>
 <Dataset name='Result 35'>
+ <Row><Result_35>1</Result_35></Row>
 </Dataset>
 <Dataset name='Result 36'>
+</Dataset>
+<Dataset name='Result 37'>
  <Row><name>Half keyed: grouped atmost(3), LEFT OUTER)   </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped atmost(3), LEFT OUTER)   </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped atmost(3), LEFT OUTER)   </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -1068,7 +1134,7 @@
  <Row><name>Half keyed: grouped atmost(3), LEFT OUTER)   </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped atmost(3), LEFT OUTER)   </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 37'>
+<Dataset name='Result 38'>
  <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -1102,7 +1168,73 @@
  <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 38'>
+<Dataset name='Result 39'>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>DAVID     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>CLAIRE    SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>Half keyed: grouped limit(3,skip), LEFT OUTER</name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BAYLISS   4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  DOLSON    1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  DOLSON    2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  DOLSON    3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  DOLSON    4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BILLINGTON1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BILLINGTON2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BILLINGTON3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  BILLINGTON4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  SMITH     1                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  SMITH     2                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
+ <Row><name>**onfail** Half keyed: grouped limit(3,skip),</name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
+</Dataset>
+<Dataset name='Result 40'>
  <Row><name>Full keyed: grouped inner                    </name><leftrec>CLAIRE    BAYLISS   1                        </leftrec><rightrec>CLAIRE    BAYLISS   1                        </rightrec></Row>
  <Row><name>Full keyed: grouped inner                    </name><leftrec>CLAIRE    BAYLISS   2                        </leftrec><rightrec>CLAIRE    BAYLISS   2                        </rightrec></Row>
  <Row><name>Full keyed: grouped inner                    </name><leftrec>CLAIRE    BAYLISS   3                        </leftrec><rightrec>CLAIRE    BAYLISS   3                        </rightrec></Row>
@@ -1136,7 +1268,7 @@
  <Row><name>Full keyed: grouped inner                    </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>KIMBERLY  SMITH     3                        </rightrec></Row>
  <Row><name>Full keyed: grouped inner                    </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>KIMBERLY  SMITH     4                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 39'>
+<Dataset name='Result 41'>
  <Row><name>Full keyed: grouped left only                </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped left only                </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped left only                </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -1170,7 +1302,7 @@
  <Row><name>Full keyed: grouped left only                </name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped left only                </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 40'>
+<Dataset name='Result 42'>
  <Row><name>Full keyed: grouped left outer               </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped left outer               </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped left outer               </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -1236,7 +1368,7 @@
  <Row><name>Full keyed: grouped left outer               </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>KIMBERLY  SMITH     3                        </rightrec></Row>
  <Row><name>Full keyed: grouped left outer               </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>KIMBERLY  SMITH     4                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 41'>
+<Dataset name='Result 43'>
  <Row><name>Full keyed: grouped skip                     </name><leftrec>CLAIRE    BAYLISS   1                        </leftrec><rightrec>CLAIRE    BAYLISS   1                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip                     </name><leftrec>CLAIRE    BAYLISS   2                        </leftrec><rightrec>CLAIRE    BAYLISS   2                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip                     </name><leftrec>CLAIRE    BAYLISS   3                        </leftrec><rightrec>CLAIRE    BAYLISS   3                        </rightrec></Row>
@@ -1254,7 +1386,7 @@
  <Row><name>Full keyed: grouped skip                     </name><leftrec>CLAIRE    SMITH     3                        </leftrec><rightrec>CLAIRE    SMITH     3                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip                     </name><leftrec>CLAIRE    SMITH     4                        </leftrec><rightrec>CLAIRE    SMITH     4                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 42'>
+<Dataset name='Result 44'>
  <Row><name>Full keyed: grouped skip, left only          </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip, left only          </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip, left only          </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -1288,7 +1420,7 @@
  <Row><name>Full keyed: grouped skip, left only          </name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip, left only          </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 43'>
+<Dataset name='Result 45'>
  <Row><name>Full keyed: grouped skip, left outer         </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip, left outer         </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip, left outer         </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -1338,17 +1470,17 @@
  <Row><name>Full keyed: grouped skip, left outer         </name><leftrec>KELLY     SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped skip, left outer         </name><leftrec>KELLY     SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 44'>
- <Row><Result_44>64</Result_44></Row>
-</Dataset>
-<Dataset name='Result 45'>
-</Dataset>
 <Dataset name='Result 46'>
- <Row><Result_46>1</Result_46></Row>
+ <Row><Result_46>64</Result_46></Row>
 </Dataset>
 <Dataset name='Result 47'>
 </Dataset>
 <Dataset name='Result 48'>
+ <Row><Result_48>1</Result_48></Row>
+</Dataset>
+<Dataset name='Result 49'>
+</Dataset>
+<Dataset name='Result 50'>
  <Row><name>Full keyed: grouped atmost(3), LEFT OUTER    </name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped atmost(3), LEFT OUTER    </name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped atmost(3), LEFT OUTER    </name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>
@@ -1414,7 +1546,7 @@
  <Row><name>Full keyed: grouped atmost(3), LEFT OUTER    </name><leftrec>KIMBERLY  SMITH     3                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped atmost(3), LEFT OUTER    </name><leftrec>KIMBERLY  SMITH     4                        </leftrec><rightrec>                    0                        </rightrec></Row>
 </Dataset>
-<Dataset name='Result 49'>
+<Dataset name='Result 51'>
  <Row><name>Full keyed: grouped limit(3,SKIP), LEFT OUTER</name><leftrec>DAVID     BAYLISS   1                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped limit(3,SKIP), LEFT OUTER</name><leftrec>DAVID     BAYLISS   2                        </leftrec><rightrec>                    0                        </rightrec></Row>
  <Row><name>Full keyed: grouped limit(3,SKIP), LEFT OUTER</name><leftrec>DAVID     BAYLISS   3                        </leftrec><rightrec>                    0                        </rightrec></Row>

testing/ecl/keyed_join.ecl

@@ -157,6 +157,9 @@ Out32 :=JOIN(DG_FlatFile, DG_FlatFileEvens, left.DG_firstname = right.DG_firstna
          AND left.DG_lastname=right.DG_lastname 
       , makePairFK(left, right, 'Full keyed: simple limit(3,skip), LEFT OUTER'), KEYED(DG_indexFileEvens), LIMIT(3,SKIP), LEFT OUTER);
 Out33 :=JOIN(DG_FlatFile, DG_FlatFileEvens, left.DG_firstname = right.DG_firstname 
+         AND left.DG_lastname=right.DG_lastname 
+      , makePairFK(left, right, 'Full keyed: simple limit(3,skip), LEFT OUTER'), KEYED(DG_indexFileEvens), LIMIT(3), ONFAIL(makePairFK(left, right, '**onfail** Full keyed: simple limit(3,skip), LEFT OUTER')), LEFT OUTER);
+Out34 :=JOIN(DG_FlatFile, DG_FlatFileEvens, left.DG_firstname = right.DG_firstname 
          AND left.DG_lastname=right.DG_lastname AND right.DG_parentID=99
       , makePairFK(left, right, 'Full keyed: postfilter after fetch'), KEYED(DG_indexFileEvens), LEFT OUTER);
 
@@ -204,6 +207,10 @@ Out51 :=JOIN(group(DG_FlatFile, DG_firstname), DG_indexFileEvens, left.DG_firstn
 Out52 :=JOIN(group(DG_FlatFile, DG_firstname), DG_indexFileEvens, left.DG_firstname = right.DG_firstname 
          AND left.DG_lastname=right.DG_lastname 
       , makePair(left, right, 'Half keyed: grouped limit(3,skip), LEFT OUTER)'), LIMIT(3,skip), LEFT OUTER);
+Out53 :=JOIN(group(DG_FlatFile, DG_firstname), DG_indexFileEvens, left.DG_firstname = right.DG_firstname 
+         AND left.DG_lastname=right.DG_lastname 
+      , makePair(left, right, 'Half keyed: grouped limit(3,skip), LEFT OUTER)'), LIMIT(3), ONFAIL(makePair(left, right, '**onfail** Half keyed: grouped limit(3,skip), LEFT OUTE')), LEFT OUTER);
+
 
 //fullkeyedjoins(fullkeyedgrouped, group(DG_FlatFile, DG_firstname), 'grouped');
 Out61 :=JOIN(group(DG_FlatFile, DG_firstname), DG_FlatFileEvens, left.DG_firstname = right.DG_firstname 
@@ -281,6 +288,7 @@ output(Out30);
 output(Out31);
 output(Out32);
 output(Out33);
+output(Out34);
 
 output(GROUP(Out41));
 output(GROUP(Out42));
@@ -294,6 +302,7 @@ output(COUNT(Out49));
 output(GROUP(Out50));
 output(GROUP(Out51));
 output(GROUP(Out52));
+output(GROUP(Out53));
 
 output(GROUP(Out61));
 output(GROUP(Out62));

testing/regress/hpcc/regression/regress.py

@@ -83,39 +83,51 @@ class Regression:
         return Suite('setup', self.setupDir, self.dir_a, self.dir_ex,
                      self.dir_r)
 
-    def runSuite(self, name, suite):
-        server = self.config.ip
+    def buildLogging(self, name):
         report = Report(name)
         curTime = time.strftime("%y-%m-%d-%H-%M")
         logName = name + "." + curTime + ".log"
+        log = os.path.join(self.logDir, logName)
+        self.log.addHandler(log, 'DEBUG')
+        return (report, log)
+
+    @staticmethod
+    def displayReport(report):
+        report[0].display(report[1])
+
+    def runSuite(self, name, suite):
+        report = self.buildLogging(name)
         if name == "setup":
             cluster = 'hthor'
         else:
             cluster = name
-        log = os.path.join(self.logDir, logName)
-        self.log.addHandler(log, 'DEBUG')
+
         logging.warn("Suite: %s" % name)
         logging.warn("Queries: %s" % repr(len(suite.getSuite())))
         cnt = 1
         for query in suite.getSuite():
-            logging.warn("%s. Test: %s" % (repr(cnt), query.ecl))
-            ECLCC().makeArchive(query)
-            res = ECLcmd().runCmd("run", cluster, query, report,
-                                  server=server, username=self.config.username,
-                                  password=self.config.password)
-            wuid = query.getWuid()
-            if wuid:
-                url = "http://" + self.config.server
-                url += "/WsWorkunits/WUInfo?Wuid="
-                url += wuid
-            if res:
-                logging.info("Pass %s" % wuid)
-                logging.info("URL %s" % url)
-            else:
-                if not wuid:
-                    logging.error("Fail No WUID")
-                else:
-                    logging.error("Fail %s" % wuid)
-                    logging.error("URL %s" % url)
+            self.runQuery(cluster, query, report, cnt)
             cnt += 1
-        report.display(log)
+        Regression.displayReport(report)
+
+    def runQuery(self, cluster, query, report, cnt=1):
+        logging.warn("%s. Test: %s" % (repr(cnt), query.ecl))
+        ECLCC().makeArchive(query)
+        res = ECLcmd().runCmd("run", cluster, query, report[0],
+                              server=self.config.ip,
+                              username=self.config.username,
+                              password=self.config.password)
+        wuid = query.getWuid()
+        if wuid:
+            url = "http://" + self.config.server
+            url += "/WsWorkunits/WUInfo?Wuid="
+            url += wuid
+        if res:
+            logging.info("Pass %s" % wuid)
+            logging.info("URL %s" % url)
+        else:
+            if not wuid:
+                logging.error("Fail No WUID")
+            else:
+                logging.error("Fail %s" % wuid)
+                logging.error("URL %s" % url)

testing/regress/regress

@@ -21,9 +21,10 @@
 
 import argparse
 import logging
+import os
 
 from hpcc.regression.regress import Regression
-
+from hpcc.util.ecl.file import ECLFile
 
 if __name__ == "__main__":
     prog = "regress"
@@ -45,6 +46,11 @@ if __name__ == "__main__":
     parser_run = subparsers.add_parser('run', help='run help')
     parser_run.add_argument('cluster', help="Run the cluster suite.",
                             nargs='?', default='setup')
+    parser_query = subparsers.add_parser('query', help='query help')
+    parser_query.add_argument('query', help="Run a single query.",
+                              nargs='?')
+    parser_query.add_argument('cluster', help="Cluster for single query run.",
+                            nargs='?', default='setup')
     args = parser.parse_args()
 
     suiteDir = ""
@@ -59,7 +65,15 @@ if __name__ == "__main__":
             print "Avaliable Clusters: "
             for i in Clusters:
                 print i
-        if 'cluster' in args:
+        if 'query' in args:
+            ecl = os.path.join(regress.dir_ec, args.query)
+            eclfile = ECLFile(ecl, regress.dir_a, regress.dir_ex,
+                              regress.dir_r)
+            if not eclfile.testSkip(args.cluster)['skip']:
+                report = regress.buildLogging(args.query)
+                regress.runQuery(args.cluster, eclfile, report)
+                Regression.displayReport(report)
+        elif 'cluster' in args:
             regress.bootstrap()
             if 'setup' in args.cluster:
                 regress.runSuite('setup', regress.setup)

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -1741,38 +1741,45 @@ public:
             input = NULL;
         }
     }
-    bool abortLimitAction(CJoinGroup *jg, OwnedConstThorRow &row)
+    void doAbortLimit(CJoinGroup *jg)
+    {
+        helper->onMatchAbortLimitExceeded();
+        CommonXmlWriter xmlwrite(0);
+        if (inputHelper && inputHelper->queryOutputMeta() && inputHelper->queryOutputMeta()->hasXML())
+        {
+            inputHelper->queryOutputMeta()->toXML((byte *) jg->queryLeft(), xmlwrite);
+        }
+        throw MakeActivityException(this, 0, "More than %d match candidates in keyed join for row %s", abortLimit, xmlwrite.str());
+    }
+    bool checkAbortLimit(CJoinGroup *jg)
     {
         if (jg->candidateCount() > abortLimit)
         {
             if (0 == (joinFlags & JFmatchAbortLimitSkips))
-            {
-                Owned<IException> e;
-                try
-                {
-                    helper->onMatchAbortLimitExceeded();
-                    CommonXmlWriter xmlwrite(0);
-                    if (inputHelper && inputHelper->queryOutputMeta() && inputHelper->queryOutputMeta()->hasXML())
-                    {
-                        inputHelper->queryOutputMeta()->toXML((byte *) jg->queryLeft(), xmlwrite);
-                    }
-                    throw MakeActivityException(this, 0, "More than %d match candidates in keyed join for row %s", abortLimit, xmlwrite.str());
-                }
-                catch (IException *_e)
-                {
-                    if (!onFailTransform)
-                        throw;
-                    e.setown(_e);
-                }
-                RtlDynamicRowBuilder trow(queryRowAllocator());
-                size32_t transformedSize = helper->onFailTransform(trow, jg->queryLeft(), defaultRight, 0, e.get());
-                if (0 != transformedSize)
-                    row.setown(trow.finalizeRowClear(transformedSize));
-            }
+                doAbortLimit(jg);
             return true;
         }
         return false;
     }
+    bool abortLimitAction(CJoinGroup *jg, OwnedConstThorRow &row)
+    {
+        Owned<IException> e;
+        try
+        {
+            return checkAbortLimit(jg);
+        }
+        catch (IException *_e)
+        {
+            if (!onFailTransform)
+                throw;
+            e.setown(_e);
+        }
+        RtlDynamicRowBuilder trow(queryRowAllocator());
+        size32_t transformedSize = helper->onFailTransform(trow, jg->queryLeft(), defaultRight, 0, e.get());
+        if (0 != transformedSize)
+            row.setown(trow.finalizeRowClear(transformedSize));
+        return true;
+    }
     virtual void onLimitExceeded() { return; }
     
     // IThorSlaveActivity overloaded methods
@@ -2210,12 +2217,15 @@ public:
                             OwnedConstThorRow abortRow;
                             if (abortLimitAction(doneJG, abortRow)) // discard lhs row (yes, even if it is an outer join)
                             {
-                                doneJG.clear();
+                                // don't clear doneJG, in preserveGroups case, it will advance to next, next time around.
+                                if (!preserveGroups)
+                                    doneJG.clear();
                                 if (abortRow.get())
                                 {
                                     dataLinkIncrement();
                                     return abortRow.getClear();
                                 }
+                                continue; // throw away this match
                             }
                             djg.set(next);
                         }
@@ -2244,25 +2254,22 @@ public:
                         doneGroupsDeQueued++;
 #endif
                         OwnedConstThorRow abortRow;
-                        if (abortLimitAction(doneJG, abortRow))
+                        if (!abortLimitAction(doneJG, abortRow))
                         {
-                            doneJG.clear();
-                            if (abortRow.get())
-                            {
-                                dataLinkIncrement();
-                                return abortRow.getClear();
-                            }
-                            continue;
-                        }
-                        else
                             djg.set(doneJG);
+                            currentMatched = djg->rowsSeen();
+                        }
+                        currentAdded = 0;
+                        currentMatchIdx = 0;
                         if (preserveGroups)
                             currentJoinGroupSize = 0;
                         else
                             doneJG.clear();
-                        currentAdded = 0;
-                        currentMatchIdx = 0;
-                        currentMatched = djg->rowsSeen();
+                        if (abortRow.get())
+                        {
+                            dataLinkIncrement();
+                            return abortRow.getClear();
+                        }
                     }
                     else
                     {
@@ -2292,16 +2299,9 @@ public:
                         {
                             unsigned candidateCount = (unsigned) (fpos & KEYEDJOIN_CANDIDATECOUNTMASK);
                             jg->noteCandidates(candidateCount);
-                            OwnedConstThorRow abortRow;
-                            if (abortLimitAction(jg, abortRow)) // might as well abort now if appropriate.
-                            {
-                                if (abortRow.get())
-                                {
-                                    dataLinkIncrement();
-                                    return abortRow.getClear();
-                                }
-                            }
-                            jg->noteEndCandidate();
+                            jg->noteEndCandidate(); // any onFail transform will be done when dequeued
+                            if (!onFailTransform) // unless going to transform later, check and abort now if necessary.
+                                checkAbortLimit(jg);
                         }
                         else
                         {

thorlcr/activities/loop/thloop.cpp

@@ -147,7 +147,6 @@ public:
 class CLoopActivityMaster : public CLoopActivityMasterBase
 {
     IHThorLoopArg *helper;
-    IThorBoundLoopGraph *boundGraph;
     unsigned flags;
     Owned<IBarrier> barrier;
 
@@ -191,7 +190,6 @@ public:
     void init()
     {
         helper = (IHThorLoopArg *) queryHelper();
-        boundGraph = queryContainer().queryLoopGraph();
         flags = helper->getFlags();
         if (TAKloopdataset == container.getKind())
             assertex(flags & IHThorLoopArg::LFnewloopagain);
@@ -201,25 +199,6 @@ public:
             if (container.queryOwner().isGlobal())
                 global = true;
         }
-        if (!global)
-            return;
-        initLoopResults(1);
-    }
-    void initLoopResults(unsigned loopCounter)
-    {
-        unsigned doLoopCounter = (flags & IHThorLoopArg::LFcounter) ? loopCounter : 0;
-        unsigned doLoopAgain = (flags & IHThorLoopArg::LFnewloopagain) ? helper->loopAgainResult() : 0;
-        ownedResults.setown(queryGraph().createThorGraphResults(3)); // will not be cleared until next sync
-        // ensures remote results are available, via owning activity (i.e. this loop act)
-        // so that when the master/slave result parts are fetched, it will retreive from the act, not the (already cleaed) graph localresults
-        ownedResults->setOwner(container.queryId());
-
-        boundGraph->prepareLoopResults(*this, ownedResults);
-        if (doLoopCounter) // cannot be 0
-            boundGraph->prepareCounterResult(*this, ownedResults, loopCounter, 2);
-        if (doLoopAgain) // cannot be 0
-            boundGraph->prepareLoopAgainResult(*this, ownedResults, helper->loopAgainResult());
-
     }
     void process()
     {
@@ -235,9 +214,23 @@ public:
             {
                 if (sync(loopCounter))
                     break;
-                if (loopCounter > 1)
-                    initLoopResults(loopCounter);
-                boundGraph->execute(*this, (flags & IHThorLoopArg::LFcounter)?loopCounter:0, ownedResults, (IRowWriterMultiReader *)NULL, 0, extractBuilder.size(), extractBuilder.getbytes());
+
+                // NB: This is exactly the same as the slave implementation up until the execute().
+                IThorBoundLoopGraph *boundGraph = queryContainer().queryLoopGraph();
+                unsigned condLoopCounter = (flags & IHThorLoopArg::LFcounter) ? loopCounter : 0;
+                unsigned loopAgain = (flags & IHThorLoopArg::LFnewloopagain) ? helper->loopAgainResult() : 0;
+                ownedResults.setown(queryGraph().createThorGraphResults(3)); // will not be cleared until next sync
+                // ensures remote results are available, via owning activity (i.e. this loop act)
+                // so that when the master/slave result parts are fetched, it will retreive from the act, not the (already cleaed) graph localresults
+                ownedResults->setOwner(container.queryId());
+
+                boundGraph->prepareLoopResults(*this, ownedResults);
+                if (condLoopCounter)
+                    boundGraph->prepareCounterResult(*this, ownedResults, condLoopCounter, 2);
+                if (loopAgain) // cannot be 0
+                    boundGraph->prepareLoopAgainResult(*this, ownedResults, loopAgain);
+
+                boundGraph->execute(*this, condLoopCounter, ownedResults, (IRowWriterMultiReader *)NULL, 0, extractBuilder.size(), extractBuilder.getbytes());
                 ++loopCounter;
                 if (flags & IHThorLoopArg::LFnewloopagain)
                 {

thorlcr/activities/loop/thloopslave.cpp

@@ -369,20 +369,20 @@ public:
                 loopPending->flush();
 
                 IThorBoundLoopGraph *boundGraph = queryContainer().queryLoopGraph();
-                unsigned doLoopCounter = (flags & IHThorLoopArg::LFcounter) ? loopCounter:0;
-                unsigned doLoopAgain = (flags & IHThorLoopArg::LFnewloopagain) ? helper->loopAgainResult() : 0;
+                unsigned condLoopCounter = (flags & IHThorLoopArg::LFcounter) ? loopCounter:0;
+                unsigned loopAgain = (flags & IHThorLoopArg::LFnewloopagain) ? helper->loopAgainResult() : 0;
                 ownedResults.setown(queryGraph().createThorGraphResults(3));
                 // ensures remote results are available, via owning activity (i.e. this loop act)
                 // so that when aggregate result is fetched from the master, it will retrieve from the act, not the (already cleaned) graph localresults
                 ownedResults->setOwner(container.queryId());
 
                 boundGraph->prepareLoopResults(*this, ownedResults);
-                if (doLoopCounter) // cannot be 0
-                    boundGraph->prepareCounterResult(*this, ownedResults, loopCounter, 2);
-                if (doLoopAgain) // cannot be 0
-                    boundGraph->prepareLoopAgainResult(*this, ownedResults, helper->loopAgainResult());
+                if (condLoopCounter) // cannot be 0
+                    boundGraph->prepareCounterResult(*this, ownedResults, condLoopCounter, 2);
+                if (loopAgain) // cannot be 0
+                    boundGraph->prepareLoopAgainResult(*this, ownedResults, loopAgain);
 
-                boundGraph->execute(*this, doLoopCounter, ownedResults, loopPending.getClear(), loopPendingCount, extractBuilder.size(), extractBuilder.getbytes());
+                boundGraph->execute(*this, condLoopCounter, ownedResults, loopPending.getClear(), loopPendingCount, extractBuilder.size(), extractBuilder.getbytes());
 
                 Owned<IThorResult> result0 = ownedResults->getResult(0);
                 curInput.setown(result0->getRowStream());