HPCC-8410 Move daregress to CPPUNIT

Moving daregress to CPPUNIT and adding more tests that were not commonly used.
Also making the CDistributedFile::detachLogical() private, to avoid misuse and
moving some common CPPUNIT code to a shared header.

Signed-off-by: Renato Golin <rengolin@hpccsystems.com>

dali/CMakeLists.txt

@@ -26,7 +26,6 @@ HPCC_ADD_SUBDIRECTORY (dfuplus "CLIENTTOOLS")
 HPCC_ADD_SUBDIRECTORY (dfuxref "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (dfuXRefLib "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (ft)
-HPCC_ADD_SUBDIRECTORY (regress "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (daliadmin "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (server "PLATFORM")
 HPCC_ADD_SUBDIRECTORY (treeview "PLATFORM")

dali/base/dadfs.cpp

@@ -3386,9 +3386,19 @@ public:
 #endif
     }
 
-    // MORE: This should really be an internal flag, but due to daregress bug
-    // I have to externalize it. Please, make it internal once daregress refactory
-    // is done.
+private:
+    /*
+     * Internal method (not in IDistributedFile interface) that is used
+     * when renaming files (so don't delete the physical representation).
+     *
+     * This is also used during CPPUINT tests, so we need to make them public
+     * only when tests are enabled (ie. non-production mode).
+     *
+     * See removeLogical()
+     */
+#ifdef _USE_CPPUNIT
+public:
+#endif
     void detachLogical(unsigned timeoutms=INFINITE)
     {
         detachLogic = true;
@@ -3401,6 +3411,8 @@ public:
         detachLogic = false;
     }
 
+public:
+
     void detach(unsigned timeoutms=INFINITE)
     {
         assert(proplockcount == 0 && "CDistributedFile detach: Some properties are still locked");
@@ -4886,13 +4898,6 @@ public:
         root.setown(conn->getRoot());
     }
 
-    // MORE: This should be removed when daregress gets refactored
-    // For super-files, it's not really a problem, since all removal is logical
-    void detachLogical(unsigned timeoutms=INFINITE)
-    {
-        detach(timeoutms);
-    }
-
     void detach(unsigned timeoutms=INFINITE)
     {   
         // will need more thought but this gives limited support for anon
@@ -10389,3 +10394,20 @@ IDFProtectedIterator *CDistributedFileDirectory::lookupProtectedFiles(const char
 {
     return new CDFProtectedIterator(owner,notsuper,superonly,defaultTimeout);
 }
+
+#ifdef _USE_CPPUNIT
+/*
+ * This method removes files only logically. removeEntry() used to do that, but the only
+ * external use for logical-files only is this test-suite, so I'd rather hack the test
+ * suite than expose the behaviour to more viewers.
+ */
+void removeLogical(const char *fname, IUserDescriptor *user) {
+    if (queryDistributedFileDirectory().exists(fname, user)) {
+        Owned<IDistributedFile> file = queryDistributedFileDirectory().lookup(fname, user, true);
+        CDistributedFile *f = QUERYINTERFACE(file.get(),CDistributedFile);
+        assert(f);
+        f->detachLogical();
+    }
+}
+
+#endif // _USE_CPPUNIT

dali/base/dadfs.hpp

@@ -225,7 +225,6 @@ interface IDistributedFile: extends IInterface
 
     virtual void attach(const char *logicalname,IUserDescriptor *user) = 0;     // attach to name in DFS
     virtual void detach(unsigned timeoutms=INFINITE) = 0;                       // no longer attached to name in DFS
-    virtual void detachLogical(unsigned timeoutms=INFINITE) = 0;                // INTERNAL ONLY, please do not use!
 
     virtual IPropertyTree &queryAttributes() = 0;                               // DFile attributes
 

dali/regress/CMakeLists.txt

@@ -1,53 +0,0 @@
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
-
-
-# Component: daregress 
-#####################################################
-# Description:
-# ------------
-#    Cmake Input File for daregress
-#####################################################
-
-project( daregress ) 
-
-set (    SRCS 
-         daregdfs.cpp 
-         daregress.cpp 
-    )
-
-include_directories ( 
-         ./../../common/remote 
-         ./../server 
-         ./../base 
-         ./../../system/mp 
-         . 
-         ./../../system/include 
-         ./../../system/jlib 
-    )
-
-ADD_DEFINITIONS( -D_CONSOLE )
-
-HPCC_ADD_EXECUTABLE ( daregress ${SRCS} )
-install ( TARGETS daregress RUNTIME DESTINATION ${EXEC_DIR} )
-target_link_libraries ( daregress  
-         jlib
-         mp 
-         hrpc 
-         remote 
-         dalibase 
-    )
-

dali/regress/daregdfs.cpp

@@ -1,693 +0,0 @@
-/*##############################################################################
-
-    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-
-    Licensed under the Apache License, Version 2.0 (the "License");
-    you may not use this file except in compliance with the License.
-    You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-    Unless required by applicable law or agreed to in writing, software
-    distributed under the License is distributed on an "AS IS" BASIS,
-    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-    See the License for the specific language governing permissions and
-    limitations under the License.
-############################################################################## */
-
-//TBD: add + test interleave (==number of sub-files)
-
-
-#include "platform.h"
-#include "jlib.hpp"
-#include "jmisc.hpp"
-#include "mpbase.hpp"
-#include "jptree.hpp"
-
-#include "dadfs.hpp"
-#include "dafdesc.hpp"
-#include "dasess.hpp"
-
-//#define COMPAT
-
-
-
-const char *filelist=
-"thor_data400::gong_delete_plus,"
-"thor_data400::in::npanxx,"
-"thor_data400::tpm_deduped,"
-"thor_data400::base::movers_ingest_ll,"
-"thor_hank::cemtemp::fldl,"
-"thor_data400::patch,"
-"thor_data400::in::flvehreg_01_prethor_upd200204_v3,"
-"thor_data400::in::flvehreg_01_prethor_upd20020625_v3_flag,"
-"thor_data400::in::flvehreg_01_prethor_upd20020715_v3,"
-"thor_data400::in::flvehreg_01_prethor_upd20020715_v3_v3_flag,"
-"thor_data400::in::flvehreg_01_prethor_upd20020816_v3,"
-"thor_data400::in::flvehreg_01_prethor_upd20020816_v3_flag,"
-"thor_data400::in::flvehreg_01_prethor_upd20020625_v3,"
-"thor_data400::in::fl_lic_prethor_200208v2,"
-"thor_data400::in::fl_lic_prethor_200209,"
-"thor_data400::in::fl_lic_prethor_200210,"
-"thor_data400::in::fl_lic_prethor_200210_reclean,"
-"thor_data400::in::fl_lic_upd_200301,"
-"thor_data400::in::fl_lic_upd_200302,"
-"thor_data400::in::oh_lic_200209,"
-"thor_data400::in::ohio_lic_upd_200210,"
-"thor_data400::prepped_for_keys,"
-"a0e65__w20060224-155748,"
-"common_did,"
-"test::ftest1,"
-"thor_data50::BASE::chunk,"
-"hthor::key::codes_v320040901,"
-"thor_data400::in::ucc_direct_ok_99999999_event_20060124,"
-"thor400::ks_work::distancedetails";
-
-
-void testMultiCluster()
-{
-    Owned<IGroup> grp1 = createIGroup("192.168.51.1-5");
-    Owned<IGroup> grp2 = createIGroup("192.168.16.1-5");
-    Owned<IGroup> grp3 = createIGroup("192.168.53.1-5");
-    queryNamedGroupStore().add("testgrp1",grp1);
-    queryNamedGroupStore().add("testgrp2",grp2);
-    queryNamedGroupStore().add("testgrp3",grp3);
-
-    Owned<IFileDescriptor> fdesc = createFileDescriptor();
-    fdesc->setDefaultDir("/c$/thordata/test");
-    fdesc->setPartMask("testfile1._$P$_of_$N$");
-    fdesc->setNumParts(5);
-    ClusterPartDiskMapSpec mapping;
-    fdesc->addCluster(grp1,mapping);
-    fdesc->addCluster(grp2,mapping);
-    fdesc->addCluster(grp3,mapping);
-    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
-    queryDistributedFileDirectory().removeEntry("test::testfile1",UNKNOWN_USER);
-    file->attach("test::testfile1",UNKNOWN_USER);
-    StringBuffer name;
-    unsigned i;
-    for (i=0;i<file->numClusters();i++)
-        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
-    file.clear();
-    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
-    for (i=0;i<file->numClusters();i++)
-        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
-    file.clear();
-    file.setown(queryDistributedFileDirectory().lookup("test::testfile1@testgrp1",UNKNOWN_USER));
-    for (i=0;i<file->numClusters();i++)
-        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
-    file.clear();
-    queryDistributedFileDirectory().removeEntry("test::testfile1@testgrp2",UNKNOWN_USER);
-    file.setown(queryDistributedFileDirectory().lookup("test::testfile1",UNKNOWN_USER));
-    for (i=0;i<file->numClusters();i++)
-        PROGLOG("cluster[%d] = %s",i,file->getClusterName(i,name.clear()).str());
-}
-
-
-
-
-interface IChecker
-{
-    virtual void title(unsigned n,const char *s)=0;
-    virtual void add(const char *s,__int64 v)=0;
-    virtual void add(const char *s,const char* v)=0;
-    virtual void add(unsigned n,const char *s,__int64 v)=0;
-    virtual void add(unsigned n,const char *s,const char* v)=0;
-    virtual void error(const char *txt)=0;
-};
-
-void checkFilePart(IChecker *checker,IDistributedFilePart *part,bool blocked)
-{
-    StringBuffer tmp;
-    checker->add("getPartIndex",part->getPartIndex());
-    unsigned n = part->numCopies();
-    checker->add("numCopies",part->numCopies());
-    checker->add("maxCopies",n);
-    RemoteFilename rfn;
-    for (unsigned copy=0;copy<n;copy++) {
-        INode *node = part->queryNode(copy);
-        if (node)
-            checker->add(copy,"queryNode",node->endpoint().getUrlStr(tmp.clear()).str());
-        else
-            checker->error("missing node");
-        checker->add(copy,"getFilename",part->getFilename(rfn,copy).getRemotePath(tmp.clear()).str());
-    }
-    checker->add("getPartName",part->getPartName(tmp.clear()).str());
-#ifndef COMPAT
-    checker->add("getPartDirectory",part->getPartDirectory(tmp.clear()).str());
-#endif
-    checker->add("queryProperties()",toXML(&part->queryAttributes(),tmp.clear()).str());
-    checker->add("isHost",part->isHost()?1:0);
-    checker->add("getFileSize",part->getFileSize(false,false));
-    CDateTime dt;
-    if (part->getModifiedTime(false,false,dt))
-        dt.getString(tmp.clear());
-    else
-        tmp.clear().append("nodatetime");
-    checker->add("getModifiedTime",tmp.str());
-    unsigned crc;
-    if (part->getCrc(crc)&&!blocked)
-        checker->add("getCrc",crc);
-    else
-        checker->add("getCrc","nocrc");
-}
-
-
-void checkFile(IChecker *checker,IDistributedFile *file)
-{
-    StringBuffer tmp;
-    checker->add("queryLogicalName",file->queryLogicalName());
-    unsigned np = file->numParts();
-    checker->add("numParts",np);
-    checker->add("queryDefaultDir",file->queryDefaultDir());
-    if (np>1)
-        checker->add("queryPartMask",file->queryPartMask());
-    checker->add("queryProperties()",toXML(&file->queryAttributes(),tmp.clear()).str());
-    CDateTime dt;
-    if (file->getModificationTime(dt))
-        dt.getString(tmp.clear());
-    else
-        tmp.clear().append("nodatetime");
-
-    // Owned<IFileDescriptor> desc = getFileDescriptor();
-    // checkFileDescriptor(checker,desc);
-
-    //virtual bool existsPhysicalPartFiles(unsigned short port) = 0;                // returns true if physical patrs all exist (on primary OR secondary)
-
-    //Owned<IPropertyTree> tree = getTreeCopy();
-    //checker->add("queryProperties()",toXML(tree,tmp.clear()).str());
-
-    checker->add("getFileSize",file->getFileSize(false,false));
-    bool blocked;
-    checker->add("isCompressed",file->isCompressed(&blocked)?1:0);
-    checker->add("blocked",blocked?1:0);
-    unsigned csum;
-    if (file->getFileCheckSum(csum)&&!blocked) 
-        checker->add("getFileCheckSum",csum);
-    else
-        checker->add("getFileCheckSum","nochecksum");
-    checker->add("isSubFile",file->isSubFile()?1:0);
-    StringBuffer clustname;
-    checker->add("queryClusterName(0)",file->getClusterName(0,clustname).str());
-    for (unsigned i=0;i<np;i++) {
-        Owned<IDistributedFilePart> part = file->getPart(i);
-        if (part)
-            checkFilePart(checker,part,blocked);
-    }
-        
-}
-
-void checkFiles(const char *fn)
-{
-    class cChecker: implements IChecker
-    {
-    public:
-        virtual void title(unsigned n,const char *s)
-        {
-            printf("Title[%d]='%s'\n",n,s);
-        }
-        virtual void add(const char *s,__int64 v)
-        {
-            printf("%s=%"I64F"d\n",s,v);
-        }
-        virtual void add(const char *s,const char* v)
-        {
-            printf("%s='%s'\n",s,v);
-        }
-        virtual void add(unsigned n,const char *s,__int64 v)
-        {
-            printf("%s[%d]=%"I64F"d\n",s,n,v);
-        }
-        virtual void add(unsigned n,const char *s,const char* v)
-        {
-            printf("%s[%d]='%s'\n",s,n,v);
-        }
-        virtual void error(const char *txt)
-        {
-            printf("ERROR '%s'\n",txt);
-        }
-    } checker;
-    unsigned start = msTick();
-    unsigned slowest = 0;
-    StringAttr slowname;
-    if (fn) {
-        checker.title(1,fn);
-        try {
-            Owned<IDistributedFile> file=queryDistributedFileDirectory().lookup(fn,UNKNOWN_USER);
-            if (!file)
-                printf("file '%s' not found\n",fn);
-            else
-                checkFile(&checker,file);
-        }
-        catch (IException *e) {
-            StringBuffer str;
-            e->errorMessage(str);
-            e->Release();
-            checker.error(str.str());
-        }
-    }
-    else {
-        Owned<IDistributedFileIterator> iter = queryDistributedFileDirectory().getIterator("*",false,UNKNOWN_USER);
-        unsigned i=0;
-        unsigned ss = msTick();
-        ForEach(*iter) {
-            i++;
-            StringBuffer lfname;
-            iter->getName(lfname);
-            checker.title(i,lfname.str());
-            try {
-                IDistributedFile &file=iter->query();
-                checkFile(&checker,&file);
-                unsigned t = (msTick()-ss);
-                if (t>slowest) {
-                    slowest = t;
-                    slowname.set(lfname.str());
-                }
-            }
-            catch (IException *e) {
-                StringBuffer str;
-                e->errorMessage(str);
-                e->Release();
-                checker.error(str.str());
-            }
-            ss = msTick();
-        }
-    }
-    unsigned t = msTick()-start;
-    printf("Complete in %ds\n",t/1000);
-    if (!slowname.isEmpty())
-        printf("Slowest %s = %dms\n",slowname.get(),slowest);
-};
-
-void testSingleFile()
-{
-//  const char *fn = "thor_data400::prepped_for_keys";
-//  const char *fn = "a0e65__w20060224-155748";
-//  const char *fn = "common_did";
-//  const char *fn = "test::ftest1";
-//  const char *fn = "thor_data50::BASE::chunk";
-//  const char *fn = "hthor::key::codes_v320040901";
-//  const char *fn = "thor_data400::in::ucc_direct_ok_99999999_event_20060124";
-    const char *fn = "thor400::ks_work::distancedetails";
-    checkFiles(fn);
-}
-
-void testFiles()
-{
-    StringBuffer fn;
-    const char *s = filelist;
-    unsigned slowest = 0;
-    StringAttr slowname;
-    unsigned tot = 0;
-    unsigned n = 0;
-    while (*s) {
-        fn.clear();
-        while (*s==',')
-            s++;
-        while (*s&&(*s!=','))
-            fn.append(*(s++));
-        if (fn.length()) {
-            n++;
-            unsigned ss = msTick();
-            checkFiles(fn);
-            unsigned t = (msTick()-ss);
-            if (t>slowest) {
-                slowest = t;
-                slowname.set(fn);
-            }
-            tot += t;
-        }
-    }
-    printf("Complete in %ds avg %dms\n",tot/1000,tot/(n?n:1));
-    if (!slowname.isEmpty())
-        printf("Slowest %s = %dms\n",slowname.get(),slowest);
-}
-
-#ifndef COMPAT
-
-void testDFile(ClusterPartDiskMapSpec &map)
-{
-    {   // 1: single part file old method
-#define TN "1"
-        queryDistributedFileDirectory().removeEntry("test::ftest"TN,UNKNOWN_USER);
-        Owned<IFileDescriptor> fdesc = createFileDescriptor();
-        RemoteFilename rfn;
-        rfn.setRemotePath("//10.150.10.1/c$/thordata/test/ftest"TN"._1_of_1");
-        fdesc->setPart(0,rfn);
-        fdesc->endCluster(map);
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
-        file->attach("test::ftest"TN,UNKNOWN_USER);
-#undef TN
-    }
-    {   // 2: single part file new method
-#define TN "2"
-        queryDistributedFileDirectory().removeEntry("test::ftest"TN,UNKNOWN_USER);
-        Owned<IFileDescriptor> fdesc = createFileDescriptor();
-        fdesc->setPartMask("ftest"TN"._$P$_of_$N$");
-        fdesc->setNumParts(1);
-        Owned<IGroup> grp = createIGroup("10.150.10.1");
-        fdesc->addCluster(grp,map);
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
-        file->attach("test::ftest"TN,UNKNOWN_USER);
-#undef TN
-    }
-    Owned<IGroup> grp3 = createIGroup("10.150.10.1,10.150.10.2,10.150.10.3");
-    queryNamedGroupStore().add("__testgroup3__",grp3,true);
-    {   // 3: three parts file old method
-#define TN "3"
-        queryDistributedFileDirectory().removeEntry("test::ftest"TN,UNKNOWN_USER);
-        Owned<IFileDescriptor> fdesc = createFileDescriptor();
-        RemoteFilename rfn;
-        rfn.setRemotePath("//10.150.10.1/c$/thordata/test/ftest"TN"._1_of_3");
-        fdesc->setPart(0,rfn);
-        rfn.setRemotePath("//10.150.10.2/c$/thordata/test/ftest"TN"._2_of_3");
-        fdesc->setPart(1,rfn);
-        rfn.setRemotePath("//10.150.10.3/c$/thordata/test/ftest"TN"._3_of_3");
-        fdesc->setPart(2,rfn);
-        fdesc->endCluster(map);
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
-        file->attach("test::ftest"TN,UNKNOWN_USER);
-#undef TN
-    }
-    {   // 4: three part file new method
-#define TN "4"
-        queryDistributedFileDirectory().removeEntry("test::ftest"TN,UNKNOWN_USER);
-        Owned<IFileDescriptor> fdesc = createFileDescriptor();
-        fdesc->setPartMask("ftest"TN"._$P$_of_$N$");
-        fdesc->setNumParts(3);
-        fdesc->addCluster(grp3,map);
-        Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
-        file->attach("test::ftest"TN,UNKNOWN_USER);
-#undef TN
-    }
-}
-
-void testDFile() 
-{
-    ClusterPartDiskMapSpec map;
-    testDFile(map);
-}
-
-#endif
-
-void mt(const char *s)
-{
-    printf("======================================\n");
-    printf("'%s'\n",s);
-    StringArray array;
-    RemoteMultiFilename::expand(s,array);
-    StringBuffer s1;
-    RemoteMultiFilename::tostr(array,s1);
-    printf("'%s'\n",s1.str());
-    ForEachItemIn(i1,array)
-        printf("%d:'%s'\n",i1,array.item(i1));
-    RemoteMultiFilename rmfn;
-    rmfn.append(s);
-    SocketEndpoint ep("10.150.10.80");
-    rmfn.setEp(ep);
-    ForEachItemIn(i2,rmfn) {
-        StringBuffer tmp;
-        rmfn.item(i2).getLocalPath(tmp);
-        printf("%d:'%s'\n",i2,tmp.str());
-    }   
-    StringBuffer sd;
-    StringBuffer sdt;
-    StringBuffer sdr;
-    StringBuffer sm;
-    const char *sds = splitDirMultiTail(s,sd,sdt);
-    printf("splitDirMultiTail('%s','%s')\n",sd.str(),sds);
-    mergeDirMultiTail(sd.str(),sdt.str(),sm);
-    printf("mergeDirMultiTail('%s')\n",sm.str());
-    removeRelativeMultiPath(sm.str(),sd.str(),sdt.clear());
-    printf("removeRelativeMultiPath('%s')\n",sdt.str());
-}
-
-#ifndef COMPAT
-
-void dispFDesc(IFileDescriptor *fdesc)
-{
-    printf("======================================\n");
-    Owned<IPropertyTree> pt = createPTree("File");
-    fdesc->serializeTree(*pt);
-    StringBuffer out;
-    toXML(pt,out);
-    printf("%s\n",out.str());
-    Owned<IFileDescriptor> fdesc2 = deserializeFileDescriptorTree(pt);
-    toXML(pt,out.clear());
-    printf("%s\n",out.str());
-    unsigned np = fdesc->numParts();
-    unsigned ncl = fdesc->numClusters();
-    printf("numclusters = %d, numparts=%d\n",ncl,np);
-    for (unsigned pass=0;pass<1;pass++) {
-        for (unsigned ip=0;ip<np;ip++) {
-            IPartDescriptor *part = fdesc->queryPart(ip);
-            unsigned nc = part->numCopies();
-            for (unsigned ic=0;ic<nc;ic++) {
-                StringBuffer tmp1;
-                StringBuffer tmp2;
-                StringBuffer tmp3;
-                StringBuffer tmp4;
-                RemoteFilename rfn;
-                bool blocked;
-                out.clear().appendf("%d,%d: '%s' '%s' '%s' '%s' '%s' '%s' %s%s%s",ip,ic,
-                    part->getDirectory(tmp1,ic).str(),
-                    part->getTail(tmp2).str(),
-                    part->getPath(tmp3,ic).str(),
-                    fdesc->getFilename(ip,ic,rfn).getRemotePath(tmp4).str(),  // multi TBD
-                    fdesc->queryPartMask()?fdesc->queryPartMask():"",
-                    fdesc->queryDefaultDir()?fdesc->queryDefaultDir():"",
-                    fdesc->isGrouped()?"GROUPED ":"",
-                    fdesc->queryKind()?fdesc->queryKind():"",
-                    fdesc->isCompressed(&blocked)?(blocked?" BLOCKCOMPRESSED":" COMPRESSED"):""
-                );
-                printf("%s\n",out.str());
-                if (1) {
-                    MemoryBuffer mb;
-                    part->serialize(mb);
-                    Owned<IPartDescriptor> copypart;
-                    copypart.setown(deserializePartFileDescriptor(mb));
-                    StringBuffer out2;
-                    out2.appendf("%d,%d: '%s' '%s' '%s' '%s' '%s' '%s' %s%s%s",ip,ic,
-                        copypart->getDirectory(tmp1.clear(),ic).str(),
-                        copypart->getTail(tmp2.clear()).str(),
-                        copypart->getPath(tmp3.clear(),ic).str(),
-                        copypart->getFilename(ic,rfn).getRemotePath(tmp4.clear()).str(),  // multi TBD
-                        copypart->queryOwner().queryPartMask()?copypart->queryOwner().queryPartMask():"",
-                        copypart->queryOwner().queryDefaultDir()?copypart->queryOwner().queryDefaultDir():"",
-                        copypart->queryOwner().isGrouped()?"GROUPED ":"",
-                        copypart->queryOwner().queryKind()?copypart->queryOwner().queryKind():"",
-                        copypart->queryOwner().isCompressed(&blocked)?(blocked?" BLOCKCOMPRESSED":" COMPRESSED"):""
-                    );
-                    if (strcmp(out.str(),out2.str())!=0)
-                        printf("FAILED!\n%s\n%s\n",out.str(),out2.str());
-                    pt.setown(createPTree("File"));
-                    copypart->queryOwner().serializeTree(*pt);
-                    StringBuffer out;
-                    toXML(pt,out);
-                //  printf("%d,%d: \n%s\n",ip,ic,out.str());
-
-                }
-            }
-        }
-    }
-}
-
-#endif
-
-void testGrp(SocketEndpointArray &epa)
-{
-    Owned<IGroup> grp = createIGroup(epa);
-    StringBuffer s;
-    grp->getText(s);
-    printf("'%s'\n",s.str());
-    Owned<IGroup> grp2 = createIGroup(s.str());
-    if (grp->compare(grp2)!=GRidentical) {
-        grp->getText(s.clear());
-        printf("^FAILED! %s\n",s.str());
-    }
-}
-
-void testGrp()
-{
-    SocketEndpointArray epa;
-    SocketEndpoint ep;
-    Owned<IGroup> grp;
-    testGrp(epa);
-    ep.set("10.150.10.80");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.81");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.82");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.83:111");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.84:111");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.84:111");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.84:111");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.84:111");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.85:111");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.86:111");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.87");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.87");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.10.88");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.150.11.88");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.173.10.88");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("10.173.10.88:22222");
-    epa.append(ep);
-    testGrp(epa);
-    ep.set("192.168.10.88");
-    epa.append(ep);
-    testGrp(epa);
-}
-
-
-#ifndef COMPAT
-
-void testDF1()
-{
-    Owned<IFileDescriptor> fdesc = createFileDescriptor();
-    Owned<IPropertyTree> pt = createPTree("Attr");
-    RemoteFilename rfn;
-    rfn.setRemotePath("//10.150.10.80/c$/thordata/test/part._1_of_3");
-    pt->setPropInt("@size",123);
-    fdesc->setPart(0,rfn,pt);
-    rfn.setRemotePath("//10.150.10.81/c$/thordata/test/part._2_of_3");
-    pt->setPropInt("@size",456);
-    fdesc->setPart(1,rfn,pt);
-    rfn.setRemotePath("//10.150.10.82/c$/thordata/test/part._3_of_3");
-    pt->setPropInt("@size",789);
-    fdesc->setPart(2,rfn,pt);
-    dispFDesc(fdesc);
-    Owned<IDistributedFile> file = queryDistributedFileDirectory().createNew(fdesc);
-    {
-        DistributedFilePropertyLock lock(file);
-        lock.queryAttributes().setProp("@testing","1");
-    }
-    file->attach("testing::propfile2",UNKNOWN_USER);
-}
-
-void testDF2() // 4*3 superfile
-{
-    Owned<IFileDescriptor> fdesc = createFileDescriptor();
-    Owned<IPropertyTree> pt = createPTree("Attr");
-    RemoteFilename rfn;
-    rfn.setRemotePath("//10.150.10.80/c$/thordata/test/partone._1_of_3");
-    pt->setPropInt("@size",1231);
-    fdesc->setPart(0,rfn,pt);
-    rfn.setRemotePath("//10.150.10.80/c$/thordata/test/parttwo._1_of_3");
-    pt->setPropInt("@size",1232);
-    fdesc->setPart(1,rfn,pt);
-    rfn.setRemotePath("//10.150.10.80/c$/thordata/test/partthree._1_of_3");
-    pt->setPropInt("@size",1233);
-    fdesc->setPart(2,rfn,pt);
-    rfn.setRemotePath("//10.150.10.80/c$/thordata/test2/partfour._1_of_3");
-    pt->setPropInt("@size",1234);
-    fdesc->setPart(3,rfn,pt);
-    rfn.setRemotePath("//10.150.10.81/c$/thordata/test/partone._2_of_3");
-    pt->setPropInt("@size",4565);
-    fdesc->setPart(4,rfn,pt);
-    rfn.setRemotePath("//10.150.10.81/c$/thordata/test/parttwo._2_of_3");
-    pt->setPropInt("@size",4566);
-    fdesc->setPart(5,rfn,pt);
-    rfn.setRemotePath("//10.150.10.81/c$/thordata/test/partthree._2_of_3");
-    pt->setPropInt("@size",4567);
-    fdesc->setPart(6,rfn,pt);
-    rfn.setRemotePath("//10.150.10.81/c$/thordata/test2/partfour._2_of_3");
-    pt->setPropInt("@size",4568);
-    fdesc->setPart(7,rfn,pt);
-    rfn.setRemotePath("//10.150.10.82/c$/thordata/test/partone._3_of_3");
-    pt->setPropInt("@size",7899);
-    fdesc->setPart(8,rfn,pt);
-    rfn.setRemotePath("//10.150.10.82/c$/thordata/test/parttwo._3_of_3");
-    pt->setPropInt("@size",78910);
-    fdesc->setPart(9,rfn,pt);
-    rfn.setRemotePath("//10.150.10.82/c$/thordata/test/partthree._3_of_3");
-    pt->setPropInt("@size",78911);
-    fdesc->setPart(10,rfn,pt);
-    rfn.setRemotePath("//10.150.10.82/c$/thordata/test2/partfour._3_of_3");
-    pt->setPropInt("@size",78912);
-    fdesc->setPart(11,rfn,pt);
-    ClusterPartDiskMapSpec mspec;
-    mspec.interleave = 4;
-    fdesc->endCluster(mspec);
-    dispFDesc(fdesc);
-}
-
-void testMisc()
-{
-    ClusterPartDiskMapSpec mspec;
-    Owned<IGroup> grp = createIGroup("10.150.10.1-3");
-    RemoteFilename rfn;
-    for (unsigned i=0;i<3;i++)
-        for (unsigned ic=0;ic<mspec.defaultCopies;ic++) {
-            constructPartFilename(grp,i+1,3,(i==1)?"test.txt":NULL,"test._$P$_of_$N$","/c$/thordata/test",ic,mspec,rfn);
-            StringBuffer tmp;
-            printf("%d,%d: %s\n",i,ic,rfn.getRemotePath(tmp).str());
-        }
-}
-
-#endif
-
-
-
-void runDafsTest()
-{
-#if 1
-//  checkFiles(NULL);
-//  testSingleFile();
-//  testFiles();
-//  testMultiCluster();
-    testDF1();
-    return; 
-#endif
-
-#if 0
-    StringArray test;
-    test.append("a1");
-    test.append("b2");
-    test.append("c3");
-    test.replace("xxx",1);
-#endif
-#if 0
-    mt("c:\\test\\x,y,z\\abc,e:\\test\\fgh,ijk,\"a, b\",eee\\fff,c:\\test\\eee\\ggg");
-    mt("/c$/test/x,y,z/abc,/e$/test/fgh,ijk,\"a, b\",eee/fff,/c$/test/eee/ggg");
-    mt("/c$/test/uvw/x,y,z/abc,/e$/test/fgh,ijk,\"a, b\",eee/fff,/c$/test/eee/ggg");
-#endif
-#if 0
-    testDF1();
-    testDF2();
-    testMisc();
-//  testGrp();
-#endif
-#ifndef COMPAT
-    testDFile();
-#endif
-}
-

dali/regress/sourcedoc.xml

@@ -1,26 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<!--
-################################################################################
-#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
-#
-#    Licensed under the Apache License, Version 2.0 (the "License");
-#    you may not use this file except in compliance with the License.
-#    You may obtain a copy of the License at
-#
-#       http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS,
-#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#    See the License for the specific language governing permissions and
-#    limitations under the License.
-################################################################################
--->
-<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
-<section>
-    <title>dali/regress</title>
-
-    <para>
-        The dali/regress directory contains the sources for the dali/regress library.
-    </para>
-</section>

roxie/ccd/CMakeLists.txt

@@ -73,6 +73,7 @@ include_directories (
          ./../../rtl/eclrtl 
          ./../../common/roxiehelper 
          ./../../rtl/include
+         ./../../testing/unittests
          ${CMAKE_BINARY_DIR}
          ${CMAKE_BINARY_DIR}/oss
     )

roxie/ccd/ccdkey.cpp

@@ -1963,8 +1963,7 @@ MODULE_EXIT()
 //=======================================================================================================
 
 #ifdef _USE_CPPUNIT
-#include <cppunit/extensions/HelperMacros.h>
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 class InMemoryIndexTest : public CppUnit::TestFixture  
 {

roxie/ccd/ccdserver.cpp

@@ -32300,14 +32300,13 @@ bool suspendRoxieListener(unsigned port, bool suspended)
 //================================================================================================================================
 
 #ifdef _USE_CPPUNIT
+#include "unittests.hpp"
+
 // There is a bug in VC6 implemetation of protected which prevents nested classes from accessing owner's data. It can be tricky to work around - hence...
 #if _MSC_VER==1200
 #undef protected
 #endif
 
-#include <cppunit/extensions/HelperMacros.h>
-
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
 static const char *sortAlgorithm;
 
 class TestMetaData : public CInterface, implements IOutputMetaData 

roxie/roxiemem/CMakeLists.txt

@@ -41,6 +41,7 @@ include_directories (
          ./../../rtl/include
          ./../../rtl/eclrtl
          ./../../common/thorhelper
+         ./../../testing/unittests
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DROXIEMEM_EXPORTS)

roxie/roxiemem/roxiemem.cpp

@@ -3269,8 +3269,7 @@ extern void setDataAlignmentSize(unsigned size)
 
 //============================================================================================================
 #ifdef _USE_CPPUNIT
-#include <cppunit/extensions/HelperMacros.h>
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 namespace roxiemem {
 

roxie/roxiemem/roxierow.cpp

@@ -495,8 +495,7 @@ IRowAllocatorMetaActIdCache *createRowAllocatorCache(IRowAllocatorMetaActIdCache
 
 
 #ifdef _USE_CPPUNIT
-#include <cppunit/extensions/HelperMacros.h>
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 namespace roxiemem {
 

roxie/roxiemem/roxierowbuff.cpp

@@ -219,8 +219,7 @@ void RoxieSimpleInputRowArray::transferFrom(RoxieOutputRowArray & donor)
 //============================================================================================================
 
 #ifdef _USE_CPPUNIT
-#include <cppunit/extensions/HelperMacros.h>
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 namespace roxiemem {
 

rtl/eclrtl/CMakeLists.txt

@@ -46,6 +46,7 @@ include_directories (
          ./../../system/include 
          ./../../common/workunit
          ./../../system/jlib 
+         ./../../testing/unittests
          ${ICU_INCLUDE_DIR}
          ${BOOST_REGEX_INCLUDE_DIR}
     )

rtl/eclrtl/eclrtl.cpp

@@ -5683,8 +5683,7 @@ void ensureRtlLoaded()
 }
 
 #ifdef _USE_CPPUNIT
-#include <cppunit/extensions/HelperMacros.h>
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 class EclRtlTests : public CppUnit::TestFixture
 {

system/jhtree/CMakeLists.txt

@@ -52,6 +52,7 @@ include_directories (
          ./../../rtl/eclrtl 
          ./../../system/include 
          ./../../system/jlib 
+         ./../../testing/unittests
     )
 
 ADD_DEFINITIONS( -D_USRDLL -DJHTREE_EXPORTS -DECLRTL_LOCAL )

system/jhtree/jhtree.cpp

@@ -2922,8 +2922,7 @@ extern jhtree_decl IKeyArray *createKeyArray()
 }
 
 #ifdef _USE_CPPUNIT
-#include <cppunit/extensions/HelperMacros.h>
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 class IKeyManagerTest : public CppUnit::TestFixture  
 {

testing/unittests/CMakeLists.txt

@@ -29,13 +29,16 @@ project( unitests )
 set (    SRCS
          unittests.cpp
          remotetests.cpp
+         dalitests.cpp
     )
 
 include_directories (
          .
          ./../../system/include
          ./../../system/jlib
+         ./../../system/mp
          ./../../common/remote
+         ./../../dali/base
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )
@@ -46,6 +49,7 @@ install ( TARGETS unittests DESTINATION ${OSSDIR}/bin )
 target_link_libraries ( unittests
          jlib
          remote
+         dalibase
          ${CPPUNIT_LIBRARIES}
     )
 

testing/unittests/remotetests.cpp

@@ -16,14 +16,7 @@
 ############################################################################## */
 
 #ifdef _USE_CPPUNIT
-#include "jlib.hpp"
-#include "jlog.hpp"
-
-#include <cppunit/extensions/TestFactoryRegistry.h>
-#include <cppunit/ui/text/TestRunner.h>
-#include <cppunit/extensions/HelperMacros.h>
-// CPPUNIT_ASSERT is too slow, even when not matching failure
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 #ifdef _USE_URIPARSER
 // =============================================================== URI parser
@@ -106,7 +99,7 @@ public:
 };
 
 CPPUNIT_TEST_SUITE_REGISTRATION( URITests );
-CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( URITests, "URITests" );
+CPPUNIT_TEST_SUITE_NAMED_REGISTRATION( URITests, "URI" );
 
 #endif // _USE_URIPARSER
 

testing/unittests/unittests.cpp

@@ -16,16 +16,7 @@
 ############################################################################## */
 
 #ifdef _USE_CPPUNIT
-#include "platform.h"
-#include "jlib.hpp"
-#include "jlog.hpp"
-#include "jmisc.hpp"
-
-#include <cppunit/extensions/TestFactoryRegistry.h>
-#include <cppunit/ui/text/TestRunner.h>
-#include <cppunit/extensions/HelperMacros.h>
-
-#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+#include "unittests.hpp"
 
 /*
  * This is the main unittest driver for HPCC. From here,
@@ -45,32 +36,6 @@
  * CPPUnit will automatically recognise and run them all.
  */
 
-/*
- * Helper class to unload libraries at the end
- * and make sure the SharedObject gets deleted
- * correctly.
- *
- * This is important to run valgrind tests and not
- * having to care about which memory leaks are "good"
- * and which are not.
- */
-class LoadedObject : public IInterface, CInterface {
-    SharedObject *so;
-public:
-    IMPLEMENT_IINTERFACE;
-
-    LoadedObject(const char * name)
-    {
-        so = new SharedObject;
-        so->load(name, true);
-    }
-    ~LoadedObject()
-    {
-        so->unload();
-        delete so;
-    }
-};
-
 int main(int argc, char* argv[])
 {
     InitModuleObjects();

testing/unittests/unittests.hpp

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "jlib.hpp"
+#include "jlog.hpp"
+#include "jmisc.hpp"
+
+#include <cppunit/extensions/TestFactoryRegistry.h>
+#include <cppunit/ui/text/TestRunner.h>
+#include <cppunit/extensions/HelperMacros.h>
+
+#define ASSERT(a) { if (!(a)) CPPUNIT_ASSERT(a); }
+
+/*
+ * Helper class to unload libraries at the end
+ * and make sure the SharedObject gets deleted
+ * correctly.
+ *
+ * This is important to run valgrind tests and not
+ * having to care about which memory leaks are "good"
+ * and which are not.
+ */
+class LoadedObject : public IInterface, CInterface {
+    SharedObject so;
+public:
+    IMPLEMENT_IINTERFACE;
+
+    LoadedObject(const char * name)
+    {
+        so.load(name, true);
+    }
+};