HPCC-9152 - Add validateStore option to daliadmin

Currently cross references GenratedDlls with WorkUnits

Signed-off-by: Jake Smith <jake.smith@lexisnexis.com>

common/dllserver/dllserver.cpp

@@ -273,7 +273,7 @@ protected:
 class DllEntry : public CInterface, implements IDllEntry
 {
 public:
-    DllEntry(IPropertyTree * root, const char *cacheRoot);
+    DllEntry(IPropertyTree *root, const char *cacheRoot, IPropertyTree *owner);
     IMPLEMENT_IINTERFACE
 
     virtual IIterator * createLocationIterator();
@@ -294,14 +294,15 @@ public:
     virtual void remove(bool removeFiles, bool removeDirectory);
 
 protected:
-    Owned<IPropertyTree> root;
+    Owned<IPropertyTree> root, owner;
     StringAttr cacheRoot;
 };
 
-DllEntry::DllEntry(IPropertyTree * _root, const char *_cacheRoot)
+DllEntry::DllEntry(IPropertyTree *_root, const char *_cacheRoot, IPropertyTree *_owner)
 : cacheRoot(_cacheRoot)
 {
     root.set(_root);
+    owner.set(_owner);
 }
 
 IIterator * DllEntry::createLocationIterator()
@@ -399,10 +400,15 @@ void DllEntry::remove(bool removeFiles, bool removeDirectory)
             isFirst = false;
         }
     }
-    StringBuffer path;
-    getPath(path, root->queryProp("@name"));
-    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
-    conn->close(true);
+    if (owner)
+        owner->removeTree(root);
+    else
+    {
+        StringBuffer path;
+        getPath(path, root->queryProp("@name"));
+        Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+        conn->close(true);
+    }
 }
 
 
@@ -420,14 +426,14 @@ public:
     {
         bool ok = TreeIteratorWrapper::first();
         if (ok)
-            cur.setown(new DllEntry(&iter->query(), cacheRoot));
+            cur.setown(new DllEntry(&iter->query(), cacheRoot, NULL));
         return ok;
     }
     virtual bool next()
     {
         bool ok = TreeIteratorWrapper::next();
         if (ok)
-            cur.setown(new DllEntry(&iter->query(), cacheRoot));
+            cur.setown(new DllEntry(&iter->query(), cacheRoot, NULL));
         return ok;
     }
 
@@ -455,6 +461,7 @@ public:
     virtual ILoadedDllEntry * loadDll(const char * name, DllLocationType location);
     virtual void removeDll(const char * name, bool removeDlls, bool removeDirectory);
     virtual void registerDll(const char * name, const char * kind, const char * dllPath);
+    virtual IDllEntry * createEntry(IPropertyTree *owner, IPropertyTree *entry);
 
 protected:
     void copyFileLocally(RemoteFilename & targetName, RemoteFilename & sourceName);
@@ -505,10 +512,14 @@ DllEntry * DllServer::doGetEntry(const char * name)
     getPath(path, name);
     Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), 0, 5000);
     if (conn)
-        return new DllEntry(conn->queryRoot(), rootDir);
+        return new DllEntry(conn->queryRoot(), rootDir, NULL);
     return NULL;
 }
 
+IDllEntry * DllServer::createEntry(IPropertyTree *owner, IPropertyTree *entry)
+{
+    return new DllEntry(entry, rootDir, owner);
+}
 
 void DllServer::doRegisterDll(const char * name, const char * kind, const char * dllPath, const char * libPath)
 {

common/dllserver/dllserver.hpp

@@ -83,6 +83,7 @@ interface IDllServer : extends IInterface
     virtual ILoadedDllEntry * loadDll(const char * name, DllLocationType location) = 0;
     virtual void removeDll(const char * name, bool removeDlls, bool removeDirectory) = 0;
     virtual void registerDll(const char * name, const char * kind, const char * dllPath) = 0;
+    virtual IDllEntry * createEntry(IPropertyTree *owner, IPropertyTree *entry) = 0;
 };
 
 extern DLLSERVER_API IDllServer & queryDllServer();

dali/daliadmin/CMakeLists.txt

@@ -38,6 +38,7 @@ include_directories (
          ./../../system/jlib 
          ./../ft 
          ./../../common/workunit
+         ./../../common/dllserver
     )
 
 ADD_DEFINITIONS( -D_CONSOLE )
@@ -50,5 +51,6 @@ target_link_libraries ( daliadmin
          remote 
          dalibase 
          workunit
+         dllserver
     )
 

dali/daliadmin/daliadmin.cpp

@@ -44,6 +44,7 @@
 #include "rmtfile.hpp"
 
 #include "workunit.hpp"
+#include "dllserver.hpp"
 
 #ifdef _WIN32
 #include <conio.h>
@@ -112,9 +113,12 @@ void usage(const char *exe)
   printf("  daliping [ <num> ]              -- time dali server connect\n");
   printf("  getxref <destxmlfile>           -- get all XREF information\n");
   printf("  dalilocks [ <ip-pattern> ] [ files ] -- get all locked files/xpaths\n");
-  printf("  unlock <xpath or logicalfile>   --  unlocks either matching xpath(s) or matching logical file(s), can contain wildcards\n");
-  printf("  wuidcompress <wildcard> <type>  --  scan workunits that match <wildcard> and compress resources of <type>\n");
-  printf("  wuiddecompress <wildcard> <type> --  scan workunits that match <wildcard> and decompress resources of <type>\n");
+  printf("  unlock <xpath or logicalfile>   -- unlocks either matching xpath(s) or matching logical file(s), can contain wildcards\n");
+  printf("  validatestore [fix=<true|false>]\n"
+         "                [verbose=<true|false>]\n"
+         "                [deletefiles=<true|false>]-- perform some checks on dali meta data an optionally fix or remove redundant info \n");
+  printf("  wuidcompress <wildcard> <type>  -- scan workunits that match <wildcard> and compress resources of <type>\n");
+  printf("  wuiddecompress <wildcard> <type> -- scan workunits that match <wildcard> and decompress resources of <type>\n");
   printf("\n");
   printf("Common options\n");
   printf("  server=<dali-server-ip>         -- server ip\n");
@@ -2110,6 +2114,111 @@ static void wuidCompress(const char *match, const char *type, bool compress)
     }
 }
 
+static void validateStore(bool fix, bool deleteFiles, bool verbose)
+{
+    /*
+     * Place holder for client-side dali store verification/validation. Currently performs:
+     * 1) validates GeneratedDll entries correspond to current workunits (see HPCC-9146)
+     */
+    CTimeMon totalTime, ts;
+
+    PROGLOG("Gathering list of workunits");
+    Owned<IRemoteConnection> conn = querySDS().connect("/WorkUnits", myProcessSession(), RTM_LOCK_READ, 10000);
+    if (!conn)
+        throw MakeStringException(0, "Failed to connect to /WorkUnits");
+    AtomRefTable wuids;
+    Owned<IPropertyTreeIterator> wuidIter = conn->queryRoot()->getElements("*");
+    ForEach(*wuidIter)
+    {
+        IPropertyTree &wuid = wuidIter->query();
+        wuids.queryCreate(wuid.queryName());
+    }
+    PROGLOG("%d workunits gathered. Took %d ms", wuids.count(), ts.elapsed());
+    ts.reset(0);
+
+    StringArray uidsToDelete;
+    UnsignedArray indexToDelete;
+
+    PROGLOG("Gathering associated files");
+    conn.setown(querySDS().connect("/GeneratedDlls", myProcessSession(), fix?RTM_LOCK_WRITE:RTM_LOCK_READ, 10000));
+    IPropertyTree *root = conn->queryRoot()->queryBranch(NULL); // force all to download
+
+    Owned<IPropertyTreeIterator> gdIter = root->getElements("*");
+    RegExpr RE("^.*{W2[0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]{-[0-9]+}?}{[^0-9].*|}$");
+
+    unsigned index=1;
+    ForEach(*gdIter)
+    {
+        IPropertyTree &gd = gdIter->query();
+        const char *name = gd.queryProp("@name");
+        if (name && *name)
+        {
+            if (RE.find(name))
+            {
+                StringBuffer wuid;
+                RE.substitute(wuid,"#1");
+                const char *w = wuid.str();
+                bool found = NULL != wuids.find(*w);
+                const char *uid = gd.queryProp("@uid");
+                if (!found)
+                {
+                    uidsToDelete.append(uid);
+                    indexToDelete.append(index);
+                }
+            }
+        }
+        ++index;
+    }
+    PROGLOG("%d out of %d workunit files not associated with any workunit. Took %d ms", indexToDelete.ordinality(), index, ts.elapsed());
+    ts.reset(0);
+
+    IArrayOf<IDllEntry> removedEntries;
+    unsigned numDeleted = 0;
+    ForEachItemInRev(d, indexToDelete)
+    {
+        const char *uid = uidsToDelete.item(d);
+        unsigned index = indexToDelete.item(d);
+        StringBuffer path("GeneratedDll[");
+        path.append(index).append("]");
+        IPropertyTree *gd = root->queryPropTree(path.str());
+        if (NULL == gd)
+            throwUnexpected();
+        const char *uidQuery = gd->queryProp("@uid");
+        if (0 != strcmp(uid, uidQuery))
+            throw MakeStringException(0, "Expecting uid=%s @ GeneratedDll[%d], but found uid=%s", uid, index, uidQuery);
+        if (verbose)
+            PROGLOG("Removing: %s, uid=%s", path.str(), uid);
+        if (fix)
+        {
+            Owned<IDllEntry> entry = queryDllServer().createEntry(root, gd);
+            entry->remove(false, false); // NB: This will remove child 'gd' element from root (GeneratedDlls)
+            if (deleteFiles) // delay until after meta info removed and /GeneratedDlls unlocked
+                removedEntries.append(*entry.getClear());
+        }
+        ++numDeleted;
+    }
+    if (fix)
+    {
+        conn->commit();
+        PROGLOG("Removed %d unassociated file entries. Took %d ms", numDeleted, ts.elapsed());
+        ts.reset(0);
+
+        if (deleteFiles)
+        {
+            PROGLOG("Deleting physical files..");
+            ForEachItemIn(r, removedEntries)
+            {
+                IDllEntry &entry = removedEntries.item(r);
+                PROGLOG("Removing files for: %s", entry.queryName());
+                entry.remove(true, false);
+            }
+            PROGLOG("Removed physical files. Took %d ms", ts.elapsed());
+        }
+    }
+    else
+        PROGLOG("%d unassociated file entries to remove - use 'fix=true'", numDeleted);
+    PROGLOG("Done time = %d secs", totalTime.elapsed()/1000);
+}
 
 //=============================================================================
 
@@ -2180,6 +2289,8 @@ int main(int argc, char* argv[])
             (memcmp(param,"user=",5)==0)||
             (memcmp(param,"password=",9)==0) ||
             (memcmp(param,"fix=",4)==0) ||
+            (memcmp(param,"verbose=",8)==0) ||
+            (memcmp(param,"deletefiles=",12)==0) ||
             (memcmp(param,"timeout=",4)==0))
             props->loadProp(param);
         else if ((i==1)&&(isdigit(*param)||(*param=='.'))&&ep.set(((*param=='.')&&param[1])?(param+1):param,DALI_SERVER_PORT))
@@ -2349,7 +2460,6 @@ int main(int argc, char* argv[])
             else if (stricmp(cmd,"listprotect")==0) {
                 CHECKPARAMS(0,2);
                 listprotect((np>1)?params.item(1):"*",(np>2)?params.item(2):"*");
-
             }
             else if (stricmp(cmd,"checksuperfile")==0) {
                 CHECKPARAMS(1,1);
@@ -2447,7 +2557,14 @@ int main(int argc, char* argv[])
                 CHECKPARAMS(2,2);
                 wuidCompress(params.item(1), params.item(2), false);
             }
-            else 
+            else if (stricmp(cmd,"validateStore")==0) {
+                CHECKPARAMS(0,2);
+                bool fix = props->getPropBool("fix");
+                bool verbose = props->getPropBool("verbose");
+                bool deleteFiles = props->getPropBool("deletefiles");
+                validateStore(fix, deleteFiles, verbose);
+             }
+             else
                 ERRLOG("Unknown command %s",cmd);
         }
         catch (IException *e) {

roxie/ccd/ccddali.cpp

@@ -617,6 +617,7 @@ public:
     virtual DllLocationType isAvailable(const char * name) { throwUnexpected(); }
     virtual void removeDll(const char * name, bool removeDlls, bool removeDirectory) { throwUnexpected(); }
     virtual void registerDll(const char * name, const char * kind, const char * dllPath) { throwUnexpected(); }
+    virtual IDllEntry * createEntry(IPropertyTree *owner, IPropertyTree *entry) { throwUnexpected(); }
 
     virtual ILoadedDllEntry * loadDll(const char * name, DllLocationType location) 
     {