Merge branch 'candidate-6.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -897,7 +897,7 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
   endif ( PLATFORM OR PLUGIN )
   set (CMAKE_SKIP_BUILD_RPATH  FALSE)
   set (CMAKE_BUILD_WITH_INSTALL_RPATH FALSE)
-  set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_DIR}")
+  set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/${LIB_DIR};${CMAKE_INSTALL_PREFIX}/${PLUGINS_DIR}")
   set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
   if (APPLE)
     # used to locate libraries when compiling ECL

dali/base/dasds.cpp

@@ -2052,6 +2052,7 @@ public: // data
     Owned<IPropertyTree> properties;
 private:
     void validateBackup();
+    void validateDeltaBackup();
     LockStatus establishLock(CLock &lock, __int64 treeId, ConnectionId connectionId, SessionId sessionId, unsigned mode, unsigned timeout, IUnlockCallback &lockCallback);
     void _getChildren(CRemoteTreeBase &parent, CServerRemoteTree &serverParent, CRemoteConnection &connection, unsigned levels);
     void matchServerTree(CClientRemoteTree *local, IPropertyTree &matchTree, bool allTail);
@@ -2078,6 +2079,7 @@ private:
     bool doTimeComparison;
     StringBuffer blockedDelta;
     CBackupHandler backupHandler;
+    bool backupOutOfSync = false;
 };
 
 ISDSManagerServer &querySDSServer()
@@ -5889,6 +5891,22 @@ bool compareFiles(IFile *file1, IFile *file2, bool compareTimes=true)
     return false;
 }
 
+void CCovenSDSManager::validateDeltaBackup()
+{
+    // check consistency of delta
+    StringBuffer deltaFilename(dataPath);
+    iStoreHelper->getCurrentDeltaFilename(deltaFilename);
+    OwnedIFile iFileDelta = createIFile(deltaFilename.str());
+    deltaFilename.clear().append(remoteBackupLocation);
+    iStoreHelper->getCurrentDeltaFilename(deltaFilename);
+    OwnedIFile iFileDeltaBackup = createIFile(deltaFilename.str());
+    if (!compareFiles(iFileDeltaBackup, iFileDelta, false))
+    {
+        WARNLOG("Delta file backup doesn't exist or differs, filename=%s", deltaFilename.str());
+        copyFile(iFileDeltaBackup, iFileDelta);
+    }
+}
+
 void CCovenSDSManager::validateBackup()
 {
     // check consistency of store info file.
@@ -5903,14 +5921,7 @@ void CCovenSDSManager::validateBackup()
     }
 
     // check consistency of delta
-    StringBuffer deltaFilename(dataPath);
-    iStoreHelper->getCurrentDeltaFilename(deltaFilename);
-    OwnedIFile iFileDelta = createIFile(deltaFilename.str());
-    deltaFilename.clear().append(remoteBackupLocation);
-    iStoreHelper->getCurrentDeltaFilename(deltaFilename);
-    OwnedIFile iFileDeltaBackup = createIFile(deltaFilename.str());
-    if (!compareFiles(iFileDeltaBackup, iFileDelta, false))
-        WARNLOG("Delta file backup doesn't exist or differs, filename=%s", deltaFilename.str());
+    validateDeltaBackup();
 
     // ensure there's a copy of the primary store present at startup.
     StringBuffer storeFilename(dataPath);
@@ -5921,7 +5932,10 @@ void CCovenSDSManager::validateBackup()
     iStoreHelper->getCurrentStoreFilename(storeFilename);
     OwnedIFile iFileBackupStore = createIFile(storeFilename.str());
     if (!compareFiles(iFileBackupStore, iFileStore))
+    {
         WARNLOG("Store backup file doesn't exist or differs, filename=%s", storeFilename.str());
+        copyFile(iFileBackupStore, iFileStore);
+    }
 }
 
 static int uint64compare(unsigned __int64 const *i1, unsigned __int64 const *i2)
@@ -6529,29 +6543,50 @@ void CCovenSDSManager::saveDelta(const char *path, IPropertyTree &changeTree)
         }
         catch (IException *e) { EXCLOG(e, NULL); e->Release(); }
     }
+    bool first = false;
     try
     {
         StringBuffer deltaFilename(dataPath);
         iStoreHelper->getCurrentDeltaFilename(deltaFilename);
         toXML(header, blockedDelta);
         OwnedIFile iFile = createIFile(deltaFilename.str());
-        bool first = !iFile->exists() || 0 == iFile->size();
+        first = !iFile->exists() || 0 == iFile->size();
         writeDelta(blockedDelta, *iFile);
-        if (remoteBackupLocation.length())
-            backupHandler.addDelta(blockedDelta, iStoreHelper->queryCurrentEdition(), first);
-        else
-        {
-            if (blockedDelta.length() > 0x100000)
-                blockedDelta.kill();
-            else
-                blockedDelta.clear();
-        }
     }
     catch (IException *e)
     {
-        LOG(MCoperatorError, unknownJob, e, "saveDelta");
+        // NB: writeDelta retries a few times before giving up.
+        VStringBuffer errMsg("saveDelta: failed to save delta data, blockedDelta size=%d", blockedDelta.length());
+        LOG(MCoperatorError, unknownJob, e, errMsg.str());
         e->Release();
+        return;
+    }
+    if (remoteBackupLocation.length())
+    {
+        try
+        {
+            if (backupOutOfSync) // true if there was previously an exception during synchronously writing delta to backup.
+            {
+                LOG(MCoperatorError, unknownJob, "Backup delta is out of sync due to a prior backup write error, attempting to resync");
+                // catchup - check and copy primary delta to backup
+                validateDeltaBackup();
+                backupOutOfSync = false;
+                LOG(MCoperatorError, unknownJob, "Backup delta resynchronized");
+            }
+            else
+                backupHandler.addDelta(blockedDelta, iStoreHelper->queryCurrentEdition(), first);
+        }
+        catch (IException *e)
+        {
+            LOG(MCoperatorError, unknownJob, e, "saveDelta: failed to save backup delta data");
+            e->Release();
+            backupOutOfSync = true;
+        }
     }
+    if (blockedDelta.length() > 0x100000)
+        blockedDelta.kill();
+    else
+        blockedDelta.clear();
 }
 
 CSubscriberContainerList *CCovenSDSManager::getSubscribers(const char *xpath, CPTStack &stack)

ecl/eclcc/eclcc.hpp

@@ -124,6 +124,7 @@ const char * const helpText[] = {
     "?!  -fshowMetaInGraph       Add distribution/sort orders to the graph",
     "?!  -fshowRecordCountInGraph  Show estimates of record counts in the graph",
     "?!  -fspanMultipleCpp       Generate a work unit in multiple c++ files",
+    "?!  -fsubgraphToRegenerate=n Regenerate the ECL for a particular subgraph",
     "",
 };
 

ecl/hql/hqlthql.cpp

@@ -894,7 +894,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
     {
         if (expr->isDataset() || expr->isDictionary())
         {
-            if (!isNamedSymbol && expandProcessed && no != no_field && no != no_rows && !isTargetSelector(expr))
+            if (!isNamedSymbol && (expandProcessed || tryToRegenerate) && no != no_field && no != no_rows && !isTargetSelector(expr))
             {
                 if (!expr->queryTransformExtra())
                 {
@@ -1926,8 +1926,11 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         {
             s.append("DATASET(WORKUNIT(");
             toECL(child1, s, false, inType);
-            s.append(", ");
-            toECL(expr->queryChild(2), s, false, inType);
+            if (!isInternalAttribute(expr->queryChild(2)) || expandProcessed)
+            {
+                s.append(", ");
+                toECL(expr->queryChild(2), s, false, inType);
+            }
             if (expandProcessed)
                 childrenToECL(expr, s, false, true, 3);
             s.append("), ");
@@ -2549,7 +2552,7 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
             break;
         case no_getgraphresult:
         case no_setgraphresult:
-            if (expandProcessed)
+            if (expandProcessed || tryToRegenerate)
                 defaultToECL(expr, s, inType);
             else
             {

ecl/hqlcpp/hqlcpp.cpp

@@ -1571,7 +1571,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.optimizeBoolReturn,"optimizeBoolReturn", true),
         DebugOption(options.freezePersists,"freezePersists", false),
         DebugOption(options.maxRecordSize, "defaultMaxLengthRecord", MAX_RECORD_SIZE),
-        DebugOption(options.subgraphToRegeneate, "subgraphToRegeneate", 0),
+        DebugOption(options.subgraphToRegenerate, "subgraphToRegenerate", 0),
 
         DebugOption(options.checkRoxieRestrictions,"checkRoxieRestrictions", true),     // a debug aid for running regression suite
         DebugOption(options.checkThorRestrictions,"checkThorRestrictions", true),       // a debug aid for running regression suite

ecl/hqlcpp/hqlcpp.ipp

@@ -586,7 +586,7 @@ struct HqlCppOptions
     unsigned            applyInstantEclTransformationsLimit;
     unsigned            complexClassesThreshold;
     unsigned            complexClassesActivityFilter;
-    unsigned            subgraphToRegeneate;
+    unsigned            subgraphToRegenerate;
     unsigned            defaultPersistExpiry;
     unsigned            defaultExpiry;
     int                 defaultNumPersistInstances;

ecl/hqlcpp/hqlhtcpp.cpp

@@ -9243,7 +9243,7 @@ unsigned HqlCppTranslator::doBuildThorChildSubGraph(BuildCtx & ctx, IHqlExpressi
 
     OwnedHqlExpr idExpr = createConstant((__int64)thisId);
     ctx.associateExpr(expr, idExpr);
-    if (thisId == options.subgraphToRegeneate)
+    if (thisId == options.subgraphToRegenerate)
     {
         StringBuffer ecl;
         regenerateECL(expr, ecl);

tools/wutool/wutool.cpp

@@ -1465,7 +1465,7 @@ protected:
             numIterated++;
         }
         DBGLOG("%d workunits ascending thortime in %d ms", numIterated, msTick()-start);
-        ASSERT(numIterated == testSize);
+        ASSERT(numIterated == before);
 
         // Test use of cache/page mechanism - on something needing a postsort
         start = msTick();
@@ -1567,7 +1567,7 @@ protected:
             startRow++;
         }
         DBGLOG("%d workunits descending thortime, page by page in %d ms", numIterated, msTick()-start);
-        ASSERT(numIterated == testSize);
+        ASSERT(numIterated == before);
     }
 
     void testListByAppValue()
@@ -1657,6 +1657,7 @@ protected:
     void testSortByThorTime()
     {
         Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+        unsigned before = factory->numWorkUnits();
         unsigned start = msTick();
         unsigned numIterated = 0;
         // Test filter by filesRead
@@ -1672,7 +1673,7 @@ protected:
             numIterated++;
         }
         DBGLOG("%d workunits by totalThorTime in %d ms", numIterated, msTick()-start);
-        ASSERT(numIterated == testSize);
+        ASSERT(numIterated == before);
         numIterated++;
     }
     void testGlobal()