Merge branch 'candidate-7.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/commonSetup.cmake

@@ -488,11 +488,27 @@ IF ("${COMMONSETUP_DONE}" STREQUAL "")
     endif ()
   endif ()
 
+  # check for old glibc and add required libs ...
+  if (NOT APPLE AND NOT WIN32)
+    execute_process(
+        COMMAND /bin/bash -c "ldd --version | head -n 1 | awk '{print $NF}'"
+        WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}
+        OUTPUT_VARIABLE GLIBC_VER
+        OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    message(STATUS "GLIBC version: ${GLIBC_VER}")
+    if ("${GLIBC_VER}" VERSION_LESS "2.18")
+        set(CMAKE_C_STANDARD_LIBRARIES "${CMAKE_C_STANDARD_LIBRARIES} -lrt")
+        set(CMAKE_CXX_STANDARD_LIBRARIES "${CMAKE_CXX_STANDARD_LIBRARIES} -lrt")
+    endif()
+  endif()
+
   if (CMAKE_COMPILER_IS_GNUCXX)
     #Ensure that missing symbols are reported as errors at link time (default for osx/windows)
     SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,-z,defs")
     SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,defs")
   endif()
+
   macro(HPCC_ADD_EXECUTABLE target)
     add_executable(${target} ${ARGN})
   endmacro(HPCC_ADD_EXECUTABLE target)

common/workunit/workunit.cpp

@@ -11277,7 +11277,7 @@ StatisticKind CLocalWUStatistic::getKind() const
 const char * CLocalWUStatistic::queryScope() const
 {
     const char * scope = p->queryProp("@scope");
-    if (scope && streq(scope, LEGACY_GLOBAL_SCOPE))
+    if (!scope || streq(scope, LEGACY_GLOBAL_SCOPE))
         scope = GLOBAL_SCOPE;
     return scope;
 }

dali/dfuplus/dfuplus.cpp

@@ -895,6 +895,9 @@ int CDfuPlusHelper::copy()
     else if(globals->hasProp("noRecover"))
         req->setNorecover(globals->getPropBool("noRecover", false));
 
+    if(globals->hasProp("expireDays"))
+        req->setExpireDays(globals->getPropInt("expireDays"));
+
     Owned<IClientCopyResponse> result = sprayclient->Copy(req);
     const char* wuid = result->getResult();
     if(wuid == nullptr || *wuid == '\0')

dali/ft/filecopy.cpp

@@ -3355,11 +3355,9 @@ void FileSprayer::updateTargetProperties()
             curHistory->addPropTree("Origin",newRecord.getClear());
         }
 
-        int expireDays = options->getPropInt("@expireDays");
+        int expireDays = options->getPropInt("@expireDays", -1);
         if (expireDays != -1)
-        {
             curProps.setPropInt("@expireDays", expireDays);
-        }
     }
     if (error)
         throw error.getClear();

docs/EN_US/ECLLanguageReference/ECLR_mods/Recrd-DATASET.xml

@@ -1137,7 +1137,7 @@ personRecord := RECORD
   UNSIGNED2 numChildren;
   DATASET(childPersonRecord) children;
 END;
-123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890
+
 personDataset := DATASET([{'Kevin','Hall',2,[{'Abby',2},{'Nat',2}]},
                           {'Jon','Simms',3,[{'Jen',18},{'Ali',16},{'Andy',13}]}],
                          personRecord);

ecl/eclcmd/eclcmd_common.cpp

@@ -453,7 +453,7 @@ public:
                 IEspNamedValue &item = cmd.debugValues.item(i);
                 const char *name = item.getName();
                 const char *value = item.getValue();
-                cmdLine.append(" '");
+                cmdLine.append(" \"");
 
                 if (name && 0==strncmp(name, "eclcc", 5))
                     name+=5;
@@ -462,7 +462,7 @@ public:
                 cmdLine.append(name);
                 if (value)
                     cmdLine.append('=').append(value);
-                cmdLine.append("'");
+                cmdLine.append("\"");
             }
         }
         if (cmd.definitions.length())

ecl/hqlcpp/hqlcpp.cpp

@@ -1860,6 +1860,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.generateDiskFormats, "generateDiskFormats", false),
         DebugOption(options.maxOptimizeSize, "maxOptimizeSize", 5),             // Remove the overhead from very small functions e.g. function prolog
         DebugOption(options.minNoOptimizeSize, "minNoOptimizeSize", 10000),     // functions larger than this will take a long time to optimize, better to not try
+        DebugOption(options.timeRegex, "timeRegex", false),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -831,6 +831,7 @@ struct HqlCppOptions
     bool                genericDiskReads;
     bool                generateActivityFormats;
     bool                generateDiskFormats;
+    bool                timeRegex;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlhtcpp.cpp

@@ -18337,6 +18337,25 @@ void HqlCppTranslator::doBuildNewRegexFindReplace(BuildCtx & ctx, const CHqlBoun
     // Because the search instance is created locally, the search parameter is always going to be valid
     // as long as the find instance.  Only exception could be if call created a temporary class instance.
     bool cloneSearch = false;
+    CHqlBoundExpr boundTimer, boundStart;
+    if (options.timeRegex)
+    {
+        StringBuffer regexName;
+        regexName.append(getOpString(expr->getOperator()));
+
+        if (pattern->queryId())
+            regexName.append("(").append(str(pattern->queryId())).append(")");
+        else if (pattern->isConstant())
+        {
+            StringBuffer patternText;
+            getUTF8Value(patternText, pattern);
+            // Replace colons in the pattern to avoid issues with scopes, and single quotes to avoid xpath quoting issues
+            patternText.replace(':', '_').replace('\'', '_');
+            regexName.append("(").append(patternText).append(")");
+        }
+
+        buildStartTimer(ctx, boundTimer, boundStart, regexName);
+    }
     IHqlExpression * findInstance = doBuildRegexFindInstance(ctx, compiled, search, cloneSearch);
     if(expr->queryType() == queryBoolType())
     {
@@ -18355,6 +18374,8 @@ void HqlCppTranslator::doBuildNewRegexFindReplace(BuildCtx & ctx, const CHqlBoun
         OwnedHqlExpr call = bindFunctionCall(func, args);
         buildExprOrAssign(ctx, target, call, bound);
     }
+    if (options.timeRegex)
+        buildStopTimer(ctx, boundTimer, boundStart);
 }
 
 void HqlCppTranslator::doBuildExprRegexFindReplace(BuildCtx & ctx, IHqlExpression * expr, CHqlBoundExpr & bound)

esp/services/ws_fs/ws_fsService.cpp

@@ -2000,7 +2000,8 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
         if (req.getRecordStructurePresent())
             options->setRecordStructurePresent(true);
 
-        options->setExpireDays(req.getExpireDays());
+        if (!req.getExpireDays_isNull())
+            options->setExpireDays(req.getExpireDays());
 
         resp.setWuid(wu->queryId());
         resp.setRedirectUrl(StringBuffer("/FileSpray/GetDFUWorkunit?wuid=").append(wu->queryId()).str());
@@ -2185,7 +2186,8 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
         if (req.getRecordStructurePresent())
             options->setRecordStructurePresent(true);
 
-        options->setExpireDays(req.getExpireDays());
+        if (!req.getExpireDays_isNull())
+            options->setExpireDays(req.getExpireDays());
 
         resp.setWuid(wu->queryId());
         resp.setRedirectUrl(StringBuffer("/FileSpray/GetDFUWorkunit?wuid=").append(wu->queryId()).str());

roxie/ccd/ccddali.cpp

@@ -606,6 +606,8 @@ public:
         if (!w)
             return NULL;
         w->setAgentSession(myProcessSession());
+        if(topology->getPropBool("@resetWorkflow", false))
+            w->resetWorkflow();
         if (source)
         {
             StringBuffer wuXML;

roxie/ccd/ccdmain.cpp

@@ -638,6 +638,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         useOldTopology = checkFileExists(topologyFile.str());
         topology = loadConfiguration(useOldTopology ? nullptr : defaultYaml, argv, "roxie", "ROXIE", topologyFile, nullptr, "@netAddress");
         saveTopology();
+        localSlave = topology->getPropBool("@localSlave", false);
         const char *channels = topology->queryProp("@channels");
         if (channels)
         {
@@ -658,6 +659,10 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
                 slaveChannels.push_back(std::pair<unsigned, unsigned>(channel, repl));
             }
         }
+#ifdef _CONTAINERIZED
+        else if (localSlave)
+            slaveChannels.push_back(std::pair<unsigned, unsigned>(1, 0));
+#endif
         const char *topos = topology->queryProp("@topologyServers");
         StringArray topoValues;
         if (topos)
@@ -851,9 +856,7 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
             Owned<IPropertyTree> nas = envGetNASConfiguration(topology);
             envInstallNASHooks(nas);
         }
-        useDynamicServers = topology->getPropBool("@useDynamicServers", !useOldTopology);
         useAeron = topology->getPropBool("@useAeron", false);
-        localSlave = topology->getPropBool("@localSlave", false);
         numChannels = topology->getPropInt("@numChannels", 0);
         doIbytiDelay = topology->getPropBool("@doIbytiDelay", true);
         minIbytiDelay = topology->getPropInt("@minIbytiDelay", 2);
@@ -1163,15 +1166,15 @@ int CCD_API roxie_main(int argc, const char *argv[], const char * defaultYaml)
         setDaliServixSocketCaching(true);  // enable daliservix caching
         enableForceRemoteReads(); // forces file reads to be remote reads if they match environment setting 'forceRemotePattern' pattern.
 
-        loadPlugins();
+        if (!standAloneDll && !wuid)
+            loadPlugins();
         createDelayedReleaser();
         globalPackageSetManager = createRoxiePackageSetManager(standAloneDll.getClear());
         globalPackageSetManager->load();
         unsigned snifferChannel = numChannels+2; // MORE - why +2 not +1??
-        if (useDynamicServers && topoValues.length())
-        {
-            startTopoThread(topoValues, myRoles, traceLevel);
-        }
+#ifdef _CONTAINERIZED
+        initializeTopology(topoValues, myRoles, traceLevel);
+#endif
         ROQ = createOutputQueueManager(snifferChannel, numSlaveThreads);
         ROQ->setHeadRegionSize(headRegionSize);
         ROQ->start();

roxie/udplib/udplib.hpp

@@ -38,8 +38,6 @@ typedef unsigned ruid_t;   // at 1000/sec recycle every 49 days
 typedef unsigned RecordLengthType;
 #define MAX_RECORD_LENGTH 0xffffffff
 
-extern UDPLIB_API bool useDynamicServers;
-
 class UDPLIB_API ServerIdentifier
 {
 private:

roxie/udplib/udpsha.cpp

@@ -40,7 +40,6 @@ unsigned udpLocalWriteSocketSize = 1024000;
 unsigned udpSnifferReadThreadPriority = 3;
 unsigned udpSnifferSendThreadPriority = 3;
 
-bool useDynamicServers = true;
 unsigned multicastTTL = 1;
 
 MODULE_INIT(INIT_PRIORITY_STANDARD)

roxie/udplib/udptopo.cpp

@@ -356,30 +356,33 @@ static std::thread topoThread;
 static Semaphore abortTopo;
 const unsigned topoUpdateInterval = 5000;
 
-extern UDPLIB_API void startTopoThread(const StringArray &topoValues, const std::vector<RoxieEndpointInfo> &myRoles, unsigned traceLevel)
+extern UDPLIB_API void initializeTopology(const StringArray &topoValues, const std::vector<RoxieEndpointInfo> &myRoles, unsigned traceLevel)
 {
     topologyManager.setServers(topoValues);
     topologyManager.setRoles(myRoles);
-    topoThread = std::thread([traceLevel]()
+    if (topoValues.length())
     {
-        topologyManager.update();
-        unsigned waitTime = 1000;  // First time around we don't wait as long, so that system comes up faster
-        while (!abortTopo.wait(waitTime))
+        topoThread = std::thread([traceLevel]()
         {
-            if (topologyManager.update() && traceLevel)
+            topologyManager.update();
+            unsigned waitTime = 1000;  // First time around we don't wait as long, so that system comes up faster
+            while (!abortTopo.wait(waitTime))
             {
-                DBGLOG("Topology information updated:");
-                Owned<const ITopologyServer> c = getTopology();
-                const SocketEndpointArray &eps = c->querySlaves(0);
-                ForEachItemIn(idx, eps)
+                if (topologyManager.update() && traceLevel)
                 {
-                    StringBuffer s;
-                    DBGLOG("Slave %d: %s", idx, eps.item(idx).getIpText(s).str());
+                    DBGLOG("Topology information updated:");
+                    Owned<const ITopologyServer> c = getTopology();
+                    const SocketEndpointArray &eps = c->querySlaves(0);
+                    ForEachItemIn(idx, eps)
+                    {
+                        StringBuffer s;
+                        DBGLOG("Slave %d: %s", idx, eps.item(idx).getIpText(s).str());
+                    }
                 }
+                waitTime = topoUpdateInterval;
             }
-            waitTime = topoUpdateInterval;
-        }
-    });
+        });
+    }
 }
 
 extern UDPLIB_API void stopTopoThread()

roxie/udplib/udptopo.hpp

@@ -115,7 +115,7 @@ struct RoxieEndpointInfo
     unsigned replicationLevel;
 };
 
-extern UDPLIB_API void startTopoThread(const StringArray &topoServers, const std::vector<RoxieEndpointInfo> &myRoles, unsigned traceLevel);
+extern UDPLIB_API void initializeTopology(const StringArray &topoServers, const std::vector<RoxieEndpointInfo> &myRoles, unsigned traceLevel);
 #ifndef _CONTAINERIZED
 extern UDPLIB_API void createStaticTopology(const std::vector<RoxieEndpointInfo> &allRoles, unsigned traceLevel);
 #endif

roxie/udplib/uttest.cpp

@@ -617,7 +617,6 @@ int main(int argc, char * argv[] )
     InitModuleObjects();
     if (argc < 2)
         usage();
-    useDynamicServers = false;
 
     strdup("Make sure leak checking is working");
     queryStderrLogMsgHandler()->setMessageFields(MSGFIELD_time | MSGFIELD_thread | MSGFIELD_prefix);

system/CMakeLists.txt

@@ -29,11 +29,8 @@ if (NOT JLIB_ONLY)
    HPCC_ADD_SUBDIRECTORY (xmllibtest "PLATFORM")
    
    if (USE_AERON)
-       project (aeron_include)
-         if (NOT APPLE)
-           set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lrt")
-         endif()
-         remove_definitions(-fvisibility=hidden)
-         HPCC_ADD_SUBDIRECTORY (aeron)
+      project (aeron_include)
+      remove_definitions(-fvisibility=hidden)
+      HPCC_ADD_SUBDIRECTORY (aeron)
    endif()
 endif( )

testing/regress/ecl/aaalibrary2.ecl

@@ -37,7 +37,7 @@ end;
 
 filterDatasetLibrary(dataset(namesRecord) ds, dataset(namesRecord) unused, string search, boolean onlyOldies) := module,library(FilterDatasetInterface)
     f := ds;
-    shared g := if (onlyOldies, f(age >= 65), f);
+    shared g := if (onlyOldies, f(age >= 65, not regexfind('boris', forename)), f);
     export matches := g(surname = search);
     export others := g(surname != search);
 end;

testing/regress/ecl/getenv.ecl

@@ -18,6 +18,6 @@
 //nothor
 //nohthor
 
-// hthor and thor should probably suppoty getEnv too but not immediately clear how to test them
+// hthor and thor should probably support getEnv too but not immediately clear how to test them
 
-output(if(getenv('@description', 'DefaultVal')='DefaultVal', 'Failed', 'Passed'));
+output(if(getenv('@linuxOS', 'DefaultVal')='DefaultVal', 'Failed', 'Passed'));

testing/regress/ecl/key/spray_expire_test.xml

@@ -23,6 +23,15 @@
 <Dataset name='DFUPlus'>
  <Row><DFUPlus>Pass</DFUPlus></Row>
 </Dataset>
+<Dataset name='DFUPlus2'>
+ <Row><DFUPlus2>Pass</DFUPlus2></Row>
+</Dataset>
+<Dataset name='DFUPlus3'>
+ <Row><DFUPlus3>Pass</DFUPlus3></Row>
+</Dataset>
+<Dataset name='DFUPlus4'>
+ <Row><DFUPlus4>Pass</DFUPlus4></Row>
+</Dataset>
 <Dataset name='Copy'>
  <Row><Copy>Pass</Copy></Row>
 </Dataset>

testing/regress/ecl/spray_expire_test.ecl

@@ -25,7 +25,7 @@ import Std.File AS FileServices;
 import $.setup;
 prefix := setup.Files(false, false).QueryFilePrefix;
 
-dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+dropzonePath := FileServices.GetDefaultDropZone() : STORED('dropzonePath');
 
 unsigned VERBOSE := 0;
 
@@ -106,7 +106,7 @@ c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
 
 ClusterName := 'mythor';
 DestFile := prefix + 'spray_expire.txt';
-ESPportIP := 'http://127.0.0.1:8010/FileSpray';
+ESPportIP := FileServices.GetEspURL() + '/FileSpray';
 
 expireDaysOut1 := 9;
 expireDaysIn1 := NOTHOR(FileServices.GetLogicalFileAttribute(DestFile, 'expireDays'));
@@ -124,19 +124,36 @@ expireDaysOut4 := 13;
 expireDaysIn4 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
 res4:=if(expireDaysIn4 = intformat(expireDaysOut4,2,0), 'Pass', 'Fail ('+intformat(expireDaysOut4,2,0)+','+expireDaysIn4+')');
 
+
+// DFUPlus tests
+
 expireDaysOut5 := 17;
 expireDaysIn5 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
 res5:=if(expireDaysIn5 = intformat(expireDaysOut5,2,0), 'Pass', 'Fail ('+intformat(expireDaysOut5,2,0)+','+expireDaysIn5+')');
 
+expireDaysOut6 := 0;
+expireDaysIn6 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
+res6:=if(expireDaysIn6 = intformat(expireDaysOut6,1,0), 'Pass', 'Fail ('+intformat(expireDaysOut6,2,0)+','+expireDaysIn6+')');
+
+expireDaysOut7 := -1;
+expireDaysIn7 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
+res7:=if(expireDaysIn7 = '', 'Pass', 'Fail ('+intformat(expireDaysOut7,2,0)+','+expireDaysIn7+')');
+
+expireDaysOut8 := -1; // but omit expireDay parameter
+expireDaysIn8 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
+res8:=if(expireDaysIn8 = '', 'Pass', 'Fail ('+intformat(expireDaysOut8,2,0)+','+expireDaysIn8+')');
+
+
+
 CopyDestFile := prefix + 'copy_expire.txt';
-expireDaysOut6 := 19;
-expireDaysIn6 := FileServices.GetLogicalFileAttribute(CopyDestFile, 'expireDays');
-res6:=if(expireDaysIn6 = intformat(expireDaysOut6,2,0), 'Pass', 'Fail ('+intformat(expireDaysOut6,2,0)+','+expireDaysIn6+')');
+expireDaysOut9 := 19;
+expireDaysIn9 := FileServices.GetLogicalFileAttribute(CopyDestFile, 'expireDays');
+res9:=if(expireDaysIn9 = intformat(expireDaysOut9,2,0), 'Pass', 'Fail ('+intformat(expireDaysOut9,2,0)+','+expireDaysIn9+')');
 
 RemotePullDestFile := prefix + 'remote_pull_expire.txt';
-expireDaysOut7 := 23;
-expireDaysIn7 := FileServices.GetLogicalFileAttribute(RemotePullDestFile, 'expireDays');
-res7:=if(expireDaysIn7 = intformat(expireDaysOut7,2,0), 'Pass', 'Fail ('+intformat(expireDaysOut7,2,0)+','+expireDaysIn7+')');
+expireDaysOut10 := 23;
+expireDaysIn10 := FileServices.GetLogicalFileAttribute(RemotePullDestFile, 'expireDays');
+res10:=if(expireDaysIn10 = intformat(expireDaysOut10,2,0), 'Pass', 'Fail ('+intformat(expireDaysOut10,2,0)+','+expireDaysIn10+')');
 
 
 sequential (
@@ -218,9 +235,20 @@ sequential (
                             expireDays :=expireDaysOut4
                             );
     output(res4,NAMED('SprayXml')),
+
     FileServices.DfuPlusExec('action=spray srcip=. srcfile='+desprayOutFileName+'_CSV dstname='+DestFile+' jobname=spray_expire_csv server=. dstcluster=mythor format=csv overwrite=1 replicate=0 expireDays='+intformat(expireDaysOut5,2,0)),
     output(res5,NAMED('DFUPlus')),
 
+    FileServices.DfuPlusExec('action=spray srcip=. srcfile='+desprayOutFileName+'_CSV dstname='+DestFile+' jobname=spray_expire_csv server=. dstcluster=mythor format=csv overwrite=1 replicate=0 expireDays='+intformat(expireDaysOut6,2,0)),
+    output(res6,NAMED('DFUPlus2')),
+
+    FileServices.DfuPlusExec('action=spray srcip=. srcfile='+desprayOutFileName+'_CSV dstname='+DestFile+' jobname=spray_expire_csv server=. dstcluster=mythor format=csv overwrite=1 replicate=0 expireDays='+intformat(expireDaysOut7,2,0)),
+    output(res7,NAMED('DFUPlus3')),
+
+    FileServices.DfuPlusExec('action=spray srcip=. srcfile='+desprayOutFileName+'_CSV dstname='+DestFile+' jobname=spray_expire_csv server=. dstcluster=mythor format=csv overwrite=1 replicate=0'),
+    output(res8,NAMED('DFUPlus4')),
+
+
     FileServices.Copy(
                             sourceLogicalName := sprayPrepFileName+'_CSV',
                             destinationGroup := ClusterName,
@@ -229,9 +257,9 @@ sequential (
                             timeOut := -1,
                             espServerIpPort := ESPportIP,
                             ALLOWOVERWRITE := true,
-                            expireDays := expireDaysOut6
+                            expireDays := expireDaysOut9
                             );
-    output(res6,NAMED('Copy')),
+    output(res9,NAMED('Copy')),
 
     FileServices.RemotePull(
                              remoteEspFsURL := ESPportIP,
@@ -240,9 +268,9 @@ sequential (
                              destinationLogicalName := RemotePullDestFile,
                              TIMEOUT := -1,
                              ALLOWOVERWRITE := true,
-                             expireDays := expireDaysOut7
+                             expireDays := expireDaysOut10
                              );
-     output(res7,NAMED('RemotePull')),
+     output(res10, NAMED('RemotePull')),
 
     // Clean-up
     FileServices.DeleteExternalFile('.', desprayOutFileName+'_CSV'),