Merge remote-tracking branch 'origin/candidate-8.0.x'

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

# Conflicts:
#	version.cmake

cmake_modules/buildBOOST_REGEX.cmake

@@ -2,9 +2,9 @@
 include(ExternalProject)
 ExternalProject_Add(
   generate-boost-regex
-  URL https://dl.bintray.com/boostorg/release/1.71.0/source/boost_1_71_0.tar.gz
+  URL https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz
   URL_HASH SHA256=96b34f7468f26a141f6020efb813f1a2f3dfb9797ecf76a7d7cbd843cc95f5bd
-  TIMEOUT 15
+  TIMEOUT 60
   DOWNLOAD_DIR ${CMAKE_BINARY_DIR}/downloads
   SOURCE_DIR ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0
   CONFIGURE_COMMAND ${CMAKE_BINARY_DIR}/downloads/boost_1_71_0/bootstrap.sh

common/remote/rmtspawn.cpp

@@ -277,7 +277,7 @@ CRemoteParentInfo::CRemoteParentInfo()
 }
 
 
-bool CRemoteParentInfo::processCommandLine(int argc, char * argv[], StringBuffer &logdir)
+bool CRemoteParentInfo::processCommandLine(int argc, const char * * argv, StringBuffer &logdir)
 {
     if (argc <= 4)
         return false;
@@ -406,12 +406,13 @@ CRemoteSlave::CRemoteSlave(const char * _name, unsigned _tag, unsigned _version,
     version = _version;
 }
 
-void CRemoteSlave::run(int argc, char * argv[])
+void CRemoteSlave::run(int argc, const char * * argv)
 {
     StringBuffer logFile;
     CRemoteParentInfo info;
 
     bool paramsok = info.processCommandLine(argc, argv, logFile);
+#ifndef _CONTAINERIZED
     if (logFile.length()==0) { // not expected! Caller queries logfile location via getConfigurationDirectory
 #ifdef _WIN32
         logFile.append("c:\\HPCCSystems\\logs\\ftslave");
@@ -427,7 +428,9 @@ void CRemoteSlave::run(int argc, char * argv[])
     attachStandardFileLogMsgMonitor(logFile.str(), 0, MSGFIELD_STANDARD, MSGAUD_all, MSGCLS_all, TopDetail, false, true, true);
     queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler());        // no point logging output to screen if run remote!
     LOG(MCdebugProgress, unknownJob, "Starting %s %s %s %s %s %s %s",slaveName.get(),(argc>1)?argv[1]:"",(argc>2)?argv[2]:"",(argc>3)?argv[3]:"",(argc>4)?argv[4]:"",(argc>5)?argv[5]:"",(argc>6)?argv[6]:"");
-
+#else
+    setupContainerizedLogMsgHandler();
+#endif
 
     if (paramsok)
     {

common/remote/rmtspawn.hpp

@@ -60,7 +60,7 @@ public:
     CRemoteParentInfo();
 
     ISocket * queryMasterSocket()               { return socket; }
-    bool processCommandLine(int argc, char * argv[], StringBuffer &logdir);
+    bool processCommandLine(int argc, const char * * argv, StringBuffer &logdir);
     void log();
     bool sendReply(unsigned version);
 
@@ -78,7 +78,7 @@ class REMOTE_API CRemoteSlave
 public:
     CRemoteSlave(const char * name, unsigned _tag, unsigned _version, bool _stayAlive);
 
-    void run(int argc, char * argv[]);
+    void run(int argc, const char * * argv);
     virtual bool processCommand(byte action, ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results) = 0;
 
 protected:

common/workunit/workunit.cpp

@@ -3624,7 +3624,7 @@ public:
     virtual IConstWUAppValueIterator & getApplicationValues() const { return *new CArrayIteratorOf<IConstWUAppValue,IConstWUAppValueIterator> (appvalues, 0, (IConstWorkUnitInfo *) this); };
 protected:
     StringAttr wuid, user, jobName, clusterName, timeScheduled, wuscope;
-    mutable CachedTags<CLocalWUAppValue,IConstWUAppValue> appvalues;
+    mutable CachedWUAppValues appvalues;
     unsigned totalThorTime;
     WUState state;
     WUAction action;
@@ -8515,7 +8515,7 @@ void CLocalWorkUnit::setStatistic(StatisticCreatorType creatorType, const char *
             statTree->setProp("@desc", optDescription);
 
         if (statistics.cached)
-            statistics.append(LINK(statTree));
+            statistics.append(statTree); // links statTree
 
         mergeAction = StatsMergeAppend;
     }
@@ -11528,33 +11528,28 @@ void CLocalWUException::setPriority(unsigned _priority)
 
 //==========================================================================================
 
-CLocalWUAppValue::CLocalWUAppValue(IPropertyTree *props, unsigned child) : p(props)
+CLocalWUAppValue::CLocalWUAppValue(const IPropertyTree *_owner, const IPropertyTree *_props) : owner(_owner), props(_props)
 {
-    StringAttrBuilder propPath(prop);
-    propPath.append("*[").append(child).append("]");
 }
 
 const char * CLocalWUAppValue::queryApplication() const
 {
-    return p->queryName();
+    return owner->queryName();
 }
 
 const char * CLocalWUAppValue::queryName() const
 {
-    IPropertyTree* val=p->queryPropTree(prop.str());
-    if(val)
-        return val->queryName();
-    return ""; // Should not happen in normal usage
+    return props->queryName();
 }
 
 const char * CLocalWUAppValue::queryValue() const
 {
-    return p->queryProp(prop.str());
+    return props->queryProp(nullptr);
 }
 
 //==========================================================================================
 
-CLocalWUStatistic::CLocalWUStatistic(IPropertyTree *props) : p(props)
+CLocalWUStatistic::CLocalWUStatistic(const IPropertyTree *props) : p(props)
 {
 }
 

common/workunit/workunit.ipp

@@ -29,11 +29,10 @@
 
 class WORKUNIT_API CLocalWUAppValue : implements IConstWUAppValue, public CInterface
 {
-    Owned<IPropertyTree> p;
-    StringAttr prop;
+    Owned<const IPropertyTree> owner, props;
 public:
     IMPLEMENT_IINTERFACE;
-    CLocalWUAppValue(IPropertyTree *p,unsigned child);
+    CLocalWUAppValue(const IPropertyTree *_owner, const IPropertyTree *_props);
 
     virtual const char *queryApplication() const;
     virtual const char *queryName() const;
@@ -43,11 +42,11 @@ public:
 
 class WORKUNIT_API CLocalWUStatistic : implements IConstWUStatistic, public CInterface
 {
-    Owned<IPropertyTree> p;
+    Owned<const IPropertyTree> p;
 public:
     IMPLEMENT_IINTERFACE;
 
-    CLocalWUStatistic(IPropertyTree *p);
+    CLocalWUStatistic(const IPropertyTree *p);
 
     virtual IStringVal & getCreator(IStringVal & str) const override;
     virtual IStringVal & getDescription(IStringVal & str, bool createDefault) const override;
@@ -66,7 +65,7 @@ public:
 
 //==========================================================================================
 
-template <typename T, typename IT> struct CachedTags
+template <typename T, typename IT, void (* ADD)(const IPropertyTree *p, IArrayOf<IT> &dst)> struct CachedTags
 {
     CachedTags(): cached(false) {}
     void load(IPropertyTree* p,const char* xpath)
@@ -77,9 +76,8 @@ template <typename T, typename IT> struct CachedTags
             Owned<IPropertyTreeIterator> r = p->getElements(xpath);
             for (r->first(); r->isValid(); r->next())
             {
-                IPropertyTree *rp = &r->query();
-                rp->Link();
-                tags.append(*new T(rp));
+                const IPropertyTree &rp = r->query();
+                ADD(&rp, tags); // NB: links items it adds
             }
             cached = true;
         }
@@ -96,23 +94,23 @@ template <typename T, typename IT> struct CachedTags
                 Owned<IPropertyTreeIterator> r = branch->getElements("*");
                 for (r->first(); r->isValid(); r->next())
                 {
-                    IPropertyTree *rp = &r->query();
-                    rp->Link();
-                    tags.append(*new T(rp));
+                    const IPropertyTree &rp = r->query();
+                    ADD(&rp, tags); // NB: links items it adds
                 }
             }
             cached = true;
         }
     }
-    void append(IPropertyTree * p)
-    {
-        tags.append(*new T(p));
-    }
 
     operator IArrayOf<IT>&() { return tags; }
     unsigned ordinality() const { return tags.ordinality(); }
     IT & item(unsigned i) const { return tags.item(i); }
 
+    void append(const IPropertyTree * p)
+    {
+        ADD(p, tags); // NB: links p
+    }
+
     void kill()
     {
         cached = false;
@@ -123,66 +121,21 @@ template <typename T, typename IT> struct CachedTags
     IArrayOf<IT> tags;
 };
 
-template <>  struct CachedTags<CLocalWUAppValue, IConstWUAppValue>
+void addStatistic(const IPropertyTree * p, IArrayOf<IConstWUStatistic> &dst)
 {
-    CachedTags(): cached(false) {}
-    void load(IPropertyTree* p,const char* xpath)
-    {
-        if (!cached)
-        {
-            assertex(tags.length() == 0);
-            Owned<IPropertyTreeIterator> r = p->getElements(xpath);
-            for (r->first(); r->isValid(); r->next())
-            {
-                IPropertyTree *rp = &r->query();
-                Owned<IPropertyTreeIterator> v = rp->getElements("*");
-                unsigned pos = 1;
-                for (v->first(); v->isValid(); v->next())
-                {
-                    rp->Link();
-                    tags.append(*new CLocalWUAppValue(rp,pos++));
-                }
-            }
-            cached = true;
-        }
-    }
-    void loadBranch(IPropertyTree* p,const char* xpath)
-    {
-        if (!cached)
-        {
-            assertex(tags.length() == 0);
-            Owned<IPropertyTree> branch = p->getBranch(xpath);
-            if (branch)
-            {
-                Owned<IPropertyTreeIterator> r = branch->getElements("*");
-                for (r->first(); r->isValid(); r->next())
-                {
-                    IPropertyTree *rp = &r->query();
-                    Owned<IPropertyTreeIterator> v = rp->getElements("*");
-                    unsigned pos = 1;
-                    for (v->first(); v->isValid(); v->next())
-                    {
-                        rp->Link();
-                        tags.append(*new CLocalWUAppValue(rp,pos++));
-                    }
-                }
-            }
-            cached = true;
-        }
-    }
+    dst.append(*new CLocalWUStatistic(&OLINK(*p)));
+}
 
+void addWUAppValues(const IPropertyTree *p, IArrayOf<IConstWUAppValue> &dst)
+{
+    Owned<IPropertyTreeIterator> v = p->getElements("*");
+    for (v->first(); v->isValid(); v->next())
+        dst.append(*new CLocalWUAppValue(&OLINK(*p), &v->get()));
+}
 
-    operator IArrayOf<IConstWUAppValue>&() { return tags; }
+typedef CachedTags<CLocalWUStatistic, IConstWUStatistic, addStatistic> CachedStatistics;
+typedef CachedTags<CLocalWUStatistic, IConstWUAppValue, addWUAppValues> CachedWUAppValues;
 
-    void kill()
-    {
-        cached = false;
-        tags.kill();
-    }
-
-    bool cached;
-    IArrayOf<IConstWUAppValue> tags;
-};
 
 //==========================================================================================
 
@@ -215,8 +168,8 @@ protected:
     mutable IArrayOf<IWUResult> results;
     mutable IArrayOf<IWUResult> temporaries;
     mutable IArrayOf<IWUResult> variables;
-    mutable CachedTags<CLocalWUAppValue,IConstWUAppValue> appvalues;
-    mutable CachedTags<CLocalWUStatistic,IConstWUStatistic> statistics;
+    mutable CachedWUAppValues appvalues;
+    mutable CachedStatistics statistics;
     mutable Owned<IUserDescriptor> userDesc;
     Mutex locked;
     Owned<ISecManager> secMgr;

dali/base/dadfs.cpp

@@ -45,6 +45,7 @@
 #include <vector>
 #include <unordered_map>
 #include <algorithm>
+#include <time.h>
 
 #ifdef _DEBUG
 //#define EXTRA_LOGGING
@@ -176,6 +177,15 @@ static IPropertyTree *getEmptyAttr()
     return createPTree("Attr");
 }
 
+static double calcFileCost(const char * cluster, double sizeGB, double fileAgeDays)
+{
+    IPropertyTree * plane = queryStoragePlane(cluster);
+    if (!plane)
+        return 0.0;
+    double storageCostDaily = plane->getPropReal("cost/@storageAtRest", 0.0) * 12 / 365;
+    return storageCostDaily * sizeGB * fileAgeDays;
+}
+
 RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn)
 {
     partno--;
@@ -4705,6 +4715,27 @@ public:
         else
             return false;
     }
+    virtual double getCost(const char * cluster) override
+    {
+        CDateTime dt;
+        getModificationTime(dt);
+        double fileAgeDays = difftime(time(nullptr), dt.getSimple())/(24*60*60);
+        double sizeGB = getDiskSize(true, false) / ((double)1024 * 1024 * 1024);
+
+        if (isEmptyString(cluster))
+        {
+            StringArray clusterNames;
+            unsigned countClusters = getClusterNames(clusterNames);
+            double totalCost = 0.0;
+            for (unsigned i = 0; i < countClusters; i++)
+                totalCost += calcFileCost(clusterNames[i], sizeGB, fileAgeDays);
+            return totalCost;
+        }
+        else
+        {
+            return calcFileCost(cluster, sizeGB, fileAgeDays);
+        }
+    }
 };
 
 static unsigned findSubFileOrd(const char *name)
@@ -6654,6 +6685,18 @@ public:
     {
         return false;
     }
+
+    virtual double getCost(const char * cluster) override
+    {
+        double totalCost = 0.0;
+        CriticalBlock block (sect);
+        ForEachItemIn(i,subfiles)
+        {
+            IDistributedFile &f = subfiles.item(i);
+            totalCost += f.getCost(cluster);
+        }
+        return totalCost;
+    }
 };
 
 // --------------------------------------------------------

dali/base/dadfs.hpp

@@ -415,6 +415,7 @@ interface IDistributedFile: extends IInterface
     virtual bool getSkewInfo(unsigned &maxSkew, unsigned &minSkew, unsigned &maxSkewPart, unsigned &minSkewPart, bool calculateIfMissing) = 0;
     virtual int  getExpire() = 0;
     virtual void setExpire(int expireDays) = 0;
+    virtual double getCost(const char * cluster) = 0;
 };
 
 

dali/dfu/dfurun.cpp

@@ -1657,7 +1657,7 @@ public:
                                 if (!mspec.isReplicated())
                                     needrep = false;
                             }
-#ifndef _DEBUG
+#if !defined(_DEBUG) &&  !defined(_CONTAINERIZED)
                             StringBuffer gname;
                             if (!destination->getRemoteGroupOverride()&&!testLocalCluster(destination->getGroupName(0,gname).str())) {
                                 throw MakeStringException(-1,"IMPORT cluster %s is not recognized locally",gname.str());

dali/ft/ftslave.cpp

@@ -136,12 +136,34 @@ public:
     }
 };
 
-
-int main(int argc, char * argv[])
+static constexpr const char * defaultYaml = R"!!(
+version: "1.0"
+ftslave:
+  name: ftslave
+  logging:
+    detail: 50
+)!!";
+
+int main(int argc, const char * * argv)
 {
     InitModuleObjects();
     setDaliServixSocketCaching(true);
     installDefaultFileHooks(nullptr);
+    try
+    {
+        loadConfiguration(defaultYaml, argv, "ftslave", "FTSLAVE", nullptr, nullptr);
+    }
+    catch (IException * e)
+    {
+        OERRLOG(e);
+        e->Release();
+        return 1;
+    }
+    catch(...)
+    {
+        OERRLOG("Failed to load configuration");
+        return 1;
+    }
     FtSlave slave;
     slave.run(argc, argv);
     return 0;

deployment/deployutils/deployutils.cpp

@@ -3944,7 +3944,7 @@ bool validateEnv(IConstEnvironment* pConstEnv, bool abortOnException)
     StringBuffer sb;
     const char* inDir = pEnvRoot->queryProp(XML_TAG_ENVSETTINGS"/path");
     if (inDir)
-      sb.append("/componentfiles/configxml");
+      sb.append(inDir).append("/componentfiles/configxml");
     else
       sb.append(hpccBuildInfo.componentDir).append("/configxml");
     configGenMgr.setown(createConfigGenMgr(*pConstEnv, callback, NULL, sb.str(), tempdir, NULL, NULL, NULL));

esp/scm/ws_dfu.ecm

@@ -191,6 +191,7 @@ ESPStruct [nil_remove] DFUFileDetail
     [min_ver("1.39")] ESParray<ESPstruct DFUFileBloom> Blooms;
     [min_ver("1.40")] int ExpireDays;
     [min_ver("1.41")] string KeyType;
+    [min_ver("1.59")] string Cost;
 };
 
 ESPStruct DFUSpaceItem
@@ -967,8 +968,8 @@ ESPresponse [exceptions_inline, nil_remove] DFUFilePublishResponse
 //  ===========================================================================
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.58"),
-    default_client_version("1.58"),
+    version("1.59"),
+    default_client_version("1.59"),
     noforms,
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/services/ws_dfu/ws_dfuService.cpp

@@ -45,6 +45,7 @@
 #include "fverror.hpp"
 #include "nbcd.hpp"
 #include "thorcommon.hpp"
+#include "jstats.h"
 
 #include "jstring.hpp"
 #include "exception_util.hpp"
@@ -2644,6 +2645,13 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
                 FileDetails.setJsonInfo(jsonLayout);
         }
     }
+    if (version >= 1.59)
+    {
+        double totalCost = df->getCost(cluster);
+        StringBuffer s;
+        formatMoney(s, money2cost_type(totalCost));
+        FileDetails.setCost(s);
+    }
     PROGLOG("doGetFileDetails: %s done", name);
 }
 

helm/hpcc/values.schema.json

@@ -433,9 +433,20 @@
           "type" : "string"
         },
         "defaultSprayParts" : {
-            "description": "Number of parts sprayed by default",
-            "type" : "integer"
+          "description": "Number of parts sprayed by default",
+          "type" : "integer"
+        },
+        "cost" : {
+          "description": "Costs associated with the storage and use of the plane",
+          "type" : "object",
+          "properties": {
+            "storageAtRest" : {
+              "description": "Storage cost (GiB/month)",
+              "type": "number"
+            }
           }
+        }
+
       },
       "required": [ "name", "prefix" ],
       "additionalProperties": false

helm/hpcc/values.yaml

@@ -104,6 +104,8 @@ storage:
   #   hosts: <name>                         # Name of the host group for bare metal - must match the name of the storage plane..
   #   secret: <secret-id>                   # what secret is required to access the files.  This could optionally become a list if required (or add secrets:).
   #   defaultSprayParts: 4                  # The number of partitions created when spraying (default: 1)
+  #   cost:                                 # The storage cost
+  #     storageAtRest: 0.113                # Storage at rest cost: cost per GiB/month
   #   options:                              # not sure if it is needed
 
   dllStorage:

system/jlib/jstats.cpp

@@ -438,7 +438,7 @@ private:
 static MoneyLocale moneyLocale;
 CriticalSection MoneyLocale::cslock;
 
-static StringBuffer & formatMoney(StringBuffer &out, unsigned __int64 value)
+StringBuffer & formatMoney(StringBuffer &out, unsigned __int64 value)
 {
     std::stringstream ss;
     std::locale & loc = moneyLocale.queryMoneyLocale();

system/jlib/jstats.h

@@ -824,5 +824,6 @@ public:
     }
 };
 
+extern jlib_decl StringBuffer & formatMoney(StringBuffer &out, unsigned __int64 value);
 
 #endif

thorlcr/activities/diskread/thdiskreadslave.cpp

@@ -410,8 +410,8 @@ void CDiskRecordPartHandler::close(CRC32 &fileCRC)
     }
     if (partStream)
     {
-        partStream->stop(&fileCRC);
         mergeStats(fileStats, partStream);
+        partStream->stop(&fileCRC);
     }
 }