Merge pull request #14872 from shamser/issue25747

HPCC-25747 DFUInfo returns logical file at rest cost

Reviewed-by: Jake Smith
Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>
Merged-by: Gavin Halliday <ghalliday@hpccsystems.com>

dali/base/dadfs.cpp

@@ -45,6 +45,7 @@
 #include <vector>
 #include <unordered_map>
 #include <algorithm>
+#include <time.h>
 
 #ifdef _DEBUG
 //#define EXTRA_LOGGING
@@ -176,6 +177,15 @@ static IPropertyTree *getEmptyAttr()
     return createPTree("Attr");
 }
 
+static double calcFileCost(const char * cluster, double sizeGB, double fileAgeDays)
+{
+    IPropertyTree * plane = queryStoragePlane(cluster);
+    if (!plane)
+        return 0.0;
+    double storageCostDaily = plane->getPropReal("cost/@storageAtRest", 0.0) * 12 / 365;
+    return storageCostDaily * sizeGB * fileAgeDays;
+}
+
 RemoteFilename &constructPartFilename(IGroup *grp,unsigned partno,unsigned partmax,const char *name,const char *partmask,const char *partdir,unsigned copy,ClusterPartDiskMapSpec &mspec,RemoteFilename &rfn)
 {
     partno--;
@@ -4705,6 +4715,27 @@ public:
         else
             return false;
     }
+    virtual double getCost(const char * cluster) override
+    {
+        CDateTime dt;
+        getModificationTime(dt);
+        double fileAgeDays = difftime(time(nullptr), dt.getSimple())/(24*60*60);
+        double sizeGB = getDiskSize(true, false) / ((double)1024 * 1024 * 1024);
+
+        if (isEmptyString(cluster))
+        {
+            StringArray clusterNames;
+            unsigned countClusters = getClusterNames(clusterNames);
+            double totalCost = 0.0;
+            for (unsigned i = 0; i < countClusters; i++)
+                totalCost += calcFileCost(clusterNames[i], sizeGB, fileAgeDays);
+            return totalCost;
+        }
+        else
+        {
+            return calcFileCost(cluster, sizeGB, fileAgeDays);
+        }
+    }
 };
 
 static unsigned findSubFileOrd(const char *name)
@@ -6654,6 +6685,18 @@ public:
     {
         return false;
     }
+
+    virtual double getCost(const char * cluster) override
+    {
+        double totalCost = 0.0;
+        CriticalBlock block (sect);
+        ForEachItemIn(i,subfiles)
+        {
+            IDistributedFile &f = subfiles.item(i);
+            totalCost += f.getCost(cluster);
+        }
+        return totalCost;
+    }
 };
 
 // --------------------------------------------------------

dali/base/dadfs.hpp

@@ -415,6 +415,7 @@ interface IDistributedFile: extends IInterface
     virtual bool getSkewInfo(unsigned &maxSkew, unsigned &minSkew, unsigned &maxSkewPart, unsigned &minSkewPart, bool calculateIfMissing) = 0;
     virtual int  getExpire() = 0;
     virtual void setExpire(int expireDays) = 0;
+    virtual double getCost(const char * cluster) = 0;
 };
 
 

esp/scm/ws_dfu.ecm

@@ -191,6 +191,7 @@ ESPStruct [nil_remove] DFUFileDetail
     [min_ver("1.39")] ESParray<ESPstruct DFUFileBloom> Blooms;
     [min_ver("1.40")] int ExpireDays;
     [min_ver("1.41")] string KeyType;
+    [min_ver("1.59")] string Cost;
 };
 
 ESPStruct DFUSpaceItem
@@ -967,8 +968,8 @@ ESPresponse [exceptions_inline, nil_remove] DFUFilePublishResponse
 //  ===========================================================================
 ESPservice [
     auth_feature("DEFERRED"),
-    version("1.58"),
-    default_client_version("1.58"),
+    version("1.59"),
+    default_client_version("1.59"),
     noforms,
     exceptions_inline("./smc_xslt/exceptions.xslt")] WsDfu
 {

esp/services/ws_dfu/ws_dfuService.cpp

@@ -45,6 +45,7 @@
 #include "fverror.hpp"
 #include "nbcd.hpp"
 #include "thorcommon.hpp"
+#include "jstats.h"
 
 #include "jstring.hpp"
 #include "exception_util.hpp"
@@ -2644,6 +2645,13 @@ void CWsDfuEx::doGetFileDetails(IEspContext &context, IUserDescriptor *udesc, co
                 FileDetails.setJsonInfo(jsonLayout);
         }
     }
+    if (version >= 1.59)
+    {
+        double totalCost = df->getCost(cluster);
+        StringBuffer s;
+        formatMoney(s, money2cost_type(totalCost));
+        FileDetails.setCost(s);
+    }
     PROGLOG("doGetFileDetails: %s done", name);
 }
 

helm/hpcc/values.schema.json

@@ -433,9 +433,20 @@
           "type" : "string"
         },
         "defaultSprayParts" : {
-            "description": "Number of parts sprayed by default",
-            "type" : "integer"
+          "description": "Number of parts sprayed by default",
+          "type" : "integer"
+        },
+        "cost" : {
+          "description": "Costs associated with the storage and use of the plane",
+          "type" : "object",
+          "properties": {
+            "storageAtRest" : {
+              "description": "Storage cost (GiB/month)",
+              "type": "number"
+            }
           }
+        }
+
       },
       "required": [ "name", "prefix" ],
       "additionalProperties": false

helm/hpcc/values.yaml

@@ -104,6 +104,8 @@ storage:
   #   hosts: <name>                         # Name of the host group for bare metal - must match the name of the storage plane..
   #   secret: <secret-id>                   # what secret is required to access the files.  This could optionally become a list if required (or add secrets:).
   #   defaultSprayParts: 4                  # The number of partitions created when spraying (default: 1)
+  #   cost:                                 # The storage cost
+  #     storageAtRest: 0.113                # Storage at rest cost: cost per GiB/month
   #   options:                              # not sure if it is needed
 
   dllStorage:

system/jlib/jstats.cpp

@@ -438,7 +438,7 @@ private:
 static MoneyLocale moneyLocale;
 CriticalSection MoneyLocale::cslock;
 
-static StringBuffer & formatMoney(StringBuffer &out, unsigned __int64 value)
+StringBuffer & formatMoney(StringBuffer &out, unsigned __int64 value)
 {
     std::stringstream ss;
     std::locale & loc = moneyLocale.queryMoneyLocale();

system/jlib/jstats.h

@@ -824,5 +824,6 @@ public:
     }
 };
 
+extern jlib_decl StringBuffer & formatMoney(StringBuffer &out, unsigned __int64 value);
 
 #endif