Merge pull request #4996 from jakesmith/hpcc-9968

HPCC-9968 - New EXPIRE semantics

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

common/thorhelper/thorfile.cpp

@@ -32,15 +32,7 @@ namespace thorfile {
 
 void setExpiryTime(IPropertyTree & properties, unsigned expireDays)
 {
-    if (expireDays)
-    {
-        CDateTime expireTime;
-        StringBuffer expireTimeText;
-        expireTime.setNow();
-        expireTime.adjustTime(expireDays * 24 * 60);
-        expireTime.getString(expireTimeText, false);
-        properties.setProp("@expires", expireTimeText.str());
-    }
+    properties.setPropInt("@expireDays", expireDays);
 }
 
 

dali/daliadmin/daliadmin.cpp

@@ -1300,11 +1300,25 @@ static void listexpires(const char * lfnmask, IUserDescriptor *user)
     IDFAttributesIterator *iter = queryDistributedFileDirectory().getDFAttributesIterator(lfnmask,user,true,false);
     ForEach(*iter) {
         IPropertyTree &attr=iter->query();
-        const char * expires = attr.queryProp("@expires");
-        if (expires&&*expires) {
-            const char * name = attr.queryProp("@name");
-            if (name&&*name) {
-                OUTLOG("%s expires on %s",name,expires);
+        if (attr.hasProp("@expireDays"))
+        {
+            unsigned expireDays = attr.getPropInt("@expireDays");
+            const char *name = attr.queryProp("@name");
+            const char *lastAccessed = attr.queryProp("@accessed");
+            if (lastAccessed && name&&*name) // NB: all files that have expireDays should have lastAccessed also
+            {
+                StringBuffer days;
+                if (0 == expireDays)
+                    days.append("the sasha default number of days");
+                else
+                {
+                    days.append(expireDays);
+                    if (expireDays>1)
+                        days.append(" days");
+                    else
+                        days.append(" day");
+                }
+                OUTLOG("%s, last accessed = %s, set to expire %s after last accessed", name, lastAccessed, days.str());
             }
         }
     }

dali/sasha/saxref.cpp

@@ -34,6 +34,9 @@
 #define DEFAULT_XREF_INTERVAL       48 // hours
 #define DEFAULT_EXPIRY_INTERVAL     24 // hours
 
+#define DEFAULT_EXPIRYDAYS 14
+#define DEFAULT_PERSISTEXPIRYDAYS 7
+
 
 #define LOGPFX "XREF: "
 #define LOGPFX2 "FILEEXPIRY: "
@@ -2289,25 +2292,42 @@ public:
         if (stopped)
             return;
         PROGLOG(LOGPFX2 "Started");
+        unsigned defaultExpireDays = serverConfig->getPropInt("DfuExpiry/@expiryDefault", DEFAULT_EXPIRYDAYS);
+        unsigned defaultPersistExpireDays = serverConfig->getPropInt("DfuExpiry/@persistExpiryDefault", DEFAULT_PERSISTEXPIRYDAYS);
         StringArray expirylist;
         Owned<IDFAttributesIterator> iter = queryDistributedFileDirectory().getDFAttributesIterator("*",UNKNOWN_USER,true,false);//MORE:Pass IUserDescriptor
-        ForEach(*iter) {
+        ForEach(*iter)
+        {
             IPropertyTree &attr=iter->query();
-            const char * expires = attr.queryProp("@expires");
-            if (expires&&*expires) {
+            if (attr.hasProp("@expireDays"))
+            {
+                unsigned expireDays = attr.getPropInt("@expireDays");
                 const char * name = attr.queryProp("@name");
-                if (name&&*name) {
+                const char *lastAccessed = attr.queryProp("@accessed");
+                if (lastAccessed && name&&*name)
+                {
+                    if (0 == expireDays)
+                    {
+                        bool isPersist = attr.queryProp("@persist");
+                        expireDays = isPersist ? defaultPersistExpireDays : defaultExpireDays;
+                    }
                     CDateTime now;
                     now.setNow();
-                    CDateTime expwhen;
-                    try {
-                        expwhen.setString(expires);
-                        if (!expwhen.isNull()&&(now.compare(expwhen,false)>0)) {
+                    CDateTime expires;
+                    try
+                    {
+                        expires.setString(lastAccessed);
+                        expires.adjustTime(60*24*expireDays);
+                        if (now.compare(expires,false)>0)
+                        {
                             expirylist.append(name);
-                            PROGLOG(LOGPFX2 "%s expired on %s",name,expires);
+                            StringBuffer expiresStr;
+                            expires.getString(expiresStr);
+                            PROGLOG(LOGPFX2 "%s expired on %s", name, expiresStr.str());
                         }
                     }
-                    catch (IException *e) {
+                    catch (IException *e)
+                    {
                         StringBuffer s;
                         EXCLOG(e, LOGPFX2 "setdate");
                         e->Release();
@@ -2316,15 +2336,18 @@ public:
             }
         }
         iter.clear();
-        ForEachItemIn(i,expirylist) {
+        ForEachItemIn(i,expirylist)
+        {
             if (stopped)
                 break;
             const char *lfn = expirylist.item(i);
             PROGLOG(LOGPFX2 "Deleting %s",lfn);
-            try {
+            try
+            {
                 queryDistributedFileDirectory().removeEntry(lfn,UNKNOWN_USER);//MORE:Pass IUserDescriptor
             }
-            catch (IException *e) { // may want to just detach if fails
+            catch (IException *e) // may want to just detach if fails
+            {
                 StringBuffer s;
                 EXCLOG(e, LOGPFX2 "remove");
                 e->Release();

ecl/hthor/hthor.cpp

@@ -657,7 +657,8 @@ void CHThorDiskWriteActivity::publish()
     properties.setProp("@job", agent.queryWorkUnit()->getJobName(info).str());
     setFormat(desc);
 
-    setExpiryTime(properties, helper.getExpiryDays());
+    if (helper.getFlags() & TDWexpires)
+        setExpiryTime(properties, helper.getExpiryDays());
     if (helper.getFlags() & TDWupdate)
     {
         unsigned eclCRC;
@@ -1175,7 +1176,8 @@ void CHThorIndexWriteActivity::execute()
         properties.setProp("ECL", rececl);
     
 
-    setExpiryTime(properties, helper.getExpiryDays());
+    if (helper.getFlags() & TIWexpires)
+        setExpiryTime(properties, helper.getExpiryDays());
     if (helper.getFlags() & TIWupdate)
     {
         unsigned eclCRC;

initfiles/componentfiles/configxml/sasha.xsd

@@ -451,6 +451,20 @@
                 </xs:appinfo>
             </xs:annotation>
         </xs:attribute>
+        <xs:attribute name="PersistExpiryDefault" type="xs:nonNegativeInteger" use="optional" default="7">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Default number of days to delete unused persist files</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
+        <xs:attribute name="ExpiryDefault" type="xs:nonNegativeInteger" use="optional" default="14">
+            <xs:annotation>
+                <xs:appinfo>
+                    <tooltip>Default number of days to delete unused standard files that are flagged with EXPIRY</tooltip>
+                </xs:appinfo>
+            </xs:annotation>
+        </xs:attribute>
     </xs:attributeGroup>
     <xs:attributeGroup name="ThorQMon">
         <xs:attribute name="thorQMonQueues" type="xs:string" use="optional" default="*">

initfiles/componentfiles/configxml/sasha.xsl

@@ -216,6 +216,16 @@
                 <xsl:attribute name="at">
                    <xsl:value-of select="@ExpiryAt"/>
                 </xsl:attribute>
+                <xsl:if test="string(@PersistExpiryDefault) != ''">
+                    <xsl:attribute name="persistExpiryDefault">
+                       <xsl:value-of select="@PersistExpiryDefault"/>
+                    </xsl:attribute>
+                </xsl:if>
+                <xsl:if test="string(@ExpiryDefault) != ''">
+                    <xsl:attribute name="expiryDefault">
+                       <xsl:value-of select="@ExpiryDefault"/>
+                    </xsl:attribute>
+                </xsl:if>
             </xsl:element>
             <xsl:element name="ThorQMon">
                 <xsl:attribute name="queues">

roxie/ccd/ccdserver.cpp

@@ -10841,7 +10841,8 @@ public:
             fileProps.setProp("@workunit", workUnit->getWuid(wuid).str());
             fileProps.setProp("@job", workUnit->getJobName(job).str());
         }
-        setExpiryTime(fileProps, helper.getExpiryDays());
+        if (flags & TDWexpires)
+            setExpiryTime(fileProps, helper.getExpiryDays());
         if (flags & TDWupdate)
         {
             unsigned eclCRC;
@@ -11372,7 +11373,8 @@ public:
         if(rececl && *rececl)
             properties.setProp("ECL", rececl);
 
-        setExpiryTime(properties, helper.getExpiryDays());
+        if (helper.getFlags() & TIWexpires)
+            setExpiryTime(properties, helper.getExpiryDays());
         if (helper.getFlags() & TIWupdate)
         {
             unsigned eclCRC;

thorlcr/activities/indexwrite/thindexwrite.cpp

@@ -250,7 +250,8 @@ public:
             IPropertyTree &props = fileDesc->queryProperties();
             props.setPropInt64("@recordCount", recordsProcessed);
             props.setProp("@kind", "key");
-            setExpiryTime(props, helper->getExpiryDays());
+            if (0 != (helper->getFlags() & TIWexpires))
+                setExpiryTime(props, helper->getExpiryDays());
             if (TIWupdate & helper->getFlags())
             {
                 unsigned eclCRC;

thorlcr/activities/keydiff/thkeydiff.cpp

@@ -159,7 +159,8 @@ public:
         wu.clear();
 
         IPropertyTree &patchProps = patchDesc->queryProperties();
-        setExpiryTime(patchProps, helper->getExpiryDays());
+        if (0 != (helper->getFlags() & KDPexpires))
+            setExpiryTime(patchProps, helper->getExpiryDays());
         IPropertyTree &originalProps = originalDesc->queryProperties();;
         if (originalProps.queryProp("ECL"))
             patchProps.setProp("ECL", originalProps.queryProp("ECL"));

thorlcr/activities/keypatch/thkeypatch.cpp

@@ -148,7 +148,8 @@ public:
 
         IPropertyTree &props = newIndexDesc->queryProperties();
         props.setProp("@kind", "key");
-        setExpiryTime(props, helper->getExpiryDays());
+        if (0 != (helper->getFlags() & KDPexpires))
+            setExpiryTime(props, helper->getExpiryDays());
 
         // Fill in some logical file properties here
         IPropertyTree &originalProps = originalDesc->queryProperties();;

thorlcr/activities/thdiskbase.cpp

@@ -157,7 +157,8 @@ void CWriteMasterBase::publish()
     props.setPropInt64("@recordCount", recordsProcessed);
     if (0 == (diskHelperBase->getFlags() & TDXtemporary) || container.queryJob().queryUseCheckpoints())
     {
-        setExpiryTime(props, diskHelperBase->getExpiryDays());
+        if (0 != (diskHelperBase->getFlags() & TDWexpires))
+            setExpiryTime(props, diskHelperBase->getExpiryDays());
         if (TDWupdate & diskHelperBase->getFlags())
         {
             unsigned eclCRC;