HPCC-9970 Default persist files to expiring after 14 days

Can be configured with expirePersists and defaultPersistExpiry debug options

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hqlcpp/hqlckey.cpp

@@ -1604,6 +1604,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyDiff(BuildCtx & ctx, IHqlEx
         flags.append("|KDPnooverwrite");
     if (!output->isConstant())
         flags.append("|KDPvaroutputname");
+    if (expr->hasAttribute(expireAtom))
+        flags.append("|KDPexpires");
 
     if (flags.length())
         doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
@@ -1649,6 +1651,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityKeyPatch(BuildCtx & ctx, IHqlE
         flags.append("|KDPnooverwrite");
     if (!output->isConstant())
         flags.append("|KDPvaroutputname");
+    if (expr->hasAttribute(expireAtom))
+        flags.append("|KDPexpires");
     if (flags.length())
         doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);
 

ecl/hqlcpp/hqlcpp.cpp

@@ -67,6 +67,8 @@
 #define MAX_STATIC_ROW_SIZE             10000
 #define MAX_LOCAL_ROW_SIZE              32
 #define DEFAULT_NLP_DETAIL              1
+#define DEFAULT_EXPIRY_PERIOD               7
+#define DEFAULT_PERSIST_EXPIRY_PERIOD       0           // Persists that expire by default have a period defined in sasha
 
 
 #ifdef _WIN32
@@ -1526,6 +1528,9 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.reduceNetworkTraffic,"aggressiveOptimizeProjects", false),
         DebugOption(options.notifyOptimizedProjects, "notifyOptimizedProjects", 0),
         DebugOption(options.optimizeProjectsPreservePersists,"optimizeProjectsPreservePersists", false),
+        DebugOption(options.expirePersists, "expirePersists", true),
+        DebugOption(options.defaultPersistExpiry, "defaultPersistExpiry", DEFAULT_PERSIST_EXPIRY_PERIOD),
+        DebugOption(options.defaultExpiry, "defaultExpiry", DEFAULT_EXPIRY_PERIOD),
 
         DebugOption(options.checkAsserts,"checkAsserts", true),
         DebugOption(options.optimizeLoopInvariant,"optimizeLoopInvariant", false),      // doesn't fully work yet! and has little effect, and messes up the alias dependencies

ecl/hqlcpp/hqlcpp.ipp

@@ -558,6 +558,8 @@ struct HqlCppOptions
     unsigned            complexClassesThreshold;
     unsigned            complexClassesActivityFilter;
     unsigned            subgraphToRegeneate;
+    unsigned            defaultPersistExpiry;
+    unsigned            defaultExpiry;
     CompilerType        targetCompiler;
     DBZaction           divideByZeroAction;
     bool                peephole;
@@ -723,6 +725,7 @@ struct HqlCppOptions
     bool                multiplePersistInstances;
     bool                optimizeParentAccess;
     bool                expandPersistInputDependencies;
+    bool                expirePersists;
 };
 
 //Any information gathered while processing the query should be moved into here, rather than cluttering up the translator class

ecl/hqlcpp/hqlhtcpp.cpp

@@ -66,7 +66,6 @@
 
 #define MAX_ROWS_OUTPUT_TO_SDS              1000
 #define MAX_SAFE_RECORD_SIZE                10000000
-#define DEFAULT_EXPIRY_PERIOD               7
 #define MAX_GRAPH_ECL_LENGTH                1000
 #define MAX_ROW_VALUE_TEXT_LEN              10
 
@@ -9485,7 +9484,7 @@ void HqlCppTranslator::buildExpiryHelper(BuildCtx & ctx, IHqlExpression * expire
     {
         LinkedHqlExpr num = expireAttr->queryChild(0);
         if (!num)
-            num.setown(getSizetConstant(DEFAULT_EXPIRY_PERIOD));
+            num.setown(getSizetConstant(options.defaultExpiry));
         doBuildUnsignedFunction(ctx, "getExpiryDays", num);
     }
 }
@@ -9942,6 +9941,8 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutputIndex(BuildCtx & ctx, IH
     if (updateAttr)                       flags.append("|TIWupdatecrc");
     if (updateAttr && !updateAttr->queryAttribute(alwaysAtom)) flags.append("|TIWupdate");
     if (!hasTLK && !singlePart)           flags.append("|TIWlocal");
+    if (expr->hasAttribute(expireAtom))   flags.append("|TIWexpires");
+
     if (compressAttr)
     {
         if (compressAttr->hasAttribute(rowAtom))   flags.append("|TIWrowcompress");
@@ -10094,7 +10095,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
     IHqlExpression * program  = queryRealChild(expr, 2);
     IHqlExpression * csvAttr = expr->queryAttribute(csvAtom);
     IHqlExpression * xmlAttr = expr->queryAttribute(xmlAtom);
-    IHqlExpression * expireAttr = expr->queryAttribute(expireAtom);
+    LinkedHqlExpr expireAttr = expr->queryAttribute(expireAtom);
     IHqlExpression * seq = querySequence(expr);
 
     IHqlExpression *pipe = NULL;
@@ -10239,6 +10240,11 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
                 getNameCtx.addReturn(queryQuotedNullExpr());
             }
 
+            //Expire if explicit, or if a persisted output, and persists default to expiring
+            bool expires = expireAttr || (expr->hasAttribute(_workflowPersist_Atom) && options.expirePersists);
+            if (expires && !expireAttr && options.defaultPersistExpiry)
+                expireAttr.setown(createExprAttribute(expireAtom, getSizetConstant(options.defaultPersistExpiry)));
+
             //virtual unsigned getFlags() = 0;
             IHqlExpression * updateAttr = expr->queryAttribute(updateAtom);
             StringBuffer s;
@@ -10260,6 +10266,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityOutput(BuildCtx & ctx, IHqlExp
             if (expr->hasAttribute(jobTempAtom)) flags.append("|TDXjobtemp");
             if (updateAttr) flags.append("|TDWupdatecrc");
             if (updateAttr && !updateAttr->queryAttribute(alwaysAtom)) flags.append("|TDWupdate");
+            if (expires) flags.append("|TDWexpires");
 
             if (flags.length())
                 doBuildUnsignedFunction(instance->classctx, "getFlags", flags.str()+1);

rtl/include/eclhelper.hpp

@@ -1075,6 +1075,7 @@ enum
     TDWnewcompress      = 0x20000,      // new compressed format - only specified on output
     TDWnooverwrite      = 0x40000,
     TDWupdatecrc        = 0x80000,      // has format crc
+    TDWexpires          = 0x100000,
 };
 
 //flags for thor index read
@@ -1114,6 +1115,7 @@ enum
     TIWnooverwrite      = 0x0200,
     TIWupdatecrc        = 0x0400,
     TIWhaswidth         = 0x0800,
+    TIWexpires          = 0x1000,
 };
 
 //flags for thor dataset/temp tables
@@ -1854,6 +1856,7 @@ enum
     KDPtransform            = 0x0002,
     KDPvaroutputname        = 0x0004,
     KDPnooverwrite          = 0x0008,
+    KDPexpires              = 0x0010,
 };
 
 struct IHThorKeyDiffArg : public IHThorArg