HPCC-10022 Allow the max number of persist instances to be set

Set with #option('defaultNumPersistInstances', <n>);

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

ecl/hqlcpp/hqlcpp.cpp

@@ -1458,7 +1458,7 @@ void HqlCppTranslator::cacheOptions()
 
     struct DebugOption 
     {
-        typedef enum { typeByte, typeUnsigned, typeBool } OptionType;
+        typedef enum { typeByte, typeUnsigned, typeSigned, typeBool } OptionType;
 
         DebugOption (bool & _option, const char * name, bool defaultValue) : option(&_option), optName(name)
         { 
@@ -1474,7 +1474,12 @@ void HqlCppTranslator::cacheOptions()
         { 
             _option = defaultValue;
             type = typeUnsigned;
-        };
+        }
+        DebugOption (int & _option, const char * name, unsigned defaultValue) : option(&_option), optName(name)
+        {
+            _option = defaultValue;
+            type = typeSigned;
+        }
 
         void setValue(const char * val)
         {
@@ -1492,6 +1497,12 @@ void HqlCppTranslator::cacheOptions()
                     *u = (unsigned)atoi(val);
                     break;
                 }
+            case typeSigned:
+                {
+                    signed * u = (signed*)option;
+                    *u = (signed)atoi(val);
+                    break;
+                }
             case typeByte:
                 {
                     byte * b = (byte*)option;
@@ -1713,6 +1724,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.optimizeParentAccess,"optimizeParentAccess",false),
         DebugOption(options.expandPersistInputDependencies,"expandPersistInputDependencies",true),
         DebugOption(options.multiplePersistInstances,"multiplePersistInstances",true),
+        DebugOption(options.defaultNumPersistInstances,"defaultNumPersistInstances",-1),
     };
 
     //get options values from workunit

ecl/hqlcpp/hqlcpp.ipp

@@ -560,6 +560,7 @@ struct HqlCppOptions
     unsigned            subgraphToRegeneate;
     unsigned            defaultPersistExpiry;
     unsigned            defaultExpiry;
+    int                 defaultNumPersistInstances;
     CompilerType        targetCompiler;
     DBZaction           divideByZeroAction;
     bool                peephole;

ecl/hqlcpp/hqlttcpp.cpp

@@ -142,7 +142,7 @@ public:
     }
 
     void extractGlobal(IHqlExpression * global, ClusterType platform);
-    void extractStoredInfo(IHqlExpression * expr, IHqlExpression * codehash, bool isRoxie, bool multiplePersistInstances);
+    void extractStoredInfo(IHqlExpression * expr, IHqlExpression * codehash, bool isRoxie, int multiplePersistInstances);
     void checkFew(HqlCppTranslator & translator);
     void splitGlobalDefinition(ITypeInfo * type, IHqlExpression * value, IConstWorkUnit * wu, SharedHqlExpr & setOutput, OwnedHqlExpr * getOutput, bool isRoxie);
     IHqlExpression * getStoredKey();
@@ -4952,7 +4952,7 @@ void GlobalAttributeInfo::extractGlobal(IHqlExpression * global, ClusterType pla
     persistOp = no_global;
 }
 
-void GlobalAttributeInfo::extractStoredInfo(IHqlExpression * expr, IHqlExpression * _codehash, bool isRoxie, bool multiplePersistInstances)
+void GlobalAttributeInfo::extractStoredInfo(IHqlExpression * expr, IHqlExpression * _codehash, bool isRoxie, int multiplePersistInstances)
 {
     node_operator op = expr->getOperator();
     few = expr->hasAttribute(fewAtom) || (isRoxie) || (value->isDictionary() && !expr->hasAttribute(manyAtom));
@@ -4983,7 +4983,7 @@ void GlobalAttributeInfo::extractStoredInfo(IHqlExpression * expr, IHqlExpressio
         setCluster(queryRealChild(expr, 1));
         few = expr->hasAttribute(fewAtom);       // PERSISTs need a consistent format.
         extraOutputAttr.setown(createComma(LINK(expr->queryAttribute(expireAtom)), LINK(expr->queryAttribute(clusterAtom))));
-        numPersistInstances = multiplePersistInstances ? -1 : 0;
+        numPersistInstances = multiplePersistInstances;
         if (expr->hasAttribute(multipleAtom))
             numPersistInstances = (int)getIntValue(queryAttributeChild(expr, multipleAtom, 0), -1);
         else if (expr->hasAttribute(singleAtom))
@@ -5396,13 +5396,14 @@ static HqlTransformerInfo workflowTransformerInfo("WorkflowTransformer");
 WorkflowTransformer::WorkflowTransformer(IWorkUnit * _wu, HqlCppTranslator & _translator)
 : NewHqlTransformer(workflowTransformerInfo), wu(_wu), translator(_translator), wfidCount(0)
 {
+    const HqlCppOptions & options = translator.queryOptions();
     trivialStoredWfid = 0;
     nextInternalFunctionId = 0;
     onceWfid = 0;
-    combineAllStored = translator.queryOptions().combineAllStored;
-    combineTrivialStored = translator.queryOptions().combineTrivialStored;
-    expandPersistInputDependencies = translator.queryOptions().expandPersistInputDependencies;
-    multiplePersistInstances = translator.queryOptions().multiplePersistInstances;
+    combineAllStored = options.combineAllStored;
+    combineTrivialStored = options.combineTrivialStored;
+    expandPersistInputDependencies = options.expandPersistInputDependencies;
+    multiplePersistInstances = options.multiplePersistInstances ? options.defaultNumPersistInstances : 0;
     isRootAction = true;
     isRoxie = (translator.getTargetClusterType() == RoxieCluster);
     workflowOut = NULL;

ecl/hqlcpp/hqlttcpp.ipp

@@ -536,7 +536,7 @@ protected:
     bool                      isRootAction;
     bool                      isRoxie;
     bool                      expandPersistInputDependencies;
-    bool                      multiplePersistInstances;
+    int                       multiplePersistInstances;
     UnsignedArray             cumulativeDependencies;
     UnsignedArray             emptyDependencies;
     UnsignedArray             storedWfids;

ecl/regress/issue10022c.ecl

@@ -0,0 +1,14 @@
+#option ('multiplePersistInstances', true);
+#option ('defaultNumPersistInstances', 4);
+
+ds := dataset('x', { string line }, FLAT);
+
+p1 := ds(line <> '') : persist('p1', many, single);
+
+p2 := ds(line = '') : persist('p2', multiple);
+
+p3 := ds(line[1]='!') : persist('p3','10Mway',multiple(10));
+
+p4 := ds(line[1] = ' ') : persist('p4');
+
+output(p1 + p2 + p3 + p4);