Add DISTRIBUTED flag to DATASET(N, trans(COUNTER))

This is patch is just adding the keyword and persisting
it in the object (via isDistributed() function), but
it's not being used yet. We need to fix the limits of
getRow() to obey numRows() in the engine before coding
the distributed part.

The fix mentioned above is my next step.

ecl/hql/hqlgram.y

@@ -2846,6 +2846,24 @@ expireAttr
                         }
     ;
 
+optDatasetFlags
+    :                   { $$.setNullExpr(); }
+    | ',' datasetFlags    { $$.inherit($2); }
+    ;
+
+datasetFlags
+    : datasetFlag
+    | datasetFlag ',' datasetFlags
+                        { $$.setExpr(createComma($1.getExpr(), $3.getExpr()), $1); }
+    ;
+
+datasetFlag
+    : DISTRIBUTED       {
+                            $$.setExpr(createExprAttribute(distributedAtom));
+                            $$.setPosition($1);
+                        }
+    ;
+
 optIndexFlags
     :                   { $$.setNullExpr(); $$.clearPosition(); }
     | ',' indexFlags    { $$.setExpr($2.getExpr()); $$.setPosition($1); }
@@ -8034,14 +8052,13 @@ simpleDataSet
                             $$.setExpr(createDataset(no_workunit_dataset, $8.getExpr(), arg));
                             $$.setPosition($1);
                         }
-    | DATASET '(' thorFilenameOrList ',' beginCounterScope transform endCounterScope ')'
+    | DATASET '(' thorFilenameOrList ',' beginCounterScope transform endCounterScope optDatasetFlags ')'
                         {
-                            // TODO: use DISTRIBUTED flag
                             parser->normalizeExpression($3, type_int, false);
                             IHqlExpression * counter = $7.getExpr();
                             if (counter)
                                 counter = createAttribute(_countProject_Atom, counter);
-                            $$.setExpr(createDataset(no_dataset_from_transform, $3.getExpr(), createComma($6.getExpr(), counter)));
+                            $$.setExpr(createDataset(no_dataset_from_transform, $3.getExpr(), createComma($6.getExpr(), counter, $8.getExpr())));
                             $$.setPosition($1);
                         }
     | ENTH '(' dataSet ',' expression optCommonAttrs ')'

ecl/hqlcpp/hqlhtcpp.cpp

@@ -15768,8 +15768,15 @@ ABoundActivity * HqlCppTranslator::doBuildActivityTempTable(BuildCtx & ctx, IHql
     }
 
     doBuildUnsignedFunction(instance->startctx, "numRows", rowsExpr);
+
+    // unsigned getFlags()
+    StringBuffer flags;
+    if (expr->hasProperty(distributedAtom))
+        flags.append("|TTFdistributed");
     if (!values->isConstant())
-        doBuildBoolFunction(instance->startctx, "isConstant", false);
+        flags.append("|TTFnoconstant");
+    if (flags.length())
+        doBuildUnsignedFunction(instance->startctx, "getFlags", flags.str()+1);
 
     buildInstanceSuffix(instance);
 
@@ -15910,8 +15917,14 @@ ABoundActivity * HqlCppTranslator::doBuildActivityInlineTable(BuildCtx & ctx, IH
     OwnedHqlExpr rowsExpr = getSizetConstant(maxRows);
     doBuildUnsignedFunction(instance->startctx, "numRows", rowsExpr);
 
+    // unsigned getFlags()
+    StringBuffer flags;
+    if (expr->hasProperty(distributedAtom))
+        flags.append("|TTFdistributed");
     if (!values->isConstant())
-        doBuildBoolFunction(instance->startctx, "isConstant", false);
+        flags.append("|TTFnoconstant");
+    if (flags.length())
+        doBuildUnsignedFunction(instance->startctx, "getFlags", flags.str()+1);
 
     buildInstanceSuffix(instance);
 
@@ -15945,9 +15958,14 @@ ABoundActivity * HqlCppTranslator::doBuildActivityCountTransform(BuildCtx & ctx,
     // unsigned numRows() - count is guaranteed by lexer
     doBuildUnsignedFunction(instance->startctx, "numRows", count);
 
-    // bool isConstant() - default is true
+    // unsigned getFlags()
+    StringBuffer flags;
+    if (expr->hasProperty(distributedAtom))
+        flags.append("|TTFdistributed");
     if (!isConstantTransform(transform))
-        doBuildBoolFunction(instance->startctx, "isConstant", false);
+        flags.append("|TTFnoconstant");
+    if (flags.length())
+        doBuildUnsignedFunction(instance->startctx, "getFlags", flags.str()+1);
 
     buildInstanceSuffix(instance);
 

ecl/regress/dataset_transform.ecl

@@ -49,3 +49,8 @@ output(ds50);
 output(true);
 ds5 := DATASET(C, t2());
 output(ds5);
+
+// distributed
+output(true);
+dsd := DATASET(10, t1(COUNTER), DISTRIBUTED);
+output(dsd);

rtl/include/eclhelper.hpp

@@ -956,6 +956,9 @@ enum ActivityInterfaceEnum
     TAIexternal_1,
     TAIpipethrougharg_2,
     TAIpipewritearg_2,
+    TAItemptablearg_2,
+
+//Should remain as last of all meaningful tags, but before aliases
     TAImax,
 
 //Only aliases follow - for interfaces implemented via typedefs
@@ -998,6 +1001,7 @@ interface IHThorArg : public IInterface
 
 typedef IHThorArg * (*EclHelperFactory)();
 
+//flags for thor disk access
 enum 
 {
 //General disk access flags
@@ -1042,7 +1046,7 @@ enum
     TDWupdatecrc        = 0x80000,      // has format crc
 };
 
-
+//flags for thor index read
 enum
 {
     TIRsorted           = 0x00000001,
@@ -1080,6 +1084,13 @@ enum
     TIWhaswidth         = 0x0800,
 };
 
+//flags for thor dataset/temp tables
+enum
+{
+    TTFnoconstant        = 0x0001,      // default flags is zero
+    TTFdistributed       = 0x0002,
+};
+
 struct IHThorIndexWriteArg : public IHThorArg
 {
     virtual const char * getFileName() = 0;
@@ -1400,10 +1411,21 @@ struct IHThorTempTableArg : public IHThorArg
 {
     virtual size32_t getRow(ARowBuilder & rowBuilder, unsigned row) = 0;
     virtual unsigned numRows() = 0;
-    virtual bool isConstant()                           { return true; }
+    virtual bool isConstant()                           { return true; }    // deprecate in favour of getFlags
     virtual size32_t getRowSingle(ARowBuilder & rowBuilder) = 0;            // only valid for TAKtemprow, could be called directly
 };
 
+/*
+ * New Temp table that allows flags to be set and retrieved on one
+ * single method. Future-proof and should merge with the interface
+ * above in the next major release.
+ */
+struct IHThorTempTableExtraArg : public IHThorTempTableArg
+{
+    virtual unsigned getFlags() = 0;
+};
+
+
 struct IHThorSampleArg : public IHThorArg
 {
     virtual unsigned getProportion() = 0;

rtl/include/eclhelper_base.hpp

@@ -1299,7 +1299,7 @@ class CThorHashAggregateArg : public CThorArg, implements IHThorHashAggregateArg
     virtual size32_t mergeAggregate(ARowBuilder & rowBuilder, const void * src) { rtlFailUnexpected(); return 0; }
 };
 
-class CThorTempTableArg : public CThorArg, implements IHThorTempTableArg
+class CThorTempTableArg : public CThorArg, implements IHThorTempTableExtraArg
 {
 public:
     virtual void Link() const { RtlCInterface::Link(); }
@@ -1313,11 +1313,14 @@ public:
         case TAIarg:
         case TAItemptablearg_1:
             return static_cast<IHThorTempTableArg *>(this);
+        case TAItemptablearg_2:
+            return static_cast<IHThorTempTableExtraArg *>(this);
         }
         return NULL;
     }
 
-    virtual bool isConstant()                           { return true; }
+    virtual unsigned getFlags()                         { return 0; }
+    virtual bool isConstant()                           { return (getFlags() & TTFnoconstant) == 0; }
     virtual size32_t getRowSingle(ARowBuilder & rowBuilder) { return 0; }
 };