HPCC-12872 TRACE activity

Implement a new TRACE activity.

Syntax is:

myds := TRACE(ds [, traceOptions]);

The available options are:

 - Zero or more expressions, which act as a filter. Only rows matching the
   filter will be included in the tracing.
 - KEEP (n) indicating how many rows will be traced.
 - SKIP (n) indicating that n rows will be skipped before tracing starts.
 - SAMPLE (n) indicating that only every nth row is traced.
 - NAMED(string) providing the name for the rows in the tracing.

Tracing is output to the log file, in the form
 TRACE: <name><fieldname>value</fieldname>...</name>

Tracing is not output by default even if TRACE statements are present - only
if the workunit debug value traceEnabled is set or if the default platform
settings are changed to always output tracing. In Roxie you can also request
tracing on a deployed query by specifying traceEnabled=1 in the query XML. It
is therefore possible to leave trace statements in the ECL without any
detectable overhead until tracing is enabled.

It is also possible to override the default value for KEEP at a global,
per-workunit, or per-query level.

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/thorhelper/thorcommon.cpp

@@ -776,8 +776,9 @@ extern const char * getActivityText(ThorActivityKind kind)
     case TAKsmartjoin:              return "Smart Join";
     case TAKsmartdenormalize:       return "Smart Denormalize";
     case TAKsmartdenormalizegroup:  return "Smart Denormalize Group";
-    case TAKselfdenormalize:       return "Self Denormalize";
-    case TAKselfdenormalizegroup:  return "Self Denormalize Group";
+    case TAKselfdenormalize:        return "Self Denormalize";
+    case TAKselfdenormalizegroup:   return "Self Denormalize Group";
+    case TAKtrace:                  return "Trace";
     }
     throwUnexpected();
 }

ecl/eclagent/eclgraph.cpp

@@ -293,6 +293,8 @@ static IHThorActivity * createActivity(IAgentContext & agent, unsigned activityI
         return createLibraryCallActivity(agent, activityId, subgraphId, (IHThorLibraryCallArg &)arg, kind, node);
     case TAKsorted:
         return createSortedActivity(agent, activityId, subgraphId, (IHThorSortedArg &)arg, kind);
+    case TAKtrace:
+        return createTraceActivity(agent, activityId, subgraphId, (IHThorTraceArg &)arg, kind);
     case TAKgrouped:
         return createGroupedActivity(agent, activityId, subgraphId, (IHThorGroupedArg &)arg, kind);
     case TAKnwayjoin:

ecl/hql/hqlatoms.cpp

@@ -388,6 +388,7 @@ IAtom * soapActionAtom;
 IAtom * httpHeaderAtom;
 IAtom * prototypeAtom;
 IAtom * proxyAddressAtom;
+IAtom * sampleAtom;
 IAtom * sort_AllAtom;
 IAtom * sort_KeyedAtom;
 IAtom * sortedAtom;
@@ -416,6 +417,7 @@ IAtom * trimAtom;
 IAtom * trueAtom;
 IAtom * tomitaAtom;
 IAtom * topAtom;
+IAtom * traceAtom;
 IAtom * typeAtom;
 IAtom * _uid_Atom;
 IAtom * unknownAtom;
@@ -796,6 +798,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKESYSATOM(rowsid);
     MAKEATOM(rowLimit);
     MAKEATOM(rule);
+    MAKEATOM(sample);
     MAKEATOM(save);
     MAKEATOM(scan);
     MAKEATOM(scanAll);
@@ -849,6 +852,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEATOM(tiny);
     MAKEATOM(tomita);
     MAKEATOM(top);
+    MAKEATOM(trace);
     MAKEATOM(trim);
     MAKEATOM(true);
     MAKEATOM(type);

ecl/hql/hqlatoms.hpp

@@ -390,6 +390,7 @@ extern HQL_API IAtom * soapActionAtom;
 extern HQL_API IAtom * httpHeaderAtom;
 extern HQL_API IAtom * prototypeAtom;
 extern HQL_API IAtom * proxyAddressAtom;
+extern HQL_API IAtom * sampleAtom;
 extern HQL_API IAtom * sort_AllAtom;
 extern HQL_API IAtom * sort_KeyedAtom;
 extern HQL_API IAtom * sortedAtom;
@@ -417,6 +418,7 @@ extern HQL_API IAtom * timestampAtom;
 extern HQL_API IAtom * tinyAtom;
 extern HQL_API IAtom * tomitaAtom;
 extern HQL_API IAtom * topAtom;
+extern HQL_API IAtom * traceAtom;
 extern HQL_API IAtom * trimAtom;
 extern HQL_API IAtom * trueAtom;
 extern HQL_API IAtom * typeAtom;

ecl/hql/hqlgram.y

@@ -439,6 +439,7 @@ static void eclsyntaxerror(HqlGram * parser, const char * s, short yystate, int
   TOPN
   TOUNICODE
   TOXML
+  TRACE
   TRANSFER
   TRANSFORM
   TRIM
@@ -8322,6 +8323,11 @@ simpleDataSet
                             $$.setExpr(createDataset(no_metaactivity, $3.getExpr(), createAttribute(pullAtom)));
                             $$.setPosition($1);
                         }
+    | TRACE '(' startTopLeftRightSeqFilter optTraceFlags ')' endTopLeftRightFilter endSelectorSequence
+                        {
+                            $$.setExpr(createDataset(no_metaactivity, $3.getExpr(), createComma(createAttribute(traceAtom), $4.getExpr())));
+                            $$.setPosition($1);
+                        }
     | DENORMALIZE '(' startLeftDelaySeqFilter ',' startRightFilter ',' expression ',' beginCounterScope transform endCounterScope optJoinFlags ')' endSelectorSequence
                         {
                             parser->normalizeExpression($7, type_boolean, false);
@@ -10560,6 +10566,44 @@ optSortList
     | ',' sortList
     ;
 
+optTraceFlags
+    : traceFlags
+    |                   {   $$.setNullExpr(); }
+    ;
+
+traceFlags
+    : ',' traceFlag     {   $$.setExpr($2.getExpr()); }
+    | traceFlags ',' traceFlag
+                        {   $$.setExpr(createComma($1.getExpr(), $3.getExpr())); }
+    ;
+
+traceFlag
+    : KEEP '(' expression ')'  {
+                            parser->normalizeExpression($3, type_int, false);
+                            $$.setExpr(createExprAttribute(keepAtom, $3.getExpr()), $1);
+                        }
+    | SKIP '(' expression ')'  {
+                            parser->normalizeExpression($3, type_int, false);
+                            $$.setExpr(createExprAttribute(skipAtom, $3.getExpr()), $1);
+                        }
+    | SAMPLE '(' expression  ')' {
+                            parser->normalizeExpression($3, type_int, false);
+                            $$.setExpr(createExprAttribute(sampleAtom, $3.getExpr()), $1);
+                        }
+    | NAMED '(' constExpression ')'
+                        {
+                            parser->normalizeStoredNameExpression($3);
+                            $$.setExpr(createExprAttribute(namedAtom, $3.getExpr()), $1);
+                        }
+    | expression
+                        {
+                            //MORE:SORTLIST  Allow a sortlist to be expanded!
+                            parser->normalizeExpression($1, type_boolean, false);
+                            $$.inherit($1);
+                        }
+    ;
+
+    
 doParseFlags
     : parseFlags
     ;

ecl/hql/hqllex.l

@@ -926,6 +926,7 @@ TOJSON              { RETURNSYM(TOJSON); }
 TOPN                { RETURNSYM(TOPN); }
 TOUNICODE           { RETURNSYM(TOUNICODE); }
 TOXML               { RETURNSYM(TOXML); }
+TRACE               { RETURNSYM(TRACE); }
 TRANSFER            { RETURNSYM(TRANSFER); }
 TRANSFORM           { RETURNHARD(TRANSFORM); }
 TRIM                { RETURNSYM(TRIM); }

ecl/hql/hqlthql.cpp

@@ -2511,6 +2511,8 @@ void HqltHql::toECL(IHqlExpression *expr, StringBuffer &s, bool paren, bool inTy
         case no_metaactivity:
             if (expr->hasAttribute(pullAtom))
                 s.append("PULL");
+            else if (expr->hasAttribute(traceAtom))
+                s.append("TRACE");
             else
                 s.append("no_metaactivity:unknown");
             defaultChildrenToECL(expr, s, inType);

ecl/hqlcpp/hqlcpp.ipp

@@ -1463,7 +1463,6 @@ public:
     ABoundActivity * doBuildActivityLinkedRawChildDataset(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityLoop(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityMerge(BuildCtx & ctx, IHqlExpression * expr);
-    ABoundActivity * doBuildActivityMetaActivity(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityNonEmpty(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityNWayMerge(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityNWayMergeJoin(BuildCtx & ctx, IHqlExpression * expr);
@@ -1480,6 +1479,7 @@ public:
     ABoundActivity * doBuildActivityPrefetchProject(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityProject(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityProcess(BuildCtx & ctx, IHqlExpression * expr);
+    ABoundActivity * doBuildActivityPullActivity(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityRegroup(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityRemote(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
     ABoundActivity * doBuildActivityReturnResult(BuildCtx & ctx, IHqlExpression * expr, bool isRoot);
@@ -1509,6 +1509,7 @@ public:
     ABoundActivity * doBuildActivityTable(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityFirstN(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityTempTable(BuildCtx & ctx, IHqlExpression * expr);
+    ABoundActivity * doBuildActivityTraceActivity(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityUngroup(BuildCtx & ctx, IHqlExpression * expr, ABoundActivity * boundDataset);
     ABoundActivity * doBuildActivityWorkunitRead(BuildCtx & ctx, IHqlExpression * expr);
     ABoundActivity * doBuildActivityXmlParse(BuildCtx & ctx, IHqlExpression * expr);

ecl/hqlcpp/hqlhtcpp.cpp

@@ -6706,7 +6706,12 @@ ABoundActivity * HqlCppTranslator::buildActivity(BuildCtx & ctx, IHqlExpression
                 result = doBuildActivityAggregate(ctx, expr);
                 break;
             case no_metaactivity:
-                result = doBuildActivityMetaActivity(ctx, expr);
+                if (expr->hasAttribute(pullAtom))
+                    result = doBuildActivityPullActivity(ctx, expr);
+                else if (expr->hasAttribute(traceAtom))
+                    result = doBuildActivityTraceActivity(ctx, expr);
+                else
+                    throwUnexpected();
                 break;
             case no_choosen:
                 result = doBuildActivityFirstN(ctx, expr);
@@ -15729,7 +15734,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivitySectionInput(BuildCtx & ctx, I
 
 //---------------------------------------------------------------------------
 
-ABoundActivity * HqlCppTranslator::doBuildActivityMetaActivity(BuildCtx & ctx, IHqlExpression * expr)
+ABoundActivity * HqlCppTranslator::doBuildActivityPullActivity(BuildCtx & ctx, IHqlExpression * expr)
 {
     Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, expr->queryChild(0));
     if (targetHThor())
@@ -15747,6 +15752,53 @@ ABoundActivity * HqlCppTranslator::doBuildActivityMetaActivity(BuildCtx & ctx, I
     return instance->getBoundActivity();
 }
 
+ABoundActivity * HqlCppTranslator::doBuildActivityTraceActivity(BuildCtx & ctx, IHqlExpression * expr)
+{
+    IHqlExpression * dataset = expr->queryChild(0);
+    Owned<ABoundActivity> boundDataset = buildCachedActivity(ctx, dataset);
+    assertex(expr->hasAttribute(traceAtom));
+    Owned<ActivityInstance> instance = new ActivityInstance(*this, ctx, TAKtrace, expr, "Trace");
+
+    buildActivityFramework(instance);
+    buildInstancePrefix(instance);
+
+    IHqlExpression *keepLimit = queryAttributeChild(expr, keepAtom, 0);
+    if (keepLimit)
+        doBuildUnsignedFunction(instance->startctx, "getKeepLimit", keepLimit);
+
+    IHqlExpression *skip = queryAttributeChild(expr, skipAtom, 0);
+    if (skip)
+        doBuildUnsignedFunction(instance->startctx, "getSkip", skip);
+
+    IHqlExpression *sample = queryAttributeChild(expr, sampleAtom, 0);
+    if (sample)
+        doBuildUnsignedFunction(instance->startctx, "getSample", sample);
+
+    IHqlExpression *named = queryAttributeChild(expr, namedAtom, 0);
+    if (named)
+        doBuildVarStringFunction(instance->startctx, "getName", named);
+
+    HqlExprAttr invariant;
+    OwnedHqlExpr cond = extractFilterConditions(invariant, expr, dataset, options.spotCSE, queryOptions().spotCseInIfDatasetConditions);
+
+    //Base class returns true, so only generate if no non-invariant conditions
+    if (cond)
+    {
+        BuildCtx funcctx(instance->startctx);
+        funcctx.addQuotedCompound("virtual bool isValid(const void * _self)");
+        funcctx.addQuotedLiteral("unsigned char * self = (unsigned char *) _self;");
+
+        bindTableCursor(funcctx, dataset, "self");
+        buildReturn(funcctx, cond);
+    }
+    if (invariant)
+        doBuildBoolFunction(instance->startctx, "canMatchAny", invariant);
+
+    buildInstanceSuffix(instance);
+    buildConnectInputOutput(ctx, instance, boundDataset, 0, 0);
+    return instance->getBoundActivity();
+}
+
 //---------------------------------------------------------------------------
 //-- no_sub --
 

ecl/hthor/hthor.cpp

@@ -4239,6 +4239,83 @@ const void * CHThorSortedActivity::nextGE(const void * seek, unsigned numFields)
 
 //=====================================================================================================
 
+CHThorTraceActivity::CHThorTraceActivity(IAgentContext &_agent, unsigned _activityId, unsigned _subgraphId, IHThorTraceArg &_arg, ThorActivityKind _kind)
+: CHThorSteppableActivityBase(_agent, _activityId, _subgraphId, _arg, _kind),
+  helper(_arg),  keepLimit(0), skip(0), sample(0), traceEnabled(false)
+{
+}
+
+void CHThorTraceActivity::ready()
+{
+    CHThorSimpleActivityBase::ready();
+    traceEnabled = agent.queryWorkUnit()->getDebugValueBool("traceEnabled", false);
+    if (traceEnabled && helper.canMatchAny())
+    {
+        keepLimit = helper.getKeepLimit();
+        if (keepLimit==(unsigned) -1)
+            keepLimit = agent.queryWorkUnit()->getDebugValueInt("traceLimit", 10);
+        skip = helper.getSkip();
+        sample = helper.getSample();
+        if (sample)
+            sample--;
+        name.setown(helper.getName());
+        if (!name)
+            name.set("Row");
+    }
+    else
+        keepLimit = 0;
+}
+
+void CHThorTraceActivity::done()
+{
+    CHThorSimpleActivityBase::done();
+    name.clear();
+}
+
+const void *CHThorTraceActivity::nextInGroup()
+{
+    OwnedConstRoxieRow ret(input->nextInGroup());
+    if (!ret)
+        return NULL;
+    onTrace(ret);
+    processed++;
+    return ret.getClear();
+}
+
+const void * CHThorTraceActivity::nextGE(const void * seek, unsigned numFields)
+{
+    OwnedConstRoxieRow ret(input->nextGE(seek, numFields));
+    if (ret)
+    {
+        onTrace(ret);
+        processed++;
+    }
+    return ret.getClear();
+}
+
+void CHThorTraceActivity::onTrace(const void *row)
+{
+    if (keepLimit && helper.isValid(row))
+    {
+        if (skip)
+            skip--;
+        else if (sample)
+            sample--;
+        else
+        {
+            CommonXmlWriter xmlwrite(XWFnoindent);
+            outputMeta.toXML((const byte *) row, xmlwrite);
+            DBGLOG("TRACE: <%s>%s<%s>", name.get(), xmlwrite.str(), name.get());
+            keepLimit--;
+            sample = helper.getSample();
+            if (sample)
+                sample--;
+        }
+    }
+}
+
+//=====================================================================================================
+
 void getLimitType(unsigned flags, bool & limitFail, bool & limitOnFail)
 {
     if((flags & JFmatchAbortLimitSkips) != 0)
@@ -10113,6 +10190,7 @@ MAKEFACTORY(Loop)
 MAKEFACTORY(Process)
 MAKEFACTORY(Grouped)
 MAKEFACTORY(Sorted)
+MAKEFACTORY(Trace)
 MAKEFACTORY(NWayInput)
 MAKEFACTORY(NWaySelect)
 MAKEFACTORY(NonEmpty)

ecl/hthor/hthor.hpp

@@ -194,6 +194,7 @@ extern HTHOR_API IHThorActivity *createLibraryCallActivity(IAgentContext &_agent
 extern HTHOR_API IHThorActivity *createGraphLoopResultReadActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorGraphLoopResultReadArg &arg, ThorActivityKind kind, __int64 graphId);
 extern HTHOR_API IHThorActivity *createGraphLoopResultWriteActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorGraphLoopResultWriteArg &arg, ThorActivityKind kind, __int64 graphId);
 extern HTHOR_API IHThorActivity *createSortedActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorSortedArg &arg, ThorActivityKind kind);
+extern HTHOR_API IHThorActivity *createTraceActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorTraceArg &arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createGroupedActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorGroupedArg &arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createNWayInputActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorNWayInputArg &arg, ThorActivityKind kind);
 extern HTHOR_API IHThorActivity *createNWayGraphLoopResultReadActivity(IAgentContext &, unsigned _activityId, unsigned _subgraphId, IHThorNWayGraphLoopResultReadArg &arg, ThorActivityKind kind, __int64 graphId);

ecl/hthor/hthor.ipp

@@ -1235,6 +1235,27 @@ public:
     virtual const void *nextGE(const void * seek, unsigned numFields);
 };
 
+class CHThorTraceActivity : public CHThorSteppableActivityBase
+{
+    IHThorTraceArg &helper;
+    roxiemem::OwnedRoxieString name;
+    unsigned keepLimit;
+    unsigned skip;
+    unsigned sample;
+    bool traceEnabled;
+public:
+    CHThorTraceActivity(IAgentContext &agent, unsigned _activityId, unsigned _subgraphId, IHThorTraceArg &_arg, ThorActivityKind _kind);
+
+    virtual void ready();
+    virtual void done();
+
+    //interface IHThorInput
+    virtual const void *nextInGroup();
+    virtual const void *nextGE(const void * seek, unsigned numFields);
+protected:
+    void onTrace(const void *row);
+};
+
 class CHThorJoinActivity : public CHThorActivityBase
 {
     enum { JSfill, JSfillleft, JSfillright, JScollate, JScompare, JSleftonly, JSrightonly } state;

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -908,6 +908,20 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="traceEnabled" type="xs:boolean" use="optional" default="false">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>TRACE activity output enabled by default (can be overridden in workunit or query)</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="traceLimit" type="xs:nonNegativeInteger" use="optional" default="10">
+      <xs:annotation>
+        <xs:appinfo>
+          <tooltip>Number of rows output by TRACE activity</tooltip>
+        </xs:appinfo>
+      </xs:annotation>
+    </xs:attribute>
     <xs:attribute name="udpTraceLevel" type="xs:nonNegativeInteger" use="optional" default="1">
       <xs:annotation>
         <xs:appinfo>

roxie/ccd/ccd.hpp

@@ -373,6 +373,8 @@ extern unsigned preabortIndexReadsThreshold;
 extern bool traceStartStop;
 extern bool traceServerSideCache;
 extern bool defaultTimeActivities;
+extern bool defaultTraceEnabled;
+extern unsigned defaultTraceLimit;
 extern unsigned watchActivityId;
 extern unsigned testSlaveFailure;
 extern unsigned dafilesrvLookupTimeout;

roxie/ccd/ccdmain.cpp

@@ -72,6 +72,8 @@ bool pretendAllOpt = false;
 bool traceStartStop = false;
 bool traceServerSideCache = false;
 bool defaultTimeActivities = true;
+bool defaultTraceEnabled = false;
+unsigned defaultTraceLimit = 10;
 unsigned watchActivityId = 0;
 unsigned testSlaveFailure = 0;
 unsigned restarts = 0;
@@ -781,6 +783,8 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         traceStartStop = topology->getPropBool("@traceStartStop", false);
         traceServerSideCache = topology->getPropBool("@traceServerSideCache", false);
         defaultTimeActivities = topology->getPropBool("@timeActivities", true);
+        defaultTraceEnabled = topology->getPropBool("@traceEnabled", false);
+        defaultTraceLimit = topology->getPropInt("@traceLimit", 10);
         clientCert.certificate.set(topology->queryProp("@certificateFileName"));
         clientCert.privateKey.set(topology->queryProp("@privateKeyFileName"));
         clientCert.passphrase.set(topology->queryProp("@passphrase"));

roxie/ccd/ccdquery.cpp

@@ -296,6 +296,8 @@ QueryOptions::QueryOptions()
     skipFileFormatCrcCheck = false;
     stripWhitespaceFromStoredDataset = ((ptr_ignoreWhiteSpace & defaultXmlReadFlags) != 0);
     timeActivities = defaultTimeActivities;
+    traceEnabled = defaultTraceEnabled;
+    traceLimit = defaultTraceLimit;
     allSortsMaySpill = false; // No global default for this
 }
 
@@ -321,6 +323,8 @@ QueryOptions::QueryOptions(const QueryOptions &other)
     skipFileFormatCrcCheck = other.skipFileFormatCrcCheck;
     stripWhitespaceFromStoredDataset = other.stripWhitespaceFromStoredDataset;
     timeActivities = other.timeActivities;
+    traceEnabled = other.traceEnabled;
+    traceLimit = other.traceLimit;
     allSortsMaySpill = other.allSortsMaySpill;
 }
 
@@ -356,6 +360,8 @@ void QueryOptions::setFromWorkUnit(IConstWorkUnit &wu, const IPropertyTree *stat
     updateFromWorkUnit(skipFileFormatCrcCheck, wu, "skipFileFormatCrcCheck");
     updateFromWorkUnit(stripWhitespaceFromStoredDataset, wu, "stripWhitespaceFromStoredDataset");
     updateFromWorkUnit(timeActivities, wu, "timeActivities");
+    updateFromWorkUnit(traceEnabled, wu, "traceEnabled");
+    updateFromWorkUnit(traceLimit, wu, "traceLimit");
     updateFromWorkUnit(allSortsMaySpill, wu, "allSortsMaySpill");
 }
 
@@ -401,6 +407,8 @@ void QueryOptions::setFromContext(const IPropertyTree *ctx)
         updateFromContext(skipFileFormatCrcCheck, ctx, "_SkipFileFormatCrcCheck", "@skipFileFormatCrcCheck");
         updateFromContext(stripWhitespaceFromStoredDataset, ctx, "_StripWhitespaceFromStoredDataset", "@stripWhitespaceFromStoredDataset");
         updateFromContext(timeActivities, ctx, "@timeActivities", "_TimeActivities");
+        updateFromContext(traceEnabled, ctx, "@traceEnabled", "_TraceEnabled");
+        updateFromContext(traceLimit, ctx, "@traceLimit", "_TraceLimit");
         // Note: allSortsMaySpill is not permitted at context level (too late anyway, unless I refactored)
     }
 }
@@ -697,6 +705,8 @@ protected:
             return createRoxieServerPipeWriteActivityFactory(id, subgraphId, *this, helperFactory, kind, usageCount(node), isRootAction(node));
         case TAKpull:
             return createRoxieServerPullActivityFactory(id, subgraphId, *this, helperFactory, kind);
+        case TAKtrace:
+            return createRoxieServerTraceActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKlinkedrawiterator:
             return createRoxieServerLinkedRawIteratorActivityFactory(id, subgraphId, *this, helperFactory, kind);
         case TAKremoteresult:

roxie/ccd/ccdquery.hpp

@@ -98,6 +98,7 @@ public:
     unsigned priority;
     unsigned timeLimit;
     unsigned warnTimeLimit;
+    unsigned traceLimit;
 
     memsize_t memoryLimit;
 
@@ -116,6 +117,7 @@ public:
     bool stripWhitespaceFromStoredDataset;
     bool timeActivities;
     bool allSortsMaySpill;
+    bool traceEnabled;
 
 private:
     static const char *findProp(const IPropertyTree *ctx, const char *name1, const char *name2);

roxie/ccd/ccdserver.cpp

@@ -18435,6 +18435,139 @@ IRoxieServerActivityFactory *createRoxieServerPullActivityFactory(unsigned _id,
 {
     return new CRoxieServerPullActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind);
 }
+//=================================================================================
+
+class CRoxieServerTraceActivity : public CRoxieServerActivity
+{
+public:
+    CRoxieServerTraceActivity(const IRoxieServerActivityFactory *_factory, IProbeManager *_probeManager)
+        : CRoxieServerActivity(_factory, _probeManager), helper((IHThorTraceArg &) basehelper),
+          keepLimit(0), skip(0), sample(0)
+    {
+        assertex(meta.hasXML());
+        traceEnabled = defaultTraceEnabled && !isBlind();
+    }
+
+    virtual void onCreate(IRoxieSlaveContext *_ctx, IHThorArg *_colocalParent)
+    {
+        CRoxieServerActivity::onCreate(_ctx, _colocalParent);
+        if (ctx)
+            traceEnabled = ctx->queryOptions().traceEnabled && !isBlind();
+    }
+    virtual void start(unsigned parentExtractSize, const byte *parentExtract, bool paused)
+    {
+        CRoxieServerActivity::start(parentExtractSize, parentExtract, paused);
+        if (traceEnabled && helper.canMatchAny())
+        {
+            keepLimit = helper.getKeepLimit();
+            if (keepLimit==(unsigned) -1)
+                keepLimit = ctx->queryOptions().traceLimit;
+            skip = helper.getSkip();
+            sample = helper.getSample();
+            if (sample)
+                sample--;
+            name.setown(helper.getName());
+            if (!name)
+                name.set("Row");
+        }
+        else
+            keepLimit = 0;
+    }
+    virtual void stop(bool aborting)
+    {
+        name.clear();
+        CRoxieServerActivity::stop(aborting);
+    }
+
+    virtual bool isPassThrough()
+    {
+        return true;
+    }
+
+    virtual const void *nextInGroup()
+    {
+        ActivityTimer t(totalCycles, timeActivities);
+        const void *row = input->nextInGroup();
+        if (row)
+        {
+            onTrace(row);
+            processed++;
+        }
+        return row;
+    }
+    virtual const void * nextSteppedGE(const void * seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra & stepExtra)
+    {
+        // MORE - will need rethinking once we rethink the nextSteppedGE interface for global smart-stepping.
+        ActivityTimer t(totalCycles, timeActivities);
+        const void * row = input->nextSteppedGE(seek, numFields, wasCompleteMatch, stepExtra);
+        if (row)
+        {
+            onTrace(row);
+            processed++;
+        }
+        return row;
+    }
+
+    virtual bool gatherConjunctions(ISteppedConjunctionCollector & collector)
+    {
+        return input->gatherConjunctions(collector);
+    }
+    virtual void resetEOF()
+    {
+        input->resetEOF();
+    }
+    IInputSteppingMeta * querySteppingMeta()
+    {
+        return input->querySteppingMeta();
+    }
+
+protected:
+    void onTrace(const void *row)
+    {
+        if (keepLimit && helper.isValid(row))
+        {
+            if (skip)
+                skip--;
+            else if (sample)
+                sample--;
+            else
+            {
+                CommonXmlWriter xmlwrite(XWFnoindent);
+                meta.toXML((const byte *) row, xmlwrite);
+                CTXLOG("TRACE: <%s>%s<%s>", name.get(), xmlwrite.str(), name.get());
+                keepLimit--;
+                sample = helper.getSample();
+                if (sample)
+                    sample--;
+            }
+        }
+    }
+    OwnedRoxieString name;
+    IHThorTraceArg &helper;
+    unsigned keepLimit;
+    unsigned skip;
+    unsigned sample;
+    bool traceEnabled;
+};
+
+class CRoxieServerTraceActivityFactory : public CRoxieServerActivityFactory
+{
+public:
+    CRoxieServerTraceActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind)
+        : CRoxieServerActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind)
+    {
+    }
+
+    virtual IRoxieServerActivity *createActivity(IProbeManager *_probeManager) const
+    {
+        return new CRoxieServerTraceActivity(this, _probeManager);
+    }
+};
+
+IRoxieServerActivityFactory *createRoxieServerTraceActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind)
+{
+    return new CRoxieServerTraceActivityFactory(_id, _subgraphId, _queryFactory, _helperFactory, _kind);
+}
 
 //=================================================================================
 

roxie/ccd/ccdserver.hpp

@@ -428,6 +428,7 @@ extern IRoxieServerActivityFactory *createRoxieServerWhenActionActivityFactory(u
 
 extern IRoxieServerActivityFactory *createRoxieServerDistributionActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind, bool _isRoot);
 extern IRoxieServerActivityFactory *createRoxieServerPullActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
+extern IRoxieServerActivityFactory *createRoxieServerTraceActivityFactory(unsigned _id, unsigned _subgraphId, IQueryFactory &_queryFactory, HelperFactory *_helperFactory, ThorActivityKind _kind);
 
 extern void throwRemoteException(IMessageUnpackCursor *extra);
 

roxie/roxiemem/roxiemem.hpp

@@ -393,7 +393,7 @@ public:
     inline const char * getLink() const { LinkRoxieRow(ptr); return ptr; }
     inline const char * set(const char * _ptr) { const char * temp = ptr; if (_ptr) LinkRoxieRow(_ptr); ptr = _ptr; ReleaseRoxieRow(temp); return ptr; }
     inline const char * setown(const char * _ptr) { const char * temp = ptr; ptr = _ptr; ReleaseRoxieRow(temp); return ptr; }
-
+    inline void clear() { const char * temp = ptr; ptr = NULL; ReleaseRoxieRow(temp);  }
 private:
     /* Disable use of some constructs that often cause memory leaks by creating private members */
     void operator = (const void * _ptr)              {  }

rtl/include/eclhelper.hpp

@@ -922,6 +922,7 @@ enum ThorActivityKind
     TAKlastdenormalizegroup,
     TAKjsonwrite,
     TAKjsonread,
+    TAKtrace,
 
     TAKlast
 };
@@ -1060,7 +1061,7 @@ enum ActivityInterfaceEnum
     TAIsubsortextra_1,
     TAIdictionaryworkunitwritearg_1,
     TAIdictionaryresultwritearg_1,
-
+    TAItracearg_1,
 //Should remain as last of all meaningful tags, but before aliases
     TAImax,
 
@@ -2779,6 +2780,17 @@ struct IHThorDictionaryResultWriteArg : public IHThorArg
     virtual IHThorHashLookupInfo * queryHashLookupInfo() = 0;
 };
 
+struct IHThorTraceArg : public IHThorArg
+{
+    virtual bool isValid(const void * _left) = 0;
+    virtual bool canMatchAny() = 0;
+    virtual unsigned getKeepLimit() = 0;
+    virtual unsigned getSample() = 0;
+    virtual unsigned getSkip() = 0;
+    virtual const char *getName() = 0;
+};
+
+
 //------------------------- Other stuff -------------------------
 
 struct IRemoteConnection;

rtl/include/eclhelper_base.hpp

@@ -2601,7 +2601,6 @@ typedef CThorNullArg CThorDatasetResultArg;
 typedef CThorNullArg CThorRowResultArg;
 typedef CThorNullArg CThorPullArg;
 
-
 class CThorParseArg : public CThorArg, implements IHThorParseArg
 {
     virtual void Link() const { RtlCInterface::Link(); }
@@ -3459,6 +3458,32 @@ class CThorSectionInputArg : public CThorArg, implements IHThorSectionInputArg
     virtual unsigned getFlags() { return 0; }
 };
 
+class CThorTraceArg : public CThorArg, implements IHThorTraceArg
+{
+    virtual void Link() const { RtlCInterface::Link(); }
+    virtual bool Release() const { return RtlCInterface::Release(); }
+    virtual bool isValid(const void * _left) { return true; }
+    virtual bool canMatchAny() { return true; }
+    virtual unsigned getKeepLimit() { return (unsigned) -1; }
+    virtual unsigned getSample() { return 0; }
+    virtual unsigned getSkip() { return 0; }
+    virtual const char *getName() { return NULL; }
+
+    virtual IInterface * selectInterface(ActivityInterfaceEnum which)
+    {
+        switch (which)
+        {
+        case TAIarg:
+        case TAItracearg_1:
+            return static_cast<IHThorTraceArg *>(this);
+        default:
+            break;
+        }
+        return NULL;
+    }
+};
+
+
 class CThorWhenActionArg : public CThorArg, implements IHThorWhenActionArg
 {
     virtual void Link() const { RtlCInterface::Link(); }

testing/regress/ecl/key/trace.xml

@@ -0,0 +1,72 @@
+<Dataset name='Result 1'>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+ <Row><id>5</id></Row>
+ <Row><id>6</id></Row>
+ <Row><id>7</id></Row>
+ <Row><id>8</id></Row>
+ <Row><id>9</id></Row>
+ <Row><id>10</id></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+ <Row><id>5</id></Row>
+ <Row><id>6</id></Row>
+ <Row><id>7</id></Row>
+ <Row><id>8</id></Row>
+ <Row><id>9</id></Row>
+ <Row><id>10</id></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+ <Row><id>5</id></Row>
+ <Row><id>6</id></Row>
+ <Row><id>7</id></Row>
+ <Row><id>8</id></Row>
+ <Row><id>9</id></Row>
+ <Row><id>10</id></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+ <Row><id>5</id></Row>
+ <Row><id>6</id></Row>
+ <Row><id>7</id></Row>
+ <Row><id>8</id></Row>
+ <Row><id>9</id></Row>
+ <Row><id>10</id></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><id>1</id></Row>
+ <Row><id>2</id></Row>
+ <Row><id>3</id></Row>
+ <Row><id>4</id></Row>
+ <Row><id>5</id></Row>
+ <Row><id>6</id></Row>
+ <Row><id>7</id></Row>
+ <Row><id>8</id></Row>
+ <Row><id>9</id></Row>
+ <Row><id>10</id></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+ <Row><_unnamed_cnt_1>1</_unnamed_cnt_1></Row>
+</Dataset>

testing/regress/ecl/trace.ecl

@@ -0,0 +1,17 @@
+
+
+rec := RECORD
+    UNSIGNED id;
+END;
+
+ds := NOFOLD(DATASET(10, transform(rec, SELF.id := COUNTER;), LOCAL));
+gds := GROUP(ds, id);
+ 
+sequential(
+  output(trace(ds));
+  output(trace(ds,id=5,NAMED('ds5only')));
+  output(trace(ds,keep(2),NAMED('keep2')));
+  output(trace(ds,skip(3),keep(2),NAMED('skip3keep2')));
+  output(trace(ds,sample(2),NAMED('sample2')));
+  output(table(trace(gds,sample(2),NAMED('sample2')),{COUNT(GROUP)}));
+);

thorlcr/activities/activityslaves_lcr.cmake

@@ -78,6 +78,7 @@ set (    SRCS
          thactivityutil.cpp 
          thdiskbaseslave.cpp 
          topn/thtopnslave.cpp 
+         trace/thtraceslave.cpp
          when/thwhenslave.cpp 
          wuidread/thwuidreadslave.cpp 
          wuidwrite/thwuidwriteslave.cpp 

thorlcr/activities/trace/thtraceslave.cpp

@@ -0,0 +1,151 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+#include "platform.h"
+#include "eclhelper.hpp"
+#include "slave.ipp"
+#include "thactivityutil.ipp"
+
+class CTraceSlaveActivity : public CSlaveActivity, public CThorDataLink, public CThorSteppable
+{
+    IThorDataLink *input;
+    IHThorTraceArg *helper;
+    OwnedRoxieString name;
+    unsigned keepLimit;
+    unsigned skip;
+    unsigned sample;
+    bool traceEnabled;
+
+public:
+    IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
+
+    CTraceSlaveActivity(CGraphElementBase *_container)
+        : CSlaveActivity(_container), CThorDataLink(this), CThorSteppable(this),
+          keepLimit(0), skip(0), sample(0), traceEnabled(false)
+    {
+        helper = (IHThorTraceArg *) queryHelper();
+    }
+    void init(MemoryBuffer &data, MemoryBuffer &slaveData)
+    {
+        appendOutputLinked(this);
+        traceEnabled = getOptBool(THOROPT_TRACE_ENABLED, false);
+    }
+    void start()
+    {
+        ActivityTimer s(totalCycles, timeActivities);
+        dataLinkStart();
+        input = inputs.item(0);
+        startInput(input);
+        if (traceEnabled && helper->canMatchAny() && queryRowMetaData())
+        {
+            keepLimit = helper->getKeepLimit();
+            if (keepLimit==(unsigned) -1)
+                keepLimit = getOptUInt(THOROPT_TRACE_LIMIT, 100);
+            skip = helper->getSkip();
+            sample = helper->getSample();
+            if (sample)
+                sample--;
+            name.setown(helper->getName());
+            if (!name)
+                name.set("Row");
+        }
+        else
+            keepLimit = 0;
+    }
+    void stop()
+    {
+        name.clear();
+        stopInput(input);
+        dataLinkStop();
+    }
+    void onTrace(const void *row)
+    {
+        if (keepLimit && helper->isValid(row))
+        {
+            if (skip)
+                skip--;
+            else if (sample)
+                sample--;
+            else
+            {
+                CommonXmlWriter xmlwrite(XWFnoindent);
+                queryRowMetaData()->toXML((const byte *) row, xmlwrite);
+                ActPrintLog("TRACE: <%s>%s<%s>", name.get(), xmlwrite.str(), name.get());
+                keepLimit--;
+                sample = helper->getSample();
+                if (sample)
+                    sample--;
+            }
+        }
+    }
+    CATCH_NEXTROW()
+    {
+        ActivityTimer t(totalCycles, timeActivities);
+        OwnedConstThorRow ret = input->nextRow();
+        if (ret)
+        {
+            onTrace(ret);
+            dataLinkIncrement();
+        }
+        return ret.getClear();
+    }
+    const void *nextRowGE(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    {
+        try { return nextRowGENoCatch(seek, numFields, wasCompleteMatch, stepExtra); }
+        CATCH_NEXTROWX_CATCH;
+    }
+    const void *nextRowGENoCatch(const void *seek, unsigned numFields, bool &wasCompleteMatch, const SmartStepExtra &stepExtra)
+    {
+        ActivityTimer t(totalCycles, timeActivities);
+        OwnedConstThorRow ret = input->nextRowGE(seek, numFields, wasCompleteMatch, stepExtra);
+        if (ret)
+        {
+            onTrace(ret);
+            dataLinkIncrement();
+        }
+        return ret.getClear();
+    }
+    bool gatherConjunctions(ISteppedConjunctionCollector &collector)
+    { 
+        return input->gatherConjunctions(collector);
+    }
+    void resetEOF() 
+    { 
+        input->resetEOF(); 
+    }
+    bool isGrouped() { return input->isGrouped(); }
+    void getMetaInfo(ThorDataLinkMetaInfo &info)
+    {
+        initMetaInfo(info);
+        calcMetaInfoSize(info,inputs.item(0));
+    }
+// steppable
+    virtual void setInput(unsigned index, CActivityBase *inputActivity, unsigned inputOutIdx)
+    {
+        CSlaveActivity::setInput(index, inputActivity, inputOutIdx);
+        CThorSteppable::setInput(index, inputActivity, inputOutIdx);
+    }
+    virtual IInputSteppingMeta *querySteppingMeta() { return CThorSteppable::inputStepping; }
+};
+
+
+////////////////////
+
+CActivityBase *createTraceSlave(CGraphElementBase *container)
+{
+    return new CTraceSlaveActivity(container);
+}

thorlcr/graph/thgraph.cpp

@@ -1059,6 +1059,7 @@ bool isGlobalActivity(CGraphElementBase &container)
         case TAKsimpleaction:
         case TAKsorted:
         case TAKdistributed:
+        case TAKtrace:
             break;
 
         case TAKnwayjoin:

thorlcr/master/thactivitymaster.cpp

@@ -151,6 +151,7 @@ public:
             case TAKsoap_datasetaction:
             case TAKhttp_rowdataset:
             case TAKdistributed:
+            case TAKtrace:
                 ret = new CMasterActivity(this);
                 break;
             case TAKskipcatch:

thorlcr/slave/slave.cpp

@@ -257,6 +257,7 @@ CActivityBase *createChildThroughNormalizeSlave(CGraphElementBase *container);
 CActivityBase *createWhenSlave(CGraphElementBase *container);
 CActivityBase *createDictionaryWorkunitWriteSlave(CGraphElementBase *container);
 CActivityBase *createDictionaryResultWriteSlave(CGraphElementBase *container);
+CActivityBase *createTraceSlave(CGraphElementBase *container);
 
 
 class CGenericSlaveGraphElement : public CSlaveGraphElement
@@ -351,6 +352,9 @@ public:
             case TAKsorted:
                 ret = createSortedSlave(this);
                 break;
+            case TAKtrace:
+                ret = createTraceSlave(this);
+                break;
             case TAKdedup:
                 if (queryGrouped())
                     ret = createGroupDedupSlave(this);

thorlcr/thorutil/thormisc.hpp

@@ -69,6 +69,8 @@
 #define THOROPT_LKJOIN_HASHJOINFAILOVER "lkjoin_hashjoinfailover" // Force SMART to failover to hash join (for testing only)                     (default = false)
 #define THOROPT_MAX_KERNLOG           "max_kern_level"          // Max kernel logging level, to push to workunit, -1 to disable                  (default = 3)
 #define THOROPT_COMP_FORCELZW         "forceLZW"                // Forces file compression to use LZW                                            (default = false)
+#define THOROPT_TRACE_ENABLED         "traceEnabled"            // Output from TRACE activity enabled                                            (default = false)
+#define THOROPT_TRACE_LIMIT           "traceLimit"              // Number of rows from TRACE activity                                            (default = 10)
 
 #define INITIAL_SELFJOIN_MATCH_WARNING_LEVEL 20000  // max of row matches before selfjoin emits warning