HPCC-19348 Implement consistent source/target for edges

Signed-off-by: Gavin Halliday <gavin.halliday@lexisnexis.com>

common/workunit/workunit.cpp

@@ -852,17 +852,17 @@ protected:
         }
     }
 
-    virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const
+    virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
     {
         return collections.tos().getStatistic(kind, value);
     }
 
-    virtual const char * queryAttribute(WuAttr attr) const
+    virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
     {
         return nullptr;
     }
 
-    virtual const char * queryHint(const char * kind) const
+    virtual const char * queryHint(const char * kind) const override
     {
         return nullptr;
     }
@@ -1122,7 +1122,7 @@ public:
         }
     }
 
-    virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const
+    virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
     {
         for (unsigned i=0; i < numStatistics; i++)
         {
@@ -1136,7 +1136,7 @@ public:
         return false;
     }
 
-    virtual const char * queryAttribute(WuAttr attr) const
+    virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
     {
         return nullptr;
     }
@@ -1321,8 +1321,8 @@ public:
                 //MORE This will eventually need to walk the attributes and map the names.
                 //Need to be careful if they need to be mapped differently depending on the context.
                 playAttribute(visitor, WALabel);
-                playAttribute(visitor, WASource);
-                playAttribute(visitor, WATarget);
+                playAttribute(visitor, WAIdSource);
+                playAttribute(visitor, WAIdTarget);
                 playAttribute(visitor, WASourceIndex);
                 playAttribute(visitor, WATargetIndex);
                 playAttribute(visitor, WAIsDependency);
@@ -1331,20 +1331,20 @@ public:
         }
     }
 
-    virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const
+    virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const override
     {
         return false;
     }
 
-    virtual const char * queryAttribute(WuAttr attr) const
+    virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
     {
         if (!treeIters.ordinality())
             return nullptr;
         //MORE - check that the attribute is value for the current scope type (to prevent defaults being returned)
-        return queryAttributeValue(treeIters.tos().query(), attr);
+        return queryAttributeValue(treeIters.tos().query(), attr, scratchpad);
     }
 
-    virtual const char * queryHint(const char * kind) const
+    virtual const char * queryHint(const char * kind) const override
     {
         //MORE: Needs to be implemented!
         return nullptr;
@@ -1353,7 +1353,8 @@ public:
 private:
     void playAttribute(IWuScopeVisitor & visitor, WuAttr kind)
     {
-        const char * value = queryAttributeValue(treeIters.tos().query(), kind);
+        StringBuffer scratchpad;
+        const char * value = queryAttributeValue(treeIters.tos().query(), kind, scratchpad);
         if (value)
             visitor.noteAttribute(kind, value);
     }
@@ -1829,13 +1830,13 @@ public:
         return false;
     }
 
-    virtual const char * queryAttribute(WuAttr attr) const override
+    virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const override
     {
         ForEachItemIn(i, iters)
         {
             if (iterMatchesCurrentScope(i))
             {
-                const char * value = iters.item(i).queryAttribute(attr);
+                const char * value = iters.item(i).queryAttribute(attr, scratchpad);
                 if (value)
                     return value;
             }

common/workunit/workunit.hpp

@@ -1131,7 +1131,7 @@ interface IConstWUScopeIterator : extends IScmIterator
 
     //Return true if the stat is present, if found and update the value - queryStat() wrapper is generally easier to use.
     virtual bool getStat(StatisticKind kind, unsigned __int64 & value) const = 0;
-    virtual const char * queryAttribute(WuAttr attr) const = 0; // Multiple values can be processed via the playStatistics() function
+    virtual const char * queryAttribute(WuAttr attr, StringBuffer & scratchpad) const = 0; // Multiple values can be processed via the playStatistics() function
     virtual const char * queryHint(const char * kind) const = 0;
 
     inline unsigned __int64 queryStat(StatisticKind kind, unsigned __int64 defaultValue = 0) const

common/workunit/wuattr.cpp

@@ -21,33 +21,41 @@
 struct WuAttrInfo
 {
 public:
-    WuAttr kind;
-    StatisticMeasure measure;
-    const char * name;
-    const char * graphPath;
-    const char * childPath;
-    const char * dft;
+    WuAttr kind;                // The attribute enumeration
+    StatisticMeasure measure;   // units for the measure
+    const char * name;          // text version of the attribute
+    const char * graphPath;     // The xpath required to extract a result from a graph node.
+    const char * overridePath;  // Alternative xpath to check 1st for some overloaded attributes
+    const char * childPath;     // The name of the <atr> for setting, or matching when iterating
+    const char * dft;           // default value if not present
 };
 
 #define CHILDPATH(x) "att[@name='" x "']/@value"
-#define ATTR(kind, measure, path)           { WA ## kind, measure, #kind, path, nullptr, nullptr }
-#define CHILD(kind, measure, path)    { WA ## kind, measure, #kind, CHILDPATH(path), path, nullptr }
-#define CHILD_D(kind, measure, path, dft)    { WA ## kind, measure, #kind, CHILDPATH(path), path, dft }
+#define CHILDMPATH(x) "att[@name='" x "'][1]/@value"
+#define ATTR(kind, measure, path)           { WA ## kind, measure, #kind, path, nullptr, nullptr, nullptr }
+#define ALTATTR(kind, measure, path, alt)   { WA ## kind, measure, #kind, path, alt, nullptr }
+#define CHILD(kind, measure, path)          { WA ## kind, measure, #kind, CHILDPATH(path), nullptr, path, nullptr }
+#define CHILD_MULTI(kind, measure, path)    { WA ## kind, measure, #kind, CHILDMPATH(path), nullptr, path, nullptr }
+#define CHILD_D(kind, measure, path, dft)   { WA ## kind, measure, #kind, CHILDPATH(path), nullptr, path, dft }
 
 
 const static WuAttrInfo attrInfo[] = {
     { WANone, SMeasureNone, "none", nullptr, nullptr, nullptr },
     { WAAll, SMeasureNone, "all", nullptr, nullptr, nullptr },
     CHILD(Kind, SMeasureEnum, "_kind"),
-    ATTR(Source, SMeasureText, "@source"),
-    ATTR(Target, SMeasureText, "@target"),
+    ALTATTR(IdSource, SMeasureId, "@source", "att[@name='_sourceActivity']/@value"),
+    ALTATTR(IdTarget, SMeasureId, "@target", "att[@name='_targetActivity']/@value"),
     CHILD_D(SourceIndex, SMeasureText, "_sourceIndex", "0"),
     CHILD_D(TargetIndex, SMeasureText, "_targetIndex", "0"),
     ATTR(Label, SMeasureText, "@label"),
     CHILD(IsDependency, SMeasureBool, "_dependsOn"),
     CHILD(IsChildGraph, SMeasureBool, "_childGraph"),
-    CHILD(Definition, SMeasureText, "definition"),
-    CHILD(EclName, SMeasureText, "name"),
+    CHILD_MULTI(Definition, SMeasureText, "definition"),
+    CHILD_MULTI(EclName, SMeasureText, "name"),
+    CHILD(EclText, SMeasureText, "ecl"),
+    CHILD(RecordSize, SMeasureText, "recordSize"),
+    CHILD(PredictedCount, SMeasureText, "predictedCount"),
+    CHILD(Filename, SMeasureText, "_fileName"),
     { WAMax, SMeasureNone, nullptr, nullptr, nullptr, nullptr }
 };
 
@@ -82,16 +90,30 @@ WuAttr queryWuAttribute(const char * kind, WuAttr dft)
     return dft;
 }
 
-extern WORKUNIT_API const char * queryAttributeValue(IPropertyTree & src, WuAttr kind)
+extern WORKUNIT_API const char * queryAttributeValue(IPropertyTree & src, WuAttr kind, StringBuffer & scratchpad)
 {
     if ((kind <= WANone) || (kind >= WAMax))
         return nullptr;
 
     const WuAttrInfo & info = attrInfo[kind-WANone];
     const char * path = info.graphPath;
-    const char * value = src.queryProp(path);
+    const char * altpath = info.overridePath;
+    const char * value = altpath ? src.queryProp(altpath) : nullptr;
+    if (!value)
+        value = src.queryProp(path);
     if (!value && info.dft)
         value = info.dft;
+
+    //The following switch statement allows the value returned to be transformed from the value stored.
+    switch (kind)
+    {
+    case WAIdSource:
+    case WAIdTarget:
+        //A bit of a hack - source and target for edges are activity ids.  Return a computed string.
+        value = scratchpad.clear().append(ActivityScopePrefix).append(value).str();
+        break;
+    }
+
     return value;
 }
 

common/workunit/wuattr.hpp

@@ -33,8 +33,8 @@ enum WuAttr : unsigned
     WANone = 0x80000000,
     WAAll,
     WAKind,
-    WASource,
-    WATarget,
+    WAIdSource,
+    WAIdTarget,
     WASourceIndex,
     WATargetIndex,
     WALabel,
@@ -42,12 +42,16 @@ enum WuAttr : unsigned
     WAIsChildGraph,
     WADefinition,
     WAEclName,
+    WAEclText,
+    WARecordSize,
+    WAPredictedCount,
+    WAFilename,
     WAMax
 };
 
 extern WORKUNIT_API const char * queryWuAttributeName(WuAttr kind);
 extern WORKUNIT_API WuAttr queryWuAttribute(const char * kind, WuAttr dft);
-extern WORKUNIT_API const char * queryAttributeValue(IPropertyTree & src, WuAttr kind);
+extern WORKUNIT_API const char * queryAttributeValue(IPropertyTree & src, WuAttr kind, StringBuffer & scratchpad); // may return pointer to scratchpad if necessary
 extern WORKUNIT_API WuAttr queryGraphAttrToWuAttr(const char * name);
 extern WORKUNIT_API WuAttr queryGraphChildAttToWuAttr(const char * name);
 

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1861,6 +1861,11 @@ bool ActivityInstance::isExternal()
     return !isMember && !instanceIsLocal;
 }
 
+void ActivityInstance::addAttribute(WuAttr attr, const char * value)
+{
+    setAttributeValue(*graphNode, attr, value);
+}
+
 void ActivityInstance::addAttribute(const char * name, const char * value)
 {
     addGraphAttribute(graphNode, name, value);
@@ -1914,7 +1919,7 @@ void ActivityInstance::addLocationAttribute(IHqlExpression * location)
     if (column)
         s.append(",").append(column);
     s.append(")");
-    addAttribute("definition", s.str());
+    addAttribute(WADefinition, s.str());
 }
 
 

ecl/hqlcpp/hqlhtcpp.ipp

@@ -148,6 +148,7 @@ public:
     void buildMetaMember();
     void getScope(StringBuffer & scope) const; // return the full scope name of this activity
 
+    void addAttribute(WuAttr attr, const char * value);
     void addAttribute(const char * name, const char * value);
     void addAttribute(const char * name, IHqlExpression * expr);
     void addAttributeInt(const char * name, __int64 value);

system/jlib/jstatcodes.h

@@ -91,6 +91,8 @@ enum StatisticMeasure
     SMeasureEnum,                       // A value from an enumeration
     SMeasureText,                       // A textual value (from a graph attribute rather than a statistic)
     SMeasureBool,                       // A boolean
+    SMeasureId,                         // An Id for an element
+    SMeasureFilename,                   // A filename
     SMeasureMax,
 };
 

system/jlib/jstats.cpp

@@ -66,7 +66,7 @@ void setStatisticsComponentName(StatisticCreatorType processType, const char * p
 //--------------------------------------------------------------------------------------------------------------------
 
 // Textual forms of the different enumerations, first items are for none and all.
-static constexpr const char * const measureNames[] = { "", "all", "ns", "ts", "cnt", "sz", "cpu", "skw", "node", "ppm", "ip", "cy", "en", "txt", "bool", NULL };
+static constexpr const char * const measureNames[] = { "", "all", "ns", "ts", "cnt", "sz", "cpu", "skw", "node", "ppm", "ip", "cy", "en", "txt", "bool", "id", "fname", NULL };
 static constexpr const char * const creatorTypeNames[]= { "", "all", "unknown", "hthor", "roxie", "roxie:s", "thor", "thor:m", "thor:s", "eclcc", "esp", "summary", NULL };
 static constexpr const char * const scopeTypeNames[] = { "", "all", "global", "graph", "subgraph", "activity", "allocator", "section", "compile", "dfu", "edge", "function", "workflow", "child", "unknown", nullptr };
 
@@ -413,6 +413,10 @@ StringBuffer & formatStatistic(StringBuffer & out, unsigned __int64 value, Stati
         return out.append(value);
     case SMeasureBool:
         return out.append(boolToStr(value != 0));
+    case SMeasureText:
+    case SMeasureId:
+    case SMeasureFilename:
+        return out.append(value);
     default:
         return out.append(value).append('?');
     }
@@ -592,6 +596,8 @@ const char * queryMeasurePrefix(StatisticMeasure measure)
     case SMeasureEnum:          return "";
     case SMeasureText:          return "";
     case SMeasureBool:          return "Is";
+    case SMeasureId:            return "Id";
+    case SMeasureFilename:      return "";
     default:
         return "Unknown";
     }
@@ -640,6 +646,8 @@ StatsMergeAction queryMergeMode(StatisticMeasure measure)
     case SMeasureEnum:          return StatsMergeKeepNonZero;
     case SMeasureText:          return StatsMergeKeepNonZero;
     case SMeasureBool:          return StatsMergeKeepNonZero;
+    case SMeasureId:            return StatsMergeKeepNonZero;
+    case SMeasureFilename:      return StatsMergeKeepNonZero;
     default:
 #ifdef _DEBUG
         throwUnexpected();