Merge pull request #7238 from ghalliday/issue13381

HPCC-13381 Add #option ('obfuscateOutput') to hide original ECL

Reviewed-By: Jamie Noss <james.noss@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -1999,6 +1999,7 @@ public:
     virtual void        setQueryMainDefinition(const char * str);
     virtual void        addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc);
     virtual void        removeAssociatedFiles();
+    virtual void        removeAssociatedFile(WUFileType type, const char * name, const char * desc);
 };
 
 class CLocalWUWebServicesInfo : public CInterface, implements IWUWebServicesInfo
@@ -7649,6 +7650,23 @@ void CLocalWUQuery::addAssociatedFile(WUFileType type, const char * name, const
     associated.append(*q);
 }
 
+void CLocalWUQuery::removeAssociatedFile(WUFileType type, const char * name, const char * desc)
+{
+    CriticalBlock block(crit);
+    associatedCached = false;
+    associated.kill();
+    StringBuffer xpath;
+    xpath.append("Associated/File");
+    if (type)
+        xpath.append("[@type=\"").append(getEnumText(type, queryFileTypes)).append("\"]");
+    if (name)
+        xpath.append("[@filename=\"").append(name).append("\"]");
+    if (desc)
+        xpath.append("[@desc=\"").append(desc).append("\"]");
+
+    p->removeProp(xpath.str());
+}
+
 void CLocalWUQuery::removeAssociatedFiles()
 {
     associatedCached = false;

common/workunit/workunit.hpp

@@ -407,6 +407,7 @@ interface IWUQuery : extends IConstWUQuery
     virtual void addAssociatedFile(WUFileType type, const char * name, const char * ip, const char * desc, unsigned crc) = 0;
     virtual void removeAssociatedFiles() = 0;
     virtual void setQueryMainDefinition(const char * str) = 0;
+    virtual void removeAssociatedFile(WUFileType type, const char * name, const char * desc) = 0;
 };
 
 

ecl/eclcc/eclcc.hpp

@@ -114,6 +114,7 @@ const char * const helpText[] = {
     "?!  -fmaxCompileThreads     Number of compiler instances to compile the c++",
     "?!  -fnoteRecordSizeInGraph Add estimates of record sizes to the graph",
     "?!  -fpickBestEngine        Allow simple thor queries to be passed to thor",
+    "?!  -fobfuscateOutput       Remove details of the original ECL from output",
     "?!  -freportCppWarnings     Report warnings from c++ compilation",
     "?!  -fsaveCpp -fsaveCppTempFiles  Retain the generated c++ files",
     "?!  -fshowActivitySizeInGraph  Show estimates of generated c++ size in the graph",

ecl/eclccserver/eclccserver.cpp

@@ -399,8 +399,11 @@ class EclccCompileThread : public CInterface, implements IPooledThread, implemen
                     SCMStringBuffer jobname;
                     if (embeddedWU->getJobName(jobname).length()) //let ECL win naming job during initial compile
                         workunit->setJobName(jobname.str());
-                    Owned<IWUQuery> query = workunit->updateQuery();
-                    query->setQueryText(eclQuery.s.str());
+                    if (!workunit->getDebugValueBool("obfuscateOutput", false))
+                    {
+                        Owned<IWUQuery> query = workunit->updateQuery();
+                        query->setQueryText(eclQuery.s.str());
+                    }
                 }
 
                 createUNCFilename(realdllfilename.str(), dllurl);

ecl/hqlcpp/hqlcpp.cpp

@@ -1757,6 +1757,8 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.newBalancedSpotter,"newBalancedSpotter",true),
         DebugOption(options.keyedJoinPreservesOrder,"keyedJoinPreservesOrder",true),
         DebugOption(options.expandSelectCreateRow,"expandSelectCreateRow",false),
+        DebugOption(options.obfuscateOutput,"obfuscateOutput",false),
+        DebugOption(options.showEclInGraph,"showEclInGraph",true),
         DebugOption(options.optimizeSortAllFields,"optimizeSortAllFields",true),
         DebugOption(options.optimizeSortAllFieldsStrict,"optimizeSortAllFieldsStrict",false),
     };
@@ -4364,7 +4366,7 @@ void HqlCppTranslator::createTempFor(BuildCtx & ctx, ITypeInfo * _exprType, CHql
 
 void HqlCppTranslator::buildTempExpr(BuildCtx & ctx, BuildCtx & declareCtx, CHqlBoundTarget & tempTarget, IHqlExpression * expr, ExpressionFormat format, bool ignoreSetAll)
 {
-    if (options.addLocationToCpp)
+    if (options.addLocationToCpp && !options.obfuscateOutput)
     {
         IHqlExpression * location = queryLocation(expr);
         if (location)

ecl/hqlcpp/hqlcpp.ipp

@@ -744,6 +744,8 @@ struct HqlCppOptions
     bool                newBalancedSpotter;
     bool                keyedJoinPreservesOrder;
     bool                expandSelectCreateRow;
+    bool                obfuscateOutput;
+    bool                showEclInGraph;
     bool                optimizeSortAllFields;
     bool                optimizeSortAllFieldsStrict;
 };

ecl/hqlcpp/hqlhtcpp.cpp

@@ -1835,7 +1835,7 @@ void ActivityInstance::addAttribute(const char * name, IHqlExpression * expr)
 
 void ActivityInstance::addLocationAttribute(IHqlExpression * location)
 {
-    if (!translator.queryOptions().reportLocations)
+    if (!translator.queryOptions().reportLocations || translator.queryOptions().obfuscateOutput)
         return;
 
     unsigned line = location->getStartLine();
@@ -1858,6 +1858,9 @@ void ActivityInstance::addLocationAttribute(IHqlExpression * location)
 
 void ActivityInstance::addNameAttribute(IHqlExpression * symbol)
 {
+    if (translator.queryOptions().obfuscateOutput)
+        return;
+
     //Not so sure about adding a location for a named symbol if there are other locations already present....
     //We should probably perform some deduping instead.
     addLocationAttribute(symbol);
@@ -1934,9 +1937,12 @@ void ActivityInstance::processHint(IHqlExpression * attr)
 
 void ActivityInstance::processSection(IHqlExpression * section)
 {
-    StringBuffer sectionName;
-    getStringValue(sectionName, section->queryChild(0));
-    addAttribute("section", sectionName);
+    if (!translator.queryOptions().obfuscateOutput)
+    {
+        StringBuffer sectionName;
+        getStringValue(sectionName, section->queryChild(0));
+        addAttribute("section", sectionName);
+    }
 }
 
 void ActivityInstance::processHints(IHqlExpression * hintAttr)
@@ -1974,18 +1980,24 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     IPropertyTree * parentGraphNode = subgraph ? subgraph->tree.get() : defaultSubGraph;
     if (!parentGraphNode)
         return;
+
+    HqlCppOptions const & options = translator.queryOptions();
     assertex(kind < TAKlast);
     graphNode.set(parentGraphNode->addPropTree("node", createPTree()));
 
     graphNode->setPropInt64("@id", activityId);
-    StringBuffer label;
-    if (isGrouped)
-        label.append("Grouped ");
-    else if (isLocal)
-        label.append("Local ");
-    label.append(getActivityText(kind));
 
-    graphNode->setProp("@label", graphLabel ? graphLabel.get() : label.str());
+    if (!options.obfuscateOutput)
+    {
+        StringBuffer label;
+        if (isGrouped)
+            label.append("Grouped ");
+        else if (isLocal)
+            label.append("Local ");
+        label.append(getActivityText(kind));
+
+        graphNode->setProp("@label", graphLabel ? graphLabel.get() : label.str());
+    }
 
     IHqlExpression * cur = dataset;
     loop
@@ -2027,82 +2039,88 @@ void ActivityInstance::createGraphNode(IPropertyTree * defaultSubGraph, bool alw
     if (isNoAccess)
         addAttributeBool("noAccess", true);
 
-    if (graphEclText.length() == 0)
-        toECL(dataset->queryBody(), graphEclText, false, true);
-
-    elideString(graphEclText, MAX_GRAPH_ECL_LENGTH);
-    if (strcmp(graphEclText.str(), "<>") != 0)
-        addAttribute("ecl", graphEclText.str());
-
-    if (translator.queryOptions().showSeqInGraph)
+    if (!options.obfuscateOutput)
     {
-        IHqlExpression * selSeq = querySelSeq(dataset);
-        if (selSeq)
-            addAttributeInt("selSeq", selSeq->querySequenceExtra());
-    }
+        if (graphEclText.length() == 0)
+            toECL(dataset->queryBody(), graphEclText, false, true);
 
+        elideString(graphEclText, MAX_GRAPH_ECL_LENGTH);
+        if (options.showEclInGraph)
+        {
+            if (strcmp(graphEclText.str(), "<>") != 0)
+                addAttribute("ecl", graphEclText.str());
+        }
 
-    if (translator.queryOptions().showMetaInGraph)
-    {
-        StringBuffer s;
-        if (translator.targetThor())
+        if (options.showSeqInGraph)
         {
-            IHqlExpression * distribution = queryDistribution(dataset);
-            if (distribution && distribution->queryName() != localAtom)
-                addAttribute("metaDistribution", getExprECL(distribution, s.clear(), true).str());
+            IHqlExpression * selSeq = querySelSeq(dataset);
+            if (selSeq)
+                addAttributeInt("selSeq", selSeq->querySequenceExtra());
         }
 
-        IHqlExpression * grouping = queryGrouping(dataset);
-        if (grouping)
-            addAttribute("metaGrouping", getExprECL(grouping, s.clear(), true).str());
 
-        if (translator.targetThor())
+        if (options.showMetaInGraph)
         {
-            IHqlExpression * globalSortOrder = queryGlobalSortOrder(dataset);
-            if (globalSortOrder)
-                addAttribute("metaGlobalSortOrder", getExprECL(globalSortOrder, s.clear(), true).str());
-        }
+            StringBuffer s;
+            if (translator.targetThor())
+            {
+                IHqlExpression * distribution = queryDistribution(dataset);
+                if (distribution && distribution->queryName() != localAtom)
+                    addAttribute("metaDistribution", getExprECL(distribution, s.clear(), true).str());
+            }
 
-        IHqlExpression * localSortOrder = queryLocalUngroupedSortOrder(dataset);
-        if (localSortOrder)
-            addAttribute("metaLocalSortOrder", getExprECL(localSortOrder, s.clear(), true).str());
+            IHqlExpression * grouping = queryGrouping(dataset);
+            if (grouping)
+                addAttribute("metaGrouping", getExprECL(grouping, s.clear(), true).str());
 
-        IHqlExpression * groupSortOrder = queryGroupSortOrder(dataset);
-        if (groupSortOrder)
-            addAttribute("metaGroupSortOrder", getExprECL(groupSortOrder, s.clear(), true).str());
-    }
+            if (translator.targetThor())
+            {
+                IHqlExpression * globalSortOrder = queryGlobalSortOrder(dataset);
+                if (globalSortOrder)
+                    addAttribute("metaGlobalSortOrder", getExprECL(globalSortOrder, s.clear(), true).str());
+            }
 
-    if (translator.queryOptions().noteRecordSizeInGraph)
-    {
-        IHqlExpression * record = dataset->queryRecord();
-        if (!record && (getNumChildTables(dataset) == 1))
-            record = dataset->queryChild(0)->queryRecord();
-        if (record)
+            IHqlExpression * localSortOrder = queryLocalUngroupedSortOrder(dataset);
+            if (localSortOrder)
+                addAttribute("metaLocalSortOrder", getExprECL(localSortOrder, s.clear(), true).str());
+
+            IHqlExpression * groupSortOrder = queryGroupSortOrder(dataset);
+            if (groupSortOrder)
+                addAttribute("metaGroupSortOrder", getExprECL(groupSortOrder, s.clear(), true).str());
+        }
+
+        if (options.noteRecordSizeInGraph)
         {
-            size32_t maxSize = getMaxRecordSize(record, translator.getDefaultMaxRecordSize());
-            if (isVariableSizeRecord(record))
+            IHqlExpression * record = dataset->queryRecord();
+            if (!record && (getNumChildTables(dataset) == 1))
+                record = dataset->queryChild(0)->queryRecord();
+            if (record)
             {
-                size32_t minSize = getMinRecordSize(record);
-                size32_t expectedSize = getExpectedRecordSize(record);
-                StringBuffer temp;
-                temp.append(minSize).append("..");
-                if (maxRecordSizeUsesDefault(record))
-                    temp.append("?");
+                size32_t maxSize = getMaxRecordSize(record, translator.getDefaultMaxRecordSize());
+                if (isVariableSizeRecord(record))
+                {
+                    size32_t minSize = getMinRecordSize(record);
+                    size32_t expectedSize = getExpectedRecordSize(record);
+                    StringBuffer temp;
+                    temp.append(minSize).append("..");
+                    if (maxRecordSizeUsesDefault(record))
+                        temp.append("?");
+                    else
+                        temp.append(maxSize);
+                    temp.append("(").append(expectedSize).append(")");
+                    addAttribute("recordSize", temp.str());
+                }
                 else
-                    temp.append(maxSize);
-                temp.append("(").append(expectedSize).append(")");
-                addAttribute("recordSize", temp.str());
+                    addAttributeInt("recordSize", maxSize);
             }
-            else
-                addAttributeInt("recordSize", maxSize);
         }
-    }
 
-    if (translator.queryOptions().showRecordCountInGraph && !dataset->isAction())
-    {
-        StringBuffer text;
-        getRecordCountText(text, dataset);
-        addAttribute("predictedCount", text);
+        if (options.showRecordCountInGraph && !dataset->isAction())
+        {
+            StringBuffer text;
+            getRecordCountText(text, dataset);
+            addAttribute("predictedCount", text);
+        }
     }
 
     processAnnotations(dataset);
@@ -2248,7 +2266,7 @@ void ActivityInstance::buildSuffix()
             if ((options.complexClassesActivityFilter == 0) || (kind == options.complexClassesActivityFilter))
                 translator.WARNING2(CategoryEfficiency, HQLWRN_ComplexHelperClass, activityId, approxSize);
         }
-        if (options.showActivitySizeInGraph)
+        if (!options.obfuscateOutput && options.showActivitySizeInGraph)
             addAttributeInt("approxClassSize", approxSize);
     }
 
@@ -5787,6 +5805,12 @@ bool HqlCppTranslator::buildCpp(IHqlCppInstance & _code, HqlQueryContext & query
         wu()->setCodeVersion(ACTIVITY_INTERFACE_VERSION,BUILD_TAG,LANGUAGE_VERSION);
         cacheOptions();
 
+        if (options.obfuscateOutput)
+        {
+            Owned<IWUQuery> query = wu()->updateQuery();
+            query->setQueryText(NULL);
+        }
+
         useLibrary(ECLRTL_LIB);
         useInclude("eclrtl.hpp");
 

ecl/hqlcpp/hqlnlp.cpp

@@ -736,7 +736,7 @@ ABoundActivity * HqlCppTranslator::doBuildActivityParse(BuildCtx & ctx, IHqlExpr
     wu()->setDebugValue("maxMemory", text.str(), true);
 #endif
 
-    if (options.debugNlp != 0)
+    if ((options.debugNlp != 0) && !options.obfuscateOutput)
     {
         BuildCtx subctx(instance->classctx);
         subctx.addQuotedLiteral("#if 0\nHuman readable form of the grammar");

ecl/hqlcpp/hqlsource.cpp

@@ -1991,7 +1991,7 @@ ABoundActivity * SourceBuilder::buildActivity(BuildCtx & ctx, IHqlExpression * e
 
     IHqlExpression * spillReason = tableExpr ? queryAttributeChild(tableExpr, _spillReason_Atom, 0) : NULL;
 
-    if (spillReason)
+    if (spillReason && !translator.queryOptions().obfuscateOutput)
     {
         StringBuffer text;
         getStringValue(text, spillReason);

testing/regress/ecl/aggidx1.ecl

@@ -24,6 +24,8 @@ useSequential := #IFDEFINED(root.useSequential, false);
 
 //--- end of version configuration ---
 
+#option ('obfuscateOutput', true);
+
 import $.setup;
 sq := setup.sq(multiPart);