Merge pull request #9006 from afishbeck/eclResultMonitoring

HPCC-16095 Result Monitoring ECL code generator

Reviewed-By: Rodrigo Pastrana <rodrigo.pastrana@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

esp/esdllib/esdl_def.cpp

@@ -1470,6 +1470,8 @@ IEsdlDefObjectIterator* EsdlDefinition::getDependencies( const char* service, St
         methodIter.setown(serviceDef->getMethods());
         for( methodIter->first(); methodIter->isValid(); methodIter->next() )
         {
+            if ((flags & DEPFLAG_ECL_ONLY) && methodIter->query().getPropInt("ecl_hide"))
+                continue;
             methodArray.append( methodIter->query() );
         }
     }
@@ -1479,20 +1481,18 @@ IEsdlDefObjectIterator* EsdlDefinition::getDependencies( const char* service, St
         {
             methodDef = serviceDef->queryMethodByName(methods.item(i));
 
-            if( NULL == methodDef )
-            {
+            if(!methodDef)
                 throw( MakeStringException(0, "ESDL Method Definition not found for %s in service %s", methods.item(i), service) );
-            }
-
+            if ((flags & DEPFLAG_ECL_ONLY) && methodDef->getPropInt("ecl_hide"))
+                continue;
             methodArray.append( *methodDef );
         }
     }
 
     this->gatherMethodDependencies( *dependencies, methodArray, requestedVer, opts, flags );
 
-    // Put the highest level objects on last- the services
-    // Tony did say that these will be useful
-    if( serviceDef != NULL )
+    bool allTypes = !(flags & DEPFLAG_INCLUDE_TYPES); //not asking for any explicit types indicates all types
+    if(serviceDef && (allTypes || (flags & DEPFLAG_INCLUDE_SERVICE)))
     {
         EsdlDefObjectWrapper* wrapper = new EsdlDefObjectWrapper( *serviceDef );
         dependencies->append( *wrapper );
@@ -1505,13 +1505,8 @@ void EsdlDefinition::gatherMethodDependencies( EsdlDefObjectWrapperArray& depend
 {
     AddedObjs foundByName;
 
-    // At this point all the Methods we want to walk are in this array-
-    // either explicitly passed in to the array, or added by walking the
-    // service definitions passed in and adding all of the services' methods
+    bool allTypes = !(flags & DEPFLAG_INCLUDE_TYPES); //not asking for any explicit types indicates all types
 
-    // Q:
-    // What circumstances do we want to throw an unhandled exception
-    // vs. those that we may just want to log or DBGLOG a warning?
     int numMethods = methods.ordinality();
     for( int i = 0; i<numMethods; i++ )
     {
@@ -1520,26 +1515,29 @@ void EsdlDefinition::gatherMethodDependencies( EsdlDefObjectWrapperArray& depend
 
         if( methodObj->checkOptional(opts) && (requestedVer==0.0 || method->checkVersion(requestedVer)) )
         {
-            const char* request = method->queryRequestType();
-            IEsdlDefObject* requestObj = this->queryObj( request );
-            if( NULL == requestObj )
+            if (allTypes || (flags & DEPFLAG_INCLUDE_REQUEST))
             {
-                throw( MakeStringException(0, "Request struct %s not found in ESDL Definition", request) );
-            } else {
-                this->walkDefinitionDepthFirst( foundByName, dependencies, requestObj, requestedVer, opts, 0, flags );
+                const char* request = method->queryRequestType();
+                IEsdlDefObject* requestObj = this->queryObj( request );
+                if(!requestObj)
+                    throw( MakeStringException(0, "Request struct %s not found in ESDL Definition", request) );
+                walkDefinitionDepthFirst( foundByName, dependencies, requestObj, requestedVer, opts, 0, flags );
             }
 
-            const char* response = method->queryResponseType();
-            IEsdlDefObject* responseObj = this->queryObj( response );
-            if( NULL == responseObj )
+            if (allTypes || (flags & DEPFLAG_INCLUDE_RESPONSE))
             {
-                throw( MakeStringException(0, "Response struct %s not found in ESDL Definition", response) );
-            } else {
-                this->walkDefinitionDepthFirst( foundByName, dependencies, responseObj, requestedVer, opts, 0, flags );
+                const char* response = method->queryResponseType();
+                IEsdlDefObject* responseObj = this->queryObj( response );
+                if(!responseObj)
+                    throw( MakeStringException(0, "Response struct %s not found in ESDL Definition", response) );
+                walkDefinitionDepthFirst( foundByName, dependencies, responseObj, requestedVer, opts, 0, flags );
             }
 
-            EsdlDefObjectWrapper* wrapper = new EsdlDefObjectWrapper( *methodObj );
-            dependencies.append( *wrapper );
+            if (allTypes || (flags & DEPFLAG_INCLUDE_METHOD))
+            {
+                EsdlDefObjectWrapper* wrapper = new EsdlDefObjectWrapper( *methodObj );
+                dependencies.append( *wrapper );
+            }
         }
     }
 }
@@ -1563,6 +1561,11 @@ unsigned EsdlDefinition::walkChildrenDepthFirst( AddedObjs& foundByName, EsdlDef
             while( children->isValid() )
             {
                 IEsdlDefObject& child = children->query();
+                if ((flags & DEPFLAG_ECL_ONLY) && child.getPropInt("ecl_hide"))
+                {
+                    children->next();
+                    continue;
+                }
                 const char *childname = child.queryName();
                 EsdlDefTypeId childType = child.getEsdlType();
                 //DBGLOG("  %s<%s> child", StringBuffer(level*2, " ").str(), childname );
@@ -1593,7 +1596,7 @@ unsigned EsdlDefinition::walkChildrenDepthFirst( AddedObjs& foundByName, EsdlDef
                     }
 
                     // Should this element be included based on ESP version & optional tags?
-                    if( child.checkOptional(opts) && ( requestedVer==0.0 || child.checkVersion(requestedVer)) )
+                    if( child.checkOptional(opts) && ( requestedVer==0.0 || child.checkVersion(requestedVer)))
                     {
                         if( complexType != NULL )
                         {

esp/esdllib/esdl_def.hpp

@@ -72,8 +72,14 @@ typedef enum EsdlDefTypeId_
 
 #define DEPFLAG_COLLAPSE    (0x01)
 #define DEPFLAG_ARRAYOF     (0x02)
-#define DEPFLAG_STRINGARRAY (0x04)    // Set dynamically by gatherDependencies to indicate stylesheet should output
-                                    // an EspStringArray structure definition.
+#define DEPFLAG_STRINGARRAY (0x04)  //Set dynamically by gatherDependencies to indicate stylesheet should output an EspStringArray structure definition.
+#define DEPFLAG_ECL_ONLY    (0x08)  //ignore anything tagged with ecl_hide
+
+#define DEPFLAG_INCLUDE_REQUEST     (0x10)
+#define DEPFLAG_INCLUDE_RESPONSE    (0x20)
+#define DEPFLAG_INCLUDE_METHOD      (0x40)
+#define DEPFLAG_INCLUDE_SERVICE     (0x80)
+#define DEPFLAG_INCLUDE_TYPES       (DEPFLAG_INCLUDE_REQUEST | DEPFLAG_INCLUDE_RESPONSE | DEPFLAG_INCLUDE_METHOD | DEPFLAG_INCLUDE_SERVICE)
 
 interface IEsdlDefObject : extends IInterface
 {

esp/esdllib/esdl_def_helper.cpp

@@ -113,6 +113,47 @@ void EsdlDefinitionHelper::setTransformParams( EsdlXslTypeId xslId, IProperties
     parameters.setValue( xslId, params );
 }
 
+void removeEclHiddenStructs(IPropertyTree &depTree)
+{
+    Owned<IPropertyTreeIterator> it = depTree.getElements("*[@ecl_hide='1']");
+    ForEach(*it)
+        depTree.removeTree(&it->query());
+}
+void removeEclHiddenElements(IPropertyTree &depTree)
+{
+    Owned<IPropertyTreeIterator> it = depTree.getElements("*");
+    ForEach(*it)
+    {
+        StringArray names;
+        Owned<IPropertyTreeIterator> elements = it->query().getElements("*[@get_data_from]");
+        ForEach(*elements)
+            names.appendUniq(elements->query().queryProp("@name"));
+        elements.setown(it->query().getElements("*[@ecl_hide='1']"));
+        ForEach(*elements)
+            names.appendUniq(elements->query().queryProp("@name"));
+
+        ForEachItemIn(i, names)
+        {
+            VStringBuffer xpath("*[@name='%s']", names.item(i));
+            it->query().removeProp(xpath);
+        }
+    }
+}
+esdl_decl void removeEclHidden(IPropertyTree *depTree)
+{
+    if (!depTree)
+        return;
+    removeEclHiddenStructs(*depTree);
+    removeEclHiddenElements(*depTree);
+}
+
+esdl_decl void removeEclHidden(StringBuffer &xml)
+{
+    Owned<IPropertyTree> depTree = createPTreeFromXMLString(xml);
+    removeEclHidden(depTree);
+    toXML(depTree, xml.clear());
+}
+
 void EsdlDefinitionHelper::toXML( IEsdlDefObjectIterator& objs, StringBuffer &xml, double version, IProperties *opts, unsigned requestedFlags )
 {
     TimeSection ts("serializing EsdlObjects to XML");
@@ -166,7 +207,7 @@ void EsdlDefinitionHelper::toXML( IEsdlDefObjectIterator& objs, StringBuffer &xm
         }
 
         // Add the serialization of the current structure to the xml output StringBuffer
-        xml.append( curObjXml );
+        xml.append( curObjXml ).append('\n');
     }
 
     return;
@@ -189,6 +230,11 @@ void EsdlDefinitionHelper::toXSD( IEsdlDefObjectIterator &objs, StringBuffer &xs
         this->toXML( objs, xml, version, opts, flags );
         xml.append("</esxdl>");
 
+        if (flags & DEPFLAG_ECL_ONLY)
+        {
+            removeEclHidden(xml);
+        }
+
         xmlLen = xml.length();
         trans->setXmlSource( xml.str(), xmlLen );
         trans->transform(xsd);

esp/esdllib/esdl_def_helper.hpp

@@ -47,6 +47,8 @@ interface IEsdlDefinitionHelper : extends IInterface
     virtual void toJavaService( IEsdlDefObjectIterator& objs, StringBuffer &content, EsdlXslTypeId implType, IProperties *opts=NULL, unsigned flags=0 )=0;
 };
 
+esdl_decl void removeEclHidden(IPropertyTree *depTree);
+esdl_decl void removeEclHidden(StringBuffer &xml);
 
 esdl_decl IEsdlDefinitionHelper* createEsdlDefinitionHelper( );
 

esp/xslt/CMakeLists.txt

@@ -41,6 +41,7 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/wsecl3_result.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/passwordupdate.xsl
     ${CMAKE_CURRENT_SOURCE_DIR}/esdl2ecl.xslt
+    ${CMAKE_CURRENT_SOURCE_DIR}/esdl2monitor.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/esxdl2xsd.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/esdl2java_srvbase.xslt
     ${CMAKE_CURRENT_SOURCE_DIR}/esdl2java_srvdummy.xslt

system/xmllib/xsdparser.cpp

@@ -586,7 +586,7 @@ public:
     virtual ~CXmlSchema();
 
     IXmlType* queryElementType(const char* name);
-    IXmlType* queryTypeByName(const char* typeName) { return queryTypeByName(typeName); }
+    IXmlType* queryTypeByName(const char* typeName) { return queryTypeByName(typeName, nullptr); }
 };
 
 XMLLIB_API IXmlSchema* createXmlSchemaFromFile(const char* file)

tools/esdlcmd-xml/esdl2xml.hpp

@@ -54,12 +54,11 @@ public:
                     fprintf(stdout, "Output directory not specified\n");
             }
 
+            //prevent recursive adding of this same content
+            added.setValue(source, true);
+
             ESDLcompiler hc(source, out==NULL, outdir, outputIncludes, includedESDL);
             hc.Process();
-            if (out != NULL)
-                out->append(hc.getEsxdlContent());
-
-            added.setValue(source, true);
 
             if (optRecursive && hc.includes)
             {
@@ -73,6 +72,10 @@ public:
                    transform(subfile, outdir, out, outputIncludes, true);
                 }
             }
+
+            if (out != NULL) //includes must come before body to handle ESDLDefVersions correctly
+                out->append(hc.getEsxdlContent());
+
             if (optVerbose)
                 fprintf(stdout, "Finished processing ESDL definition: %s\n", source);
         }

tools/esdlcmd/CMakeLists.txt

@@ -34,6 +34,7 @@ set ( SRCS
          ${CMAKE_CURRENT_SOURCE_DIR}/esdlcmd_core.cpp
          ${CMAKE_CURRENT_SOURCE_DIR}/esdlcmd_common.cpp
          ${CMAKE_CURRENT_SOURCE_DIR}/esdl2ecl.cpp
+         ${CMAKE_CURRENT_SOURCE_DIR}/esdlcmd_monitor.cpp
          ${CMAKE_CURRENT_SOURCE_DIR}/esdl-publish.cpp
          ${ESPSCM_GENERATED_DIR}/ws_esdlconfig_esp.cpp
     )

tools/esdlcmd/esdlcmd_common.cpp

@@ -36,6 +36,36 @@ void outputMultiExceptions(const IMultiException &me)
     }
     fprintf(stderr, "\n");
 }
+static inline void appendFileName(StringBuffer &path, const char *filename)
+{
+    if (!filename)
+        return;
+    while (*filename==PATHSEPCHAR)
+        filename++;
+    if (!*filename)
+        return;
+    if (path.length() && path.charAt(path.length()-1) != PATHSEPCHAR)
+        path.append(PATHSEPCHAR);
+    path.append(filename);
+}
+void saveAsFile(const char * filepath, const char *filename, const char *text, const char *ext)
+{
+    StringBuffer path(filepath);
+
+    appendFileName(path, filename);
+    if(ext && *ext)
+        path.append(ext);
+
+    Owned<IFile> file = createIFile(path.str());
+    Owned<IFileIO> io = file->open(IFOcreaterw);
+
+    DBGLOG("Writing to file %s", file->queryFilename());
+
+    if (io.get())
+        io->write(0, strlen(text), text);
+    else
+        DBGLOG("File %s can't be created", file->queryFilename());
+}
 
 //=========================================================================================
 

tools/esdlcmd/esdlcmd_common.hpp

@@ -136,6 +136,7 @@ public:
     Owned<IEsdlDefinition> esdlDef;
     Owned<IEsdlDefinitionHelper> defHelper;
     Owned<IFile> serviceDefFile;
+    bool verbose = false;
 
 public:
     EsdlCmdHelper()
@@ -164,7 +165,7 @@ public:
             if (stricmp(extension.str(),LEGACY_FILE_EXTENSION)==0 || stricmp(extension.str(),ESDL_FILE_EXTENSION)==0)
             {
                 StringBuffer esxml;
-                EsdlCmdHelper::convertECMtoESXDL(sourceFileName, filename.str(), esxml, true, true, false, true);
+                EsdlCmdHelper::convertECMtoESXDL(sourceFileName, filename.str(), esxml, true, verbose, false, true);
                 VStringBuffer serviceid("%s.%f", serviceName, version);
                 esdlDef->addDefinitionFromXML(esxml, serviceid.str());
             }
@@ -326,11 +327,10 @@ static bool getCurrentFolder(StringBuffer & path)
 static bool getComponentFilesRelPathFromBin(StringBuffer & path)
 {
     if (getCurrentFolder(path))
-    {
-        path.appendf("%c%s%c%s", PATHSEPCHAR, HIGHER_DIR_RELATIVE,PATHSEPCHAR,COMPONENTS_DIR_NAME);
-        return true;
-    }
-
+        return checkDirExists(path.appendf("%c%s%c%s", PATHSEPCHAR, HIGHER_DIR_RELATIVE,PATHSEPCHAR,COMPONENTS_DIR_NAME));
     return false;
 }
+
+void saveAsFile(const char * path, const char *filename, const char *text, const char *ext="");
+
 #endif

tools/esdlcmd/esdlcmd_core.cpp

@@ -189,6 +189,7 @@ public:
         {
             optTargetNamespace.set(DEFAULT_NAMESPACE_BASE);
         }
+        cmdHelper.verbose = optVerbose;
 
         return true;
     }
@@ -720,6 +721,7 @@ public:
             else
                 optXsltPath.set(temp.set(COMPONENTFILES_DIR).append("/xslt/"));
         }
+        cmdHelper.verbose = optVerbose;
         return true;
     }
 
@@ -877,6 +879,10 @@ IEsdlCommand *createCoreEsdlCommand(const char *cmdname)
         return new EsdlListESDLDefCmd();
     if (strieq(cmdname, "LIST-BINDINGS"))
         return new EsdlListESDLBindingsCmd();
+    if (strieq(cmdname, "MONITOR"))
+        return createEsdlMonitorCommand(cmdname);
+    if (strieq(cmdname, "MONITOR-TEMPLATE"))
+        return createEsdlMonitorCommand(cmdname);
 
     return NULL;
 }

tools/esdlcmd/esdlcmd_core.hpp

@@ -21,5 +21,7 @@
 #include "esdlcmd_common.hpp"
 
 IEsdlCommand *createCoreEsdlCommand(const char *cmdname);
+IEsdlCommand *createEsdlMonitorCommand(const char *cmdname);
+
 
 #endif

tools/esdlcmd/esdlcmd_shell.cpp

@@ -195,6 +195,8 @@ void EsdlCMDShell::usage()
            "   bind-method       Configure method associated with existing ESDL binding.\n"
            "   unbind-method     Remove method associated with existing ESDL binding.\n"
            "   get-binding       Get ESDL binding.\n"
+           "   monitor           Generate ECL code for result monitoring / differencing\n"
+           "   monitor-template  Generate a template for use with 'monitor' command\n"
            ""
            "\nRun 'esdl help <command>' for more information on a specific command\n\n"
     );

tools/esdlcomp/esdlcomp.h

@@ -533,14 +533,22 @@ public:
         const char *xsd_type = getMetaString("xsd_type", NULL);
         if (xsd_type)
         {
+            const char *attr = "complex_type";
             if (*xsd_type=='\"')
                 xsd_type++;
             const char *finger = strchr(xsd_type, ':');
             if (finger)
+            {
                 xsd_type=finger+1;
+                if (!strncmp("ArrayOf", xsd_type, 7))
+                {
+                    attr = "type";
+                    xsd_type += 7;
+                }
+            }
             StringBuffer TypeName(xsd_type);
             TypeName.replace('\"', 0);
-            out.appendf(" complex_type='%s'", TypeName.str());
+            out.appendf(" %s='%s'", attr, TypeName.str());
         }
         else
         {
@@ -553,7 +561,21 @@ public:
 
     void toString(StringBuffer & out)
     {
-        if (flags & PF_TEMPLATE && !strcmp(templ, "ESParray"))
+        const char *xsd_type = getMetaString("xsd_type", NULL);
+        if (xsd_type && *xsd_type=='\"')
+           xsd_type++;
+        unsigned pcl = strlen("tns:ArrayOf");
+        //purely for compatability with scapps ESDL processing... ESDL should never have relied on interpreting the xsd_type which is for external use
+        if (xsd_type && !strncmp("tns:ArrayOf", xsd_type, pcl))
+        {
+            out.appendf("\t\t<EsdlArray name='%s' ", name);
+            toStringXmlAttr(out);
+            for (MetaTagInfo *mtag=tags; mtag; mtag=mtag->next)
+            {
+                mtag->toStringXmlAttr(out);
+            }
+        }
+        else if (flags & PF_TEMPLATE && !strcmp(templ, "ESParray"))
         {
             out.appendf("\t\t<EsdlArray name='%s' ", name);
             toStringXmlAttr(out);