Merge branch 'closedown-5.0.x' into candidate-5.0.2

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

esp/scm/ws_topology.ecm

@@ -137,6 +137,13 @@ ESPStruct TpBinding
     string Protocol;
 
 };
+
+ESPStruct TpEspServicePlugin
+{
+    string Name;
+    string Type;
+};
+
 //  ===========================================================================
 ESPStruct TpEspServer
 {
@@ -565,6 +572,15 @@ ESPresponse [exceptions_inline,encode(0)] TpThorStatusResponse
     int AutoRefresh;
 };
 
+ESPrequest TpGetServicePluginsRequest
+{
+};
+
+ESPresponse [exceptions_inline,encode(0)] TpGetServicePluginsResponse
+{
+    ESParray<ESPstruct TpEspServicePlugin, Plugin> Plugins;
+};
+
 ESPservice [noforms, version("1.20"), default_client_version("1.20"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsTopology
 {
     ESPuses ESPStruct TpBinding;
@@ -603,6 +619,7 @@ ESPservice [noforms, version("1.20"), default_client_version("1.20"), exceptions
     ESPmethod [resp_xsl_default("/esp/xslt/tplog.xslt")] TpLogFile(TpLogFileRequest, TpLogFileResponse);
     ESPmethod [resp_xsl_default("/esp/xslt/tplogdisplay.xslt")] TpLogFileDisplay(TpLogFileRequest, TpLogFileResponse);
     ESPmethod TpGetComponentFile(TpGetComponentFileRequest, TpGetComponentFileResponse);
+    ESPmethod TpGetServicePlugins(TpGetServicePluginsRequest, TpGetServicePluginsResponse);
     ESPmethod TpListTargetClusters(TpListTargetClustersRequest, TpListTargetClustersResponse);
 
     ESPmethod SystemLog(SystemLogRequest, SystemLogResponse);

esp/services/ws_smc/ws_smcService.cpp

@@ -541,6 +541,7 @@ void CWsSMCEx::readDFUWUs(IEspContext &context, const char* queueName, const cha
 {
     StringAttrArray wulist;
     unsigned running = queuedJobs(queueName, wulist);
+    Owned<IDFUWorkUnitFactory> factory = getDFUWorkUnitFactory();
     ForEachItemIn(i, wulist)
     {
         StringBuffer jname, uname, state, error;
@@ -552,7 +553,7 @@ void CWsSMCEx::readDFUWUs(IEspContext &context, const char* queueName, const cha
 
         try
         {
-            Owned<IConstDFUWorkUnit> dfuwu = getDFUWorkUnitFactory()->openWorkUnit(wuid, false);
+            Owned<IConstDFUWorkUnit> dfuwu = factory->openWorkUnit(wuid, false);
             dfuwu->getUser(uname);
             dfuwu->getJobName(jname);
         }

esp/services/ws_topology/ws_topologyService.cpp

@@ -83,6 +83,23 @@ void CWsTopologyEx::init(IPropertyTree *cfg, const char *process, const char *se
             defaultTargetClusterPrefix.set(cfg->queryProp(xpath.str()));
     }
 
+    xpath.clear().appendf("Software/EspProcess[@name=\"%s\"]/EspService[@servicePlugin]", process);
+    Owned<IPropertyTreeIterator> it= cfg->getElements(xpath.str());
+    ForEach(*it)
+    {
+        IPropertyTree& espService = it->query();
+        const char* servicePlugin = espService.queryProp("@servicePlugin");
+        const char* servicePluginType = espService.queryProp("@servicePluginType");
+        if (!servicePlugin || !*servicePlugin)
+            continue;
+        Owned<IEspTpEspServicePlugin> espServicePlugin= createTpEspServicePlugin();
+        espServicePlugin->setName(servicePlugin);
+        if (servicePluginType && *servicePluginType)
+            espServicePlugin->setType(servicePluginType);
+
+        espServicePlugins.append(*espServicePlugin.getClear());
+    }
+
     m_enableSNMP = false;
 }
 
@@ -1715,3 +1732,29 @@ bool CWsTopologyEx::onTpThorStatus(IEspContext &context, IEspTpThorStatusRequest
     }
     return false;
 }
+
+bool CWsTopologyEx::onTpGetServicePlugins(IEspContext &context, IEspTpGetServicePluginsRequest &req, IEspTpGetServicePluginsResponse &resp)
+{
+    try
+    {
+        if (!context.validateFeatureAccess(FEATURE_URL, SecAccess_Read, false))
+            throw MakeStringException(ECLWATCH_TOPOLOGY_ACCESS_DENIED, "Failed to get Service Plugins. Permission denied.");
+
+        IArrayOf<IEspTpEspServicePlugin> plugins;
+        ForEachItemIn(i,espServicePlugins)
+        {
+            IEspTpEspServicePlugin& servicePlugin = espServicePlugins.item(i);
+
+            Owned<IEspTpEspServicePlugin> plugin= createTpEspServicePlugin();
+            plugin->setName(servicePlugin.getName());
+            plugin->setType(servicePlugin.getType());
+            plugins.append(*plugin.getClear());
+        }
+        resp.setPlugins(plugins);
+    }
+    catch(IException* e)
+    {
+        FORWARDEXCEPTION(context, e,  ECLWATCH_INTERNAL_ERROR);
+    }
+    return false;
+}

esp/services/ws_topology/ws_topologyService.hpp

@@ -89,6 +89,7 @@ private:
     bool                                m_enableSNMP;
     StringAttr                          defaultTargetClusterName;
     StringAttr                          defaultTargetClusterPrefix;
+    IArrayOf<IEspTpEspServicePlugin>    espServicePlugins;
 
     void getThorXml(const char *cluster,StringBuffer& strBuff);
     void getThorLog(const char *cluster,MemoryBuffer& returnbuff);
@@ -144,7 +145,7 @@ public:
     bool onTpLogFileDisplay(IEspContext &context,IEspTpLogFileRequest  &req, IEspTpLogFileResponse &resp);
 
     bool onTpServiceQuery(IEspContext &context, IEspTpServiceQueryRequest &req, IEspTpServiceQueryResponse &resp);
-
+    bool onTpGetServicePlugins(IEspContext &context, IEspTpGetServicePluginsRequest &req, IEspTpGetServicePluginsResponse &resp);
     bool onTpGetComponentFile(IEspContext &context, IEspTpGetComponentFileRequest &req, IEspTpGetComponentFileResponse &resp);
 
     bool onTpThorStatus(IEspContext &context, IEspTpThorStatusRequest &req, IEspTpThorStatusResponse &resp);

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -800,7 +800,7 @@ void WsWuInfo::getEventScheduleFlag(IEspECLWorkunit &info)
                 if (!r)
                     continue;
 
-                IWorkflowEvent *wfevent = r->getScheduleEvent();
+                Owned<IWorkflowEvent> wfevent = r->getScheduleEvent();
                 if (!wfevent)
                     continue;
 

esp/src/eclwatch/LogsWidget.js

@@ -84,6 +84,10 @@ define([
                     case "hpp":
                         params = "/WUFile?Wuid=" + this.wu.Wuid + "&Name=" + item.Orig.Name + "&IPAddress=" + item.Orig.IPAddress + "&Description=" + item.Orig.Description + "&Type=" + item.Orig.Type;
                         break;
+                    case "xml":
+                        if (option !== undefined)
+                            params = "/WUFile?Wuid=" + this.wu.Wuid + "&Name=" + item.Orig.Name + "&IPAddress=" + item.Orig.IPAddress + "&Description=" + item.Orig.Description + "&Type=" + item.Orig.Type;
+                        break;
                 }
 
                 return ESPRequest.getBaseURL() + params + (option ? "&Option=" + option : "&Option=1");

testing/CMakeLists.txt

@@ -14,7 +14,9 @@
 #    limitations under the License.
 ################################################################################
 HPCC_ADD_SUBDIRECTORY (unittests)
+HPCC_ADD_SUBDIRECTORY (regress)
 install( DIRECTORY regress DESTINATION "./testing" COMPONENT Runtime
          USE_SOURCE_PERMISSIONS
-         PATTERN regress/ecl EXCLUDE )
-
+         PATTERN regress/ecl EXCLUDE
+         PATTERN regress/environment.xml.in EXCLUDE
+         PATTERN regress/CMakeLists.txt EXCLUDE )

testing/regress/CMakeLists.txt

@@ -0,0 +1,19 @@
+################################################################################
+#    HPCC SYSTEMS software Copyright (C) 2012 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it
+#    and/or modify
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+################################################################################
+configure_file("environment.xml.in" "environment.xml")
+
+Install ( FILES ${CMAKE_CURRENT_BINARY_DIR}/environment.xml DESTINATION "./testing/regress" COMPONENT Runtime )

testing/regress/README.rst

@@ -21,6 +21,7 @@ Result:
 |                       [--ignoreResult]
 |                       [-X name1=value1[,name2=value2...]]
 |                       [-f optionA=valueA[,optionB=valueB...]]
+|                       [--dynamic source=cluster_list|all]
 |                       {list,setup,run,query} ...
 | 
 |       HPCC Platform Regression suite
@@ -48,6 +49,8 @@ Result:
 |                                 sets the stored input value (stored('name')).
 |        -f optionA=valueA[,optionB=valueB...]
 |                                 set an ECL option (equivalent to #option).
+|        --dynamic source=cluster_list|all
+|                                 execute ECL query through a generated stub with source cluster(s).
 
 Important!
     There is a bug in Python argparse library whichis impacts the quoted parameters. So either in -X or -f or both contains a value with space(s) inside then the whole argument should be put in double quote!
@@ -461,7 +464,48 @@ The format of the output is the same as 'run', except there is a log, result and
 |
 |         End.
 
-6. Tags used in testcases:
+
+6. Use --dynamic parameter:
+---------------------------
+
+Example command:
+
+        ./ecl-test  --dynamic source='all' run -t hthor --runclass=dynamic,dfu
+
+This command executes all ECL tests which are belongs to dynamic and dfu classes. If any test contains //dynamic:source tag then that will executed with all sources via a generated stub for each.
+
+The format of the output is the same as 'run', except all dynamic executed ECL file marked with source:
+
+|         [Action] Suite: hthor
+|         [Action] Queries: 5
+|         [Action]   1. Test: dynamic_test2.ecl ( source: hthor )
+|         [Pass]   1. Pass W20140917-103147 (1 sec)
+|         [Pass]   1. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103147
+|         [Action]   2. Test: dynamic_test2.ecl ( source: thor )
+|         [Pass]   2. Pass W20140917-103149 (1 sec)
+|         [Pass]   2. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103149
+|         [Action]   3. Test: dynamic_test2.ecl ( source: roxie )
+|         [Pass]   3. Pass W20140917-103151 (1 sec)
+|         [Pass]   3. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103151
+|         [Action]   4. Test: spray_test.ecl
+|         [Pass]   4. Pass W20140917-103153 (2 sec)
+|         [Pass]   4. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103153
+|         [Action]   5. Test: spray_test2.ecl
+|         [Pass]   5. Pass W20140917-103156 (2 sec)
+|         [Pass]   5. URL http://127.0.0.1:8010/WsWorkunits/WUInfo?Wuid=W20140917-103156
+|         [Action]
+|            Results
+|            -------------------------------------------------
+|             Passing: 5
+|             Failure: 0
+|             -------------------------------------------------
+|             Log: /home/ati/HPCCSystems-regression/log/hthor.14-09-17-10-31-46.log
+|             -------------------------------------------------
+|             Elapsed time: 14 sec  (00:00:14)
+|             -------------------------------------------------
+
+
+7. Tags used in testcases:
 --------------------------
 
     To exclude testcase from cluster or clusters, the tag is:
@@ -486,8 +530,11 @@ The format of the output is the same as 'run', except there is a log, result and
     To define a class to be executed/excluded in run mode.
 //class=<class_name>
 
+    To use dynamic source to execute same ECL with different source
+//dynamic:source
+
 
-7. Key file handling:
+8. Key file handling:
 ---------------------
 
 After an ECL test case execution finished and all output collected the result checking follows these steps:
@@ -555,7 +602,7 @@ If we execute setup on any other target:
 Then the RS executes all ecl files from setup directory and 
     - the test cases results compared with corresponding file in ecl/key directory.
 
-8. Key file generation:
+9. Key file generation:
 -----------------------
 
 The regression suite stores every test case output into ~/HPCCSystems-regression/result directory. This is the latest version of result. (The previous version can be found in ~/HPCCSystems-regression/archives directory.) When a test case execution finished Regression Suite compares this output file with the relevant key file to verify the result.
@@ -568,7 +615,7 @@ So if you have a new test case and it works well on all clusters (or some of the
 
 (To check everything is fine, repeat the step 1 and the query should now pass. )
 
-9. Configuration setting in ecl-test.json file:
+10. Configuration setting in ecl-test.json file:
 -------------------------------------------------------------
 
         "IpAddress":{
@@ -698,7 +745,7 @@ Important!
 
 
 
-10. Authentication:
+11. Authentication:
 -------------------
 
 If your HPCC System is configured to use LDAP authentication you should change value of "username" and "password" fields in ecl-test.json file to yours.

testing/regress/ecl-test

@@ -160,6 +160,8 @@ class RegressMain:
                             nargs=1, type=checkXParam,  default='None',  metavar="name1=value1[,name2=value2...]")
         parser.add_argument('-f', help="set an ECL option (equivalent to #option).",
                             nargs=1, type=checkXParam,  default='None',  metavar="optionA=valueA[,optionB=valueB...]")
+        parser.add_argument('--dynamic', help="execute ECL query through a generated stub with source cluster(s).",
+                            nargs=1, type=checkXParam,  default='None',  metavar="source=cluster_list|all")
 
         subparsers = parser.add_subparsers(help='sub-command help')
 

testing/regress/ecl/dynamicsource_tag.ecl

@@ -0,0 +1,5 @@
+//class=dynamic
+//dynamic:source
+export dynamicsource_tag := MODULE
+   EXPORT execute(string source = '', boolean useLocal = false) := output('Source = fine');
+END;

testing/regress/ecl/key/dynamicsource_tag.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>Source = fine</Result_1></Row>
+</Dataset>

testing/regress/hpcc/regression/regress.py

@@ -307,6 +307,7 @@ class Regression:
             logging.warn('%s','' , extra={'filebuffer':True,  'filesort':True})
             suite.setEndTime(time.time())
             Regression.displayReport(report, suite.getElapsTime())
+            suite.close()
             self.closeLogging()
 
         except Exception as e:
@@ -385,6 +386,7 @@ class Regression:
 
             suite.setEndTime(time.time())
             Regression.displayReport(report, suite.getElapsTime())
+            suite.close()
             self.closeLogging()
 
         except Exception as e:
@@ -423,6 +425,7 @@ class Regression:
 
             self.StopTimeoutThread()
             Regression.displayReport(report,  time.time()-start)
+            eclfile.close()
             self.closeLogging()
 
         except Exception as e:
@@ -435,7 +438,7 @@ class Regression:
         self.exitmutexes[th].acquire()
 
         logging.debug("runQuery(cluster: '%s', query: '%s', cnt: %d, publish: %s, thread id: %d" % ( cluster, query.ecl, cnt, publish,  th))
-        logging.warn("%3d. Test: %s" % (cnt, query.ecl),  extra={'taskId':cnt})
+        logging.warn("%3d. Test: %s" % (cnt, query.getBaseEclRealName()),  extra={'taskId':cnt})
 
         self.loggermutex.release()
         res = 0

testing/regress/hpcc/regression/suite.py

@@ -20,9 +20,11 @@
 import os
 import sys
 import time
+import glob
 
 from ..util.ecl.file import ECLFile
 from ..common.error import Error
+from ..util.util import checkClusters
 
 class Suite:
     def __init__(self, name, dir_ec, dir_a, dir_ex, dir_r, logDir, args, isSetup=False,  fileList = None):
@@ -30,6 +32,7 @@ class Suite:
             self.name = 'setup_'+name
         else:
             self.name = name
+        self.args=args
         self.suite = []
         self.dir_ec = dir_ec
         self.dir_a = dir_a
@@ -38,6 +41,16 @@ class Suite:
         self.logDir = logDir
         self.exclude = []
         self.publish = []
+        self.isDynamicSource=False
+        if args.dynamic != 'None':
+            self.isDynamicSource=True
+            self.dynamicSources=args.dynamic[0].replace('source=', '').replace('\'','').split(',')
+            self.dynamicSources=checkClusters(self.dynamicSources,  "Dynamic source")
+            pass
+
+        # If there are some temprary files left, then remove them
+        for file in glob.glob(self.dir_ec+'/_tmp*.ecl'):
+            os.unlink(file)
 
         self.buildSuite(args, isSetup, fileList)
 
@@ -51,6 +64,10 @@ class Suite:
                 self.log.write(item+"\n")
             self.log.close();
 
+    def __del__(self):
+        print "Suite destructor."
+        pass
+
     def buildSuite(self, args, isSetup,  fileList):
         if fileList == None:
             if not os.path.isdir(self.dir_ec):
@@ -74,7 +91,7 @@ class Suite:
             if file.endswith(".ecl"):
                 ecl = os.path.join(self.dir_ec, file)
                 eclfile = ECLFile(ecl, self.dir_a, self.dir_ex,
-                                  self.dir_r,  self.name, args)
+                                  self.dir_r,  self.args.target,  args)
                 if isSetup:
                     skipResult = eclfile.testSkip('setup')
                 else:
@@ -99,7 +116,7 @@ class Suite:
                         exclusionReason=' ECL excluded'
 
                     if not exclude:
-                        self.suite.append(eclfile)
+                        self.addFileToSuite(eclfile)
                     else:
                         self.exclude.append(format(file, "30")+exclusionReason)
                 else:
@@ -108,6 +125,21 @@ class Suite:
                 if eclfile.testPublish():
                     self.publish.append(eclfile.getBaseEcl())
 
+    def addFileToSuite(self, eclfile):
+        if eclfile.testDynamicSource() and self.isDynamicSource:
+            # going through the source lists
+            basename = eclfile.getEcl()
+            for source in self.dynamicSources:
+                # generates ECLs based on sources
+                eclfile = ECLFile(basename, self.dir_a, self.dir_ex,
+                                  self.dir_r,  self.name, self.args)
+                eclfile.setDynamicSource(source)
+                # add newly generated ECL to suite
+                self.suite.append(eclfile)
+            pass
+        else:
+            self.suite.append(eclfile)
+
     def testPublish(self, ecl):
         if ecl in self.publish:
             return True
@@ -127,3 +159,7 @@ class Suite:
 
     def getSuiteName(self):
         return self.name
+
+    def close(self):
+        for ecl in self.suite:
+            ecl.close()

testing/regress/hpcc/util/ecl/cc.py

@@ -34,9 +34,15 @@ class ECLCC(Shell):
     def __ECLCC(self):
         return self.command(self.cmd, *self.defaults)
 
-    def getArchive(self, file):
+    def getArchive(self, ecl):
         try:
-            return self.__ECLCC()('-E', file)
+            if ecl.testDynamicSource():
+                stub = ecl.getRealEclSource()
+                # do eclcc with stdin
+                return self.__ECLCC()('-E', stub)
+            else:
+                file = ecl.getEcl()
+                return self.__ECLCC()('-E', file)
         except Error as err:
             logging.debug("getArchive exception:'%s'",  repr(err))
             self.makeArchiveError = str(err)
@@ -50,7 +56,7 @@ class ECLCC(Shell):
             os.mkdir(dirname)
         if os.path.isfile(filename):
             os.unlink(filename)
-        result = self.getArchive(ecl.getEcl())
+        result = self.getArchive(ecl)
 
         if result.startswith( 'Error()'):
             retVal = False
@@ -68,6 +74,7 @@ class ECLCC(Shell):
                 ecl.diff += repr(self.makeArchiveError)
             self.makeArchiveError=''
         else:
+            logging.debug("%3d. makeArchive (filename:'%s')", ecl.getTaskId(), filename )
             FILE = open(filename, "w")
             FILE.write(result)
             FILE.close()

testing/regress/hpcc/util/ecl/file.py

@@ -22,6 +22,7 @@ import logging
 import os
 import traceback
 import re
+import tempfile
 
 from ...util.util import isPositiveIntNum, getConfig
 
@@ -42,6 +43,16 @@ class ECLFile:
     abortReason = ''
     taskId = -1
     ignoreResult=False
+    isDynamicSource=None
+    dynamicSource='test'
+
+    def __del__(self):
+        logging.debug("%3d. File destructor (file:%s).", self.taskId, self.ecl )
+
+    def close(self):
+        if self.tempFile:
+            self.tempFile.close()
+            pass
 
     def __init__(self, ecl, dir_a, dir_ex, dir_r,  cluster, args):
         self.dir_ec = os.path.dirname(ecl)
@@ -49,16 +60,18 @@ class ECLFile:
         self.dir_r = dir_r
         self.dir_a = dir_a
         self.cluster = cluster;
-        baseEcl = os.path.basename(ecl)
-        self.basename = os.path.splitext(baseEcl)[0]
-        baseXml = self.basename + '.xml'
-        self.ecl = baseEcl
-        self.xml_e = baseXml
-        self.xml_r = baseXml
-        self.xml_a = 'archive_' + baseXml
+        self.baseEcl = os.path.basename(ecl)
+        self.basename = os.path.splitext(self.baseEcl)[0]
+        self.baseXml = self.basename + '.xml'
+        self.ecl = self.baseEcl
+        self.xml_e = self.baseXml
+        self.xml_r = self.baseXml
+        self.xml_a = 'archive_' + self.baseXml
         self.jobname = self.basename
         self.diff = ''
         self.abortReason =''
+        self.tags={}
+        self.tempFile=None
 
         #If there is a --publish CL parameter then force publish this ECL file
         self.forcePublish=False
@@ -77,7 +90,7 @@ class ECLFile:
                     testSpec = testSpec.replace('*',  '\w+')
 
                 testSpec = testSpec.replace('.',  '\.')
-                match = re.match(testSpec,  baseEcl)
+                match = re.match(testSpec,  self.baseEcl)
                 if match:
                     optXs = ("-X"+val.replace(',',  ',-X')).split(',')
                     self.processKeyValPairs(optXs,  self.optXHash)
@@ -158,17 +171,43 @@ class ECLFile:
         return os.path.join(self.dir_r, self.xml_r)
 
     def getArchive(self):
-        return os.path.join(self.dir_a, self.xml_a)
+        logging.debug("%3d. getArchive (isDynamicSource:'%s')", self.taskId, self.isDynamicSource )
+        if self.isDynamicSource:
+            dynamicFilename='archive_' + self.basename + '_'+ self.dynamicSource+'.xml'
+            return os.path.join(self.dir_a, dynamicFilename)
+        else:
+            return os.path.join(self.dir_a, self.xml_a)
 
     def getEcl(self):
         return os.path.join(self.dir_ec, self.ecl)
 
+    def getRealEclSource(self):
+        logging.debug("%3d. getRealEclSource (isDynamicSource:'%s')", self.taskId, self.isDynamicSource )
+        if self.isDynamicSource:
+            # generate stub and return with it
+            self.tempFile = tempfile.NamedTemporaryFile(prefix='_temp',  suffix='.ecl', dir=self.dir_ec)
+            self.tempFile.write('import '+self.basename+';\n')
+            self.tempFile.write(self.basename+'.execute(source := \''+self.dynamicSource+'\');\n')
+            self.tempFile.flush()
+            # now return with the generated
+            return os.path.join(self.dir_ec, self.tempFile.name)
+        else:
+            return os.path.join(self.dir_ec, self.ecl)
+
     def getBaseEcl(self):
         return self.ecl
 
     def getBaseEclName(self):
         return self.basename
 
+    def getBaseEclRealName(self):
+        logging.debug("%3d. getBaseEclRealName (isDynamicSource:'%s')", self.taskId, self.isDynamicSource )
+        if self.isDynamicSource:
+            realName = self.basename + '.ecl ( source: ' + self.dynamicSource + ' )'
+        else:
+            realName = self.getBaseEcl()
+        return realName
+
     def getWuid(self):
         return self.wuid
 
@@ -278,6 +317,16 @@ class ECLFile:
         logging.debug("%3d. testInClass() returns with: %s",  self.taskId,  retVal)
         return retVal
 
+    def testDynamicSource(self):
+        if self.isDynamicSource == None:
+            # Standard string has a problem with unicode characters
+            # use byte arrays and binary file open instead
+            tag = b'//dynamic:source'
+            logging.debug("%3d. testDynamicSource (ecl:'%s', tag:'%s')", self.taskId, self.ecl,  tag)
+            self.isDynamicSource = self.__checkTag(tag)
+            logging.debug("%3d. testDynamicSource() returns with: %s",  self.taskId,  self.isDynamicSource)
+        return self.isDynamicSource
+
     def getTimeout(self):
         timeout = 0
         # Standard string has a problem with unicode characters
@@ -359,3 +408,10 @@ class ECLFile:
     def getFParameters(self):
         return self.optF
 
+    def setDynamicSource(self,  source):
+        self.dynamicSource = source
+        self.isDynamicSource=True
+
+    def getDynamicSource(self):
+        return self.dynamicSource
+