Merge branch 'candidate-7.0.6' into candidate-7.2.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/base/dadfs.cpp

@@ -10365,7 +10365,13 @@ IDFAttributesIterator *CDistributedFileDirectory::getDFAttributesIterator(const
     CMessageBuffer mb;
     mb.append((int)MDFS_ITERATE_FILES).append(wildname).append(recursive).append("").append(includesuper); // "" is legacy
     if (user)
-        user->serialize(mb);
+    {
+		Owned<IUserDescriptor> tmpUDesc = createUserDescriptor();
+		StringBuffer userName;
+		user->getUserName(userName);
+		tmpUDesc->set(userName.str(), nullptr);
+		tmpUDesc->serialize(mb);//serialize without password, since it is not checked
+    }
 #ifdef NULL_DALIUSER_STACKTRACE
     else
     {
@@ -10496,7 +10502,13 @@ void CDistributedFileDirectory::setFileAccessed(CDfsLogicalFileName &dlfn,IUserD
     mb.append((int)MDFS_SET_FILE_ACCESSED).append(lname);
     dt.serialize(mb);
     if (user)
-        user->serialize(mb);
+    {
+        Owned<IUserDescriptor> tmpUDesc = createUserDescriptor();
+        StringBuffer userName;
+        user->getUserName(userName);
+        tmpUDesc->set(userName.str(), nullptr);
+        tmpUDesc->serialize(mb);//serialize without password, since it is not checked
+    }
 #ifdef NULL_DALIUSER_STACKTRACE
     else
     {
@@ -10535,7 +10547,13 @@ void CDistributedFileDirectory::setFileProtect(CDfsLogicalFileName &dlfn,IUserDe
         owner = "";
     mb.append((int)MDFS_SET_FILE_PROTECT).append(lname).append(owner).append(set);
     if (user)
-        user->serialize(mb);
+    {
+		Owned<IUserDescriptor> tmpUDesc = createUserDescriptor();
+		StringBuffer userName;
+		user->getUserName(userName);
+		tmpUDesc->set(userName.str(), nullptr);
+		tmpUDesc->serialize(mb);//serialize without password, since it is not checked
+    }
 #ifdef NULL_DALIUSER_STACKTRACE
     else
     {
@@ -10570,7 +10588,13 @@ IPropertyTree *CDistributedFileDirectory::getFileTree(const char *lname, IUserDe
     mb.append((int)MDFS_GET_FILE_TREE).append(lname);
     mb.append(MDFS_GET_FILE_TREE_V2);
     if (user)
-        user->serialize(mb);
+    {
+        Owned<IUserDescriptor> tmpUDesc = createUserDescriptor();
+        StringBuffer userName;
+        user->getUserName(userName);
+        tmpUDesc->set(userName.str(), nullptr);
+        tmpUDesc->serialize(mb);//serialize without password, since it is not checked
+    }
 #ifdef NULL_DALIUSER_STACKTRACE
     else
     {
@@ -12444,7 +12468,13 @@ IPropertyTreeIterator *CDistributedFileDirectory::getDFAttributesTreeIterator(co
         mb.append((int)MDFS_ITERATE_FILTEREDFILES2);
     mb.append(filters).append(recursive);
     if (user)
-        user->serialize(mb);
+    {
+        Owned<IUserDescriptor> tmpUDesc = createUserDescriptor();
+        StringBuffer userName;
+        user->getUserName(userName);
+        tmpUDesc->set(userName.str(), nullptr);
+        tmpUDesc->serialize(mb);//serialize without password, since it is not checked
+    }
 
     if (foreigndali)
         foreignDaliSendRecv(foreigndali,mb,foreigndalitimeout);

dali/base/dasess.cpp

@@ -937,7 +937,15 @@ public:
             PrintStackReport();
         }
 #endif
-        udesc->serialize(mb);
+
+        {
+            Owned<IUserDescriptor> tmpUDesc = createUserDescriptor();
+            StringBuffer user;
+            udesc->getUserName(user);
+            tmpUDesc->set(user.str(), nullptr);
+            tmpUDesc->serialize(mb);//serialize without password, since it is not checked
+        }
+
         mb.append(auditflags);
 
         //Serialize signature. If not provided, compute it

ecl/hql/hqlattr.cpp

@@ -3889,7 +3889,7 @@ IHqlExpression * evaluateLikelihood(IHqlExpression * expr)
                 double p2 = queryLikelihood(expr->queryChild(1));
                 if (isKnownLikelihood(p2))
                 {
-                    likelihoodExpr.set(createConstant(createRealValue(p1*p2,8)));
+                    likelihoodExpr.setown(createConstant(createRealValue(p1*p2,8)));
                     break;
                 }
             }
@@ -3904,7 +3904,7 @@ IHqlExpression * evaluateLikelihood(IHqlExpression * expr)
                 double p2 = queryLikelihood(expr->queryChild(1));
                 if (isKnownLikelihood(p2))
                 {
-                    likelihoodExpr.set(createConstant(createRealValue(p1+p2-p1*p2,8)));
+                    likelihoodExpr.setown(createConstant(createRealValue(p1+p2-p1*p2,8)));
                     break;
                 }
             }
@@ -3916,7 +3916,7 @@ IHqlExpression * evaluateLikelihood(IHqlExpression * expr)
             double p1 = queryLikelihood(expr->queryChild(0));
             if (isKnownLikelihood(p1))
             {
-                likelihoodExpr.set(createConstant(createRealValue(1.0-p1,8)));
+                likelihoodExpr.setown(createConstant(createRealValue(1.0-p1,8)));
                 break;
             }
             likelihoodExpr.set(queryConstantLikelihoodUnknown());

ecl/hql/hqlcache.cpp

@@ -429,7 +429,7 @@ static IHqlExpression * createSimplifiedBodyDefinition(IHqlExpression * expr)
     case no_macro:
         return nullptr;
     }
-    ITypeInfo * type = getFullyUnqualifiedType(expr->queryType());
+    Owned<ITypeInfo> type = getFullyUnqualifiedType(expr->queryType());
     if (!type)
         return nullptr;
 

ecl/hql/hqldesc.cpp

@@ -432,12 +432,12 @@ void expandScopeSymbolsMeta(IPropertyTree * meta, IHqlScope * scope)
     {
         IHqlExpression * curSym = &symbols.item(i);
         HqlDummyLookupContext lookupCtx(NULL);
-        IHqlExpression * lookupSym = scope->lookupSymbol(curSym->queryId(), LSFsharedOK|LSFignoreBase, lookupCtx);
+        OwnedHqlExpr lookupSym = scope->lookupSymbol(curSym->queryId(), LSFsharedOK|LSFignoreBase, lookupCtx);
         InheritType ihType = local;
         ForEachItemIn(iScope, bases)
         {
             IHqlScope * base = &bases.item(iScope);
-            IHqlExpression * baseSym = base->lookupSymbol(lookupSym->queryId(), LSFsharedOK|LSFfromderived, lookupCtx);
+            OwnedHqlExpr baseSym = base->lookupSymbol(lookupSym->queryId(), LSFsharedOK|LSFfromderived, lookupCtx);
             if (baseSym)
             {
                 ihType = override;

ecl/hql/hqlgram.y

@@ -8779,7 +8779,7 @@ simpleDataSet
                             $$.setExpr(createDataset(no_metaactivity, $3.getExpr(), createAttribute(pullAtom)));
                             $$.setPosition($1);
                         }
-    | TRACE '(' startTopLeftRightSeqFilter optTraceFlags ')' endTopLeftRightFilter endSelectorSequence
+    | TRACE '(' startTopFilter optTraceFlags ')' endTopFilter
                         {
                             $$.setExpr(createDataset(no_metaactivity, $3.getExpr(), createComma(createAttribute(traceAtom), $4.getExpr())));
                             $$.setPosition($1);

ecl/hql/hqltrans.cpp

@@ -1997,7 +1997,7 @@ bool NewHqlTransformer::needToUpdateSelectors(IHqlExpression * expr)
     ForEachItemIn(i, scopesUsed)
     {
         IHqlExpression * cur = &scopesUsed.item(i);
-        IHqlExpression * transformed = transformSelector(cur);
+        OwnedHqlExpr transformed = transformSelector(cur);
         if (cur != transformed)
             return true;
     }

ecl/hqlcpp/hqlcpp.cpp

@@ -1723,7 +1723,7 @@ void HqlCppTranslator::cacheOptions()
         DebugOption(options.constantFoldNormalize,"constantFoldNormalize", true),
         DebugOption(options.constantFoldPostNormalize,"constantFoldPostNormalize", false),
         DebugOption(options.optimizeGrouping,"optimizeGrouping", true),
-        DebugOption(options.showMetaInGraph,"showMetaInGraph", false),
+        DebugOption(options.showMetaInGraph,"showMetaInGraph", true),
         DebugOption(options.spotComplexClasses,"spotComplexClasses", true),
         DebugOption(options.complexClassesThreshold,"complexClassesThreshold", 5000),
         DebugOption(options.complexClassesActivityFilter,"complexClassesActivityFilter", 0),

ecl/hqlcpp/hqlcppds.cpp

@@ -4856,7 +4856,7 @@ BoundRow * HqlCppTranslator::buildOptimizeSelectFirstRow(BuildCtx & ctx, IHqlExp
             OwnedHqlExpr selectRow = createRow(no_selectnth, LINK(expr), getSizetConstant(1));
             BoundRow * match = static_cast<BoundRow *>(ctx.queryAssociation(selectRow, AssocRow, NULL));
             if (match)
-                return LINK(match);
+                return match;
 
             Owned<BoundRow> tempRow = declareTempRow(ctx, ctx, selectRow);
             Owned<BoundRow> rowBuilder = createRowBuilder(ctx, tempRow);

ecl/hqlcpp/hqlresource.cpp

@@ -2830,7 +2830,7 @@ IHqlExpression * SpillerInfo::createSpilledWrite(IHqlExpression * transformed, b
     }
     else
     {
-        LinkedHqlExpr dataset = LINK(transformed);
+        LinkedHqlExpr dataset = transformed;
         if (dataset->isDictionary())
             dataset.setown(createDataset(no_datasetfromdictionary, dataset.getClear()));
         if (options->createSpillAsDataset && allowCommonDataset)

ecl/hqlcpp/hqlstmt.cpp

@@ -1435,7 +1435,7 @@ int queryMemsetChar(IHqlExpression * expr)
     if (expr->getOperator() != no_constant)
         return -1;
     unsigned size = expr->queryType()->getSize();
-    if (size == 0)
+    if (size == 0 || size == UNKNOWN_LENGTH)
         return -1;
     const byte * data = (const byte *)expr->queryValue()->queryValue();
     byte match = data[0];

ecl/hqlcpp/hqltcppc.cpp

@@ -2175,7 +2175,7 @@ void CSpecialVStringColumnInfo::setColumn(HqlCppTranslator & translator, BuildCt
         args.append(*LINK(length));
         args.append(*translator.getElementPointer(bound.expr));
         translator.callProcedure(ctx, str2VStrId, args);
-        associateSizeOf(ctx, selector, LINK(targetSize), 0);
+        associateSizeOf(ctx, selector, targetSize, 0);
     }
 }
 

ecl/hthor/hthor.cpp

@@ -8266,24 +8266,83 @@ bool CHThorDiskReadBaseActivity::openNext()
                 actualFilter.appendFilters(fieldFilters);
 
             bool canSerializeTypeInfo = actualDiskMeta->queryTypeInfo()->canSerialize() && projectedDiskMeta->queryTypeInfo()->canSerialize();
-            for (unsigned copy=0; copy < numCopies; copy++)
+            if (canSerializeTypeInfo && (rt_binary == readType))
             {
-                RemoteFilename rfilename;
-                if (curPart)
-                    curPart->getFilename(rfilename,copy);
-                else
-                    ldFile->getPartFilename(rfilename,partNum,copy);
-                rfilename.getPath(file.clear());
-                filelist.append('\n').append(file);
-                try
+                for (unsigned copy=0; copy < numCopies; copy++)
                 {
-                    inputfile.setown(createIFile(rfilename));
+                    RemoteFilename rfilename;
+                    if (curPart)
+                        curPart->getFilename(rfilename,copy);
+                    else
+                        ldFile->getPartFilename(rfilename,partNum,copy);
+                    rfilename.getPath(file.clear());
+                    filelist.append('\n').append(file);
+                    try
+                    {
+                        // NB: only binary handles can be remotely processed by dafilesrv at the moment
+
+                        StringBuffer path;
+                        if (isRemoteReadCandidate(agent, rfilename, path))
+                        {
+                            // Open a stream from remote file, having passed actual, expected, projected, and filters to it
+                            SocketEndpoint ep(rfilename.queryEndpoint());
+                            setDafsEndpointPort(ep);
+
+                            Owned<IRemoteFileIO> remoteFileIO = createRemoteFilteredFile(ep, path, actualDiskMeta, projectedDiskMeta, actualFilter, compressed, grouped, remoteLimit);
+                            if (remoteFileIO)
+                            {
+                                StringBuffer tmp;
+                                remoteFileIO->addVirtualFieldMapping("logicalFilename", logicalFileName.str());
+                                remoteFileIO->addVirtualFieldMapping("baseFpos", tmp.clear().append(offsetOfPart).str());
+                                remoteFileIO->addVirtualFieldMapping("partNum", tmp.clear().append(curPart->getPartIndex()).str());
+
+                                try
+                                {
+                                    remoteFileIO->ensureAvailable(); // force open now, because want to failover to other copies or legacy if fails
+                                }
+                                catch (IException *e)
+                                {
+                                    EXCLOG(e, nullptr);
+                                    e->Release();
+                                    continue; // try next copy and ultimately failover to local when no more copies
+                                }
 
-                    // NB: only binary handles can be remotely processed by dafilesrv at the moment
+                                inputfile.setown(createIFile(rfilename));
 
-                    StringBuffer path;
-                    if ((rt_binary != readType) || !canSerializeTypeInfo || !isRemoteReadCandidate(agent, rfilename, path))
+                                actualDiskMeta.set(projectedDiskMeta);
+                                expectedDiskMeta = projectedDiskMeta;
+                                actualFilter.clear();
+                                inputfileio.setown(remoteFileIO.getClear());
+                            }
+                        }
+                        if (inputfileio)
+                            break;
+                    }
+                    catch (IException *E)
                     {
+                        if (saveOpenExc.get())
+                            E->Release();
+                        else
+                            saveOpenExc.setown(E);
+                    }
+                    closepart();
+                }
+            }
+            if (!inputfile)
+            {
+                for (unsigned copy=0; copy < numCopies; copy++)
+                {
+                    RemoteFilename rfilename;
+                    if (curPart)
+                        curPart->getFilename(rfilename,copy);
+                    else
+                        ldFile->getPartFilename(rfilename,partNum,copy);
+                    rfilename.getPath(file.clear());
+                    filelist.append('\n').append(file);
+                    try
+                    {
+                        inputfile.setown(createIFile(rfilename));
+
                         if (compressed)
                         {
                             Owned<IExpander> eexp;
@@ -8299,37 +8358,17 @@ bool CHThorDiskReadBaseActivity::openNext()
                         }
                         else
                             inputfileio.setown(inputfile->open(IFOread));
+                        if (inputfileio)
+                            break;
                     }
-                    else
+                    catch (IException *E)
                     {
-                        // Open a stream from remote file, having passed actual, expected, projected, and filters to it
-                        SocketEndpoint ep(rfilename.queryEndpoint());
-                        setDafsEndpointPort(ep);
-
-                        Owned<IRemoteFileIO> remoteFileIO = createRemoteFilteredFile(ep, path, actualDiskMeta, projectedDiskMeta, actualFilter, compressed, grouped, remoteLimit);
-                        if (remoteFileIO)
-                        {
-                            StringBuffer tmp;
-                            remoteFileIO->addVirtualFieldMapping("logicalFilename", logicalFileName.str());
-                            remoteFileIO->addVirtualFieldMapping("baseFpos", tmp.clear().append(offsetOfPart).str());
-                            remoteFileIO->addVirtualFieldMapping("partNum", tmp.clear().append(curPart->getPartIndex()).str());
-                            actualDiskMeta.set(projectedDiskMeta);
-                            expectedDiskMeta = projectedDiskMeta;
-                            actualFilter.clear();
-                            inputfileio.setown(remoteFileIO.getClear());
-                        }
+                        if (saveOpenExc.get())
+                            E->Release();
+                        else
+                            saveOpenExc.setown(E);
                     }
-                    if (inputfileio)
-                        break;
-                }
-                catch (IException *E)
-                {
-                    if (saveOpenExc.get())
-                        E->Release();
-                    else
-                        saveOpenExc.setown(E);
                 }
-                closepart();
             }
 
             //Check if the file requires translation, but translation is disabled

ecl/regress/regress.sh

@@ -93,12 +93,14 @@ if [[ $* != '' ]]; then
                 np="$OPTARG"
                 ;;
             v)
-               	valgrind="valgrind --leak-check=full --suppressions=suppressions.txt "
-               	eclcc="$valgrind $eclcc"
-               	compare_dir=
-               	;;
+                valgrind="valgrind --leak-check=full --suppressions=suppressions.txt "
+                userflags="$userflags --leakcheck"
+                eclcc="$valgrind $eclcc"
+                compare_dir=
+                ;;
             w)
                 valgrind="valgrind --leak-check=full --suppressions=suppressions.txt --vgdb-error=0 "
+                userflags="$userflags --leakcheck"
                 valgrind="$valgrind --track-origins=yes "
                 eclcc="$valgrind $eclcc"
                 compare_dir=

esp/esdllib/esdl_transformer2.cpp

@@ -1855,6 +1855,8 @@ void Esdl2Transformer::processHPCCResult(IEspContext &ctx, IEsdlDefMethod &mthde
                 xppToXmlString(*xpp, stag, logdata);
             else if (strieq(dataset, "royaltyset"))
                 xppToXmlString(*xpp, stag, logdata);
+            else if (strieq(dataset, "desdlsoaprequestecho"))
+                xppToXmlString(*xpp, stag, logdata);
             else
             {
                 WARNLOG("ESDL processing HPCC Result: Dataset ignored: %s", dataset);

esp/logging/loggingmanager/loggingmanager.cpp

@@ -162,8 +162,12 @@ bool CLoggingManager::updateLog(IEspContext* espContext, const char* option, IPr
 
             short port;
             StringBuffer sourceIP;
+            const char* esdlBindingID = espContext->queryESDLBindingID();
             espContext->getServAddress(sourceIP, port);
             espContextTree->addProp("SourceIP", sourceIP.str());
+            if (!isEmptyString(esdlBindingID))
+                espContextTree->addProp("ESDLBindingID", esdlBindingID);
+            //More information in espContext may be added to the espContextTree later.
 
             const char* userId = espContext->queryUserId();
             if (userId && *userId)

esp/platform/espcontext.cpp

@@ -43,6 +43,7 @@ private:
     StringAttr      m_acceptLanguage;
     StringAttr      httpMethod;
     StringAttr      servMethod;
+    StringAttr      esdlBindingID;
 
     StringBuffer    m_servName;
     StringBuffer    m_servHost;
@@ -477,6 +478,15 @@ public:
         servMethod.set(method);
     }
 
+    virtual void setESDLBindingID(const char *id)
+    {
+        esdlBindingID.set(id);
+    }
+    virtual const char* queryESDLBindingID()
+    {
+        return esdlBindingID.get();
+    }
+
     virtual CTxSummary* queryTxSummary()
     {
         return m_txSummary.get();

esp/scm/esp.ecm

@@ -191,6 +191,9 @@ interface IEspContext : extends IInterface
     virtual void setHTTPMethod(const char *method) = 0;
     virtual void setServiceMethod(const char *method) = 0;
 
+    virtual void setESDLBindingID(const char * id) = 0;
+    virtual const char * queryESDLBindingID() = 0;
+
     virtual void setTransactionID(const char * trxid) = 0;
     virtual const char * queryTransactionID() = 0;
 

esp/scm/ws_esdlconfig.ecm

@@ -209,6 +209,29 @@ ESPresponse [nil_remove, exceptions_inline] ConfigureESDLBindingMethodResponse
     [min_ver("1.2")] ESPStruct ESDLBindingContents ESDLBinding;
 };
 
+ESPrequest [nil_remove] ConfigureESDLBindingLogTransformRequest
+{
+    string EsdlBindingId;
+    string LogTransformName;
+    boolean Overwrite;
+    boolean Encoded(false); //The LogTransform in the Config is not encoded.
+    string Config; // dynamic xml, can have <Binding EspProcess=xxx EspBinding=WsAccurint><Definition name=xx id=xx.yy><LogTransforms><LogTransform>...
+                   //              or  <Definition name=xx id=xx.yy><LogTransforms><LogTransform>...
+                   //              or  <LogTransforms><LogTransform>....
+                   //<LogTransforms><LogTransform name="name">...</LogTransform><LogTransforms>
+    boolean EchoBinding;
+};
+
+ESPresponse [nil_remove, exceptions_inline] ConfigureESDLBindingLogTransformResponse
+{
+    string EspProcName;         //Name of ESP Process
+    string EspBindingName;
+    string EsdlDefinitionID;    //The ESDL definition name.ver
+    string EsdlServiceName;     //which ESDL definition are we configuring
+    ESPstruct BaseESDLStatus status;
+    ESPStruct ESDLBindingContents ESDLBinding;
+};
+
 ESPrequest GetESDLBindingRequest
 {
     string EsdlBindingId;
@@ -282,13 +305,14 @@ ESPresponse [exceptions_inline] ListESDLBindingsResponse
     [min_ver("1.4")] ESParray<ESPstruct EspProcessStruct, EspProcess> EspProcesses;
 };
 
-#define VERSION_FOR_ESDLCMD "1.4"
-ESPservice [auth_feature("ESDLConfigAccess:ACCESS"), version("1.4"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsESDLConfig
+#define VERSION_FOR_ESDLCMD "1.5"
+ESPservice [auth_feature("ESDLConfigAccess:ACCESS"), version("1.5"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsESDLConfig
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
     ESPmethod [auth_feature("ESDLConfigAccess:WRITE")] PublishESDLDefinition(PublishESDLDefinitionRequest, PublishESDLDefinitionResponse);
     ESPmethod [auth_feature("ESDLConfigAccess:WRITE")] PublishESDLBinding(PublishESDLBindingRequest, PublishESDLBindingResponse);
     ESPmethod [auth_feature("ESDLConfigAccess:WRITE")] ConfigureESDLBindingMethod(ConfigureESDLBindingMethodRequest, ConfigureESDLBindingMethodResponse);
+    ESPmethod [auth_feature("ESDLConfigAccess:WRITE"), min_ver("1.5")] ConfigureESDLBindingLogTransform(ConfigureESDLBindingLogTransformRequest, ConfigureESDLBindingLogTransformResponse);
     ESPmethod [auth_feature("ESDLConfigAccess:READ")]  GetESDLBinding(GetESDLBindingRequest, GetESDLBindingResponse);
     ESPmethod [auth_feature("ESDLConfigAccess:FULL")]  DeleteESDLBinding(DeleteESDLBindingRequest, DeleteESDLRegistryEntryResponse);
     ESPmethod [auth_feature("ESDLConfigAccess:FULL")]  DeleteESDLDefinition(DeleteESDLDefinitionRequest, DeleteESDLRegistryEntryResponse);

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -350,13 +350,22 @@ void EsdlServiceImpl::addMethodLevelRequestTransform(const char *method, IProper
     }
 }
 
-static void ensureMergeOrderedPTree(Owned<IPropertyTree> &target, IPropertyTree *src)
+static void ensureMergeOrderedEsdlTransform(Owned<IPropertyTree> &dest, IPropertyTree *src)
 {
     if (!src)
         return;
-    if (!target)
-        target.setown(createPTree(ipt_ordered));
-    mergePTree(target, src);
+    if (!dest)
+        dest.setown(createPTree(ipt_ordered));
+    //copy so we can make changes, like adding calculated targets
+    Owned<IPropertyTree> copy = createPTreeFromIPT(src, ipt_ordered);
+    const char *target = copy->queryProp("@target");
+    if (target && *target)
+    {
+        Owned<IPropertyTreeIterator> children = copy->getElements("*");
+        ForEach(*children)
+            children->query().setProp("@_crtTarget", target); //internal use only attribute
+    }
+    mergePTree(dest, copy);
 }
 
 void EsdlServiceImpl::configureTargets(IPropertyTree *cfg, const char *service)
@@ -370,10 +379,10 @@ void EsdlServiceImpl::configureTargets(IPropertyTree *cfg, const char *service)
         Owned<IPropertyTree> serviceCrt;
         try
         {
-            ensureMergeOrderedPTree(serviceCrt, target_cfg->queryPropTree("xsdl:CustomRequestTransform"));
+            ensureMergeOrderedEsdlTransform(serviceCrt, target_cfg->queryPropTree("xsdl:CustomRequestTransform"));
             Owned<IPropertyTreeIterator> transforms =  target_cfg->getElements("Transforms/xsdl:CustomRequestTransform");
             ForEach(*transforms)
-                ensureMergeOrderedPTree(serviceCrt, &transforms->query());
+                ensureMergeOrderedEsdlTransform(serviceCrt, &transforms->query());
         }
         catch (IPTreeException *e)
         {
@@ -416,10 +425,10 @@ void EsdlServiceImpl::configureTargets(IPropertyTree *cfg, const char *service)
             Owned<IPropertyTree> methodCrt;
             try
             {
-                ensureMergeOrderedPTree(methodCrt, methodCfg.queryPropTree("xsdl:CustomRequestTransform"));
+                ensureMergeOrderedEsdlTransform(methodCrt, methodCfg.queryPropTree("xsdl:CustomRequestTransform"));
                 Owned<IPropertyTreeIterator> transforms =  methodCfg.getElements("Transforms/xsdl:CustomRequestTransform");
                 ForEach(*transforms)
-                    ensureMergeOrderedPTree(methodCrt, &transforms->query());
+                    ensureMergeOrderedEsdlTransform(methodCrt, &transforms->query());
             }
             catch (IException *e)
             {
@@ -843,7 +852,7 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
     if(isroxie)
     {
         const char *tgtQueryName = tgtcfg->queryProp("@queryname");
-        if (tgtQueryName && *tgtQueryName)
+        if (!isEmptyString(tgtQueryName))
         {
             soapmsg.append("<soap:Body><").append(tgtQueryName).append(">");
             soapmsg.appendf("<_TransactionId>%s</_TransactionId>", context.queryTransactionID());
@@ -853,6 +862,28 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
 
             soapmsg.append(req.str());
             soapmsg.append("</").append(tgtQueryName).append("></soap:Body>");
+
+            auto cfgGateways = tgtcfg->queryBranch("Gateways");
+            if (cfgGateways)
+            {
+                auto baseXpath = cfgGateways->queryProp("@legacyTransformTarget");
+                if (!isEmptyString(baseXpath))
+                {
+                    Owned<IPTree> gws = createPTree("gateways", 0);
+                    Owned<IPTree> soapTree = createPTreeFromXMLString(soapmsg.append("</soap:Envelope>"), ipt_ordered);
+                    StringBuffer xpath(baseXpath);
+
+                    EsdlBindingImpl::transformGatewaysConfig(tgtcfg, gws, "row");
+                    xpath.replaceString("{$query}", tgtQueryName);
+                    xpath.replaceString("{$method}", mthdef.queryMethodName());
+                    xpath.replaceString("{$service}", srvdef.queryName());
+                    xpath.replaceString("{$request}", mthdef.queryRequestType());
+                    mergePTree(ensurePTree(soapTree, xpath), gws);
+                    toXML(soapTree, soapmsg.clear());
+                    soapmsg.setLength(strstr(soapmsg, "</soap:Envelope>") - soapmsg.str());
+                    soapmsg.trim();
+                }
+            }
         }
         else
             throw makeWsException( ERR_ESDL_BINDING_INTERNERR, WSERR_SERVER, "ESP",
@@ -871,7 +902,7 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
     soapmsg.append("</soap:Envelope>");
 
     const char *tgtUrl = tgtcfg->queryProp("@url");
-    if (tgtUrl && *tgtUrl)
+    if (!isEmptyString(tgtUrl))
     {
         if (crt)
         {
@@ -889,6 +920,96 @@ void EsdlServiceImpl::handleFinalRequest(IEspContext &context,
                    "No target URL configured for %s!", mthdef.queryMethodName());
     }
 
+    auto echoInsertionPoint = strstr(out, "</Result></Results>");
+
+    if (echoInsertionPoint)
+    {
+        try
+        {
+            auto queryname = (isroxie ? tgtcfg->queryProp("@queryname") : nullptr);
+            auto requestname = mthdef.queryRequestType();
+            const char* qname = nullptr;
+            const char* content = nullptr;
+            StringBuffer encodedContent;
+            StringBuffer echoDataset;
+
+            XmlPullParser xpp(soapmsg.str(), soapmsg.length());
+            StartTag stag;
+            EndTag etag;
+            int type = XmlPullParser::END_DOCUMENT;
+
+            echoDataset.ensureCapacity(soapmsg.length());
+            xpp.setSupportNamespaces(false); // parser throws exceptions when it encounters undefined namespace prefixes, such as CRT's 'xsdl'
+            while ((type = xpp.next()) != XmlPullParser::END_DOCUMENT)
+            {
+                switch (type)
+                {
+                case XmlPullParser::START_TAG:
+                    xpp.readStartTag(stag);
+                    qname = stag.getQName();
+                    if (streq(qname, "soap:Envelope"))
+                        echoDataset << "<Dataset name=\"DesdlSoapRequestEcho\">";
+                    else if (streq(qname, "soap:Body"))
+                        echoDataset << "<Row>";
+                    else if (queryname && streq(qname, queryname))
+                        echoDataset << "<roxierequest name=\"" << queryname << "\">";
+                    else if (streq(qname, requestname))
+                        echoDataset << "<esprequest name=\"" << requestname << "\">";
+                    else
+                    {
+                        echoDataset << '<';
+                        if (strncmp(qname, "xsdl:", 5) == 0)
+                            echoDataset << "xsdl_" << qname + 5; // eliminate problematic namespace prefix
+                        else
+                            echoDataset << qname;
+                        for (int idx = 0; idx < stag.getLength(); idx++)
+                        {
+                            encodeXML(stag.getValue(idx), encodedContent.clear());
+                            echoDataset << ' ' << stag.getRawName(idx) << "=\"" << encodedContent << '"';
+                        }
+                        echoDataset << '>';
+                    }
+                    break;
+
+                case XmlPullParser::END_TAG:
+                    xpp.readEndTag(etag);
+                    qname = etag.getQName();
+                    if (streq(qname, "soap:Envelope"))
+                        echoDataset << "</Dataset>";
+                    else if (streq(qname, "soap:Body"))
+                        echoDataset << "</Row>";
+                    else if (queryname && streq(qname, queryname))
+                        echoDataset << "</roxierequest>";
+                    else if (streq(qname, requestname))
+                        echoDataset << "</esprequest>";
+                    else if (strncmp(qname, "xsdl:", 5) == 0)
+                        echoDataset << "</xsdl_" << qname + 5 << '>'; // eliminate problematic namespace prefix
+                    else
+                        echoDataset << "</" << qname << '>';
+                    break;
+
+                case XmlPullParser::CONTENT:
+                    content = xpp.readContent();
+                    if (!isEmptyString(content))
+                    {
+                        encodeXML(content, encodedContent.clear());
+                        echoDataset << encodedContent;
+                    }
+                    break;
+                }
+            }
+            out.insert(echoInsertionPoint - out.str(), echoDataset);
+        }
+        catch (XmlPullParserException& xppe)
+        {
+            ERRLOG("Unable to echo transformed request to response: %s", xppe.what());
+        }
+        catch (...)
+        {
+            ERRLOG("Unable to echo transformed request to response");
+        }
+    }
+
     processResponse(context,srvdef,mthdef,ns,out);
 }
 
@@ -1580,6 +1701,7 @@ int EsdlBindingImpl::onGetInstantQuery(IEspContext &context,
                     generateNamespace(context, request, srvdef->queryName(), mthdef->queryName(), ns);
 
                     getSchemaLocation(context, request, schemaLocation);
+                    context.setESDLBindingID(m_bindingId.get());
                     m_pESDLService->handleServiceRequest(context, *srvdef, *mthdef, tgtcfg, tgtctx, ns.str(), schemaLocation.str(), req_pt.get(), out, logdata, 0);
 
                     response->setContent(out.str());
@@ -1881,8 +2003,8 @@ int EsdlBindingImpl::HandleSoapRequest(CHttpRequest* request,
             generateNamespace(*ctx, request, srvdef->queryName(), mthdef->queryName(), ns);
             getSchemaLocation(*ctx, request, schemaLocation);
 
-             m_pESDLService->handleServiceRequest(*ctx, *srvdef, *mthdef, tgtcfg, tgtctx, ns.str(),
-                                       schemaLocation.str(), pt, baseout, logdata, 0);
+            ctx->setESDLBindingID(m_bindingId.get());
+            m_pESDLService->handleServiceRequest(*ctx, *srvdef, *mthdef, tgtcfg, tgtctx, ns.str(), schemaLocation.str(), pt, baseout, logdata, 0);
 
             StringBuffer out;
             out.append(
@@ -2602,6 +2724,7 @@ void EsdlBindingImpl::handleJSONPost(CHttpRequest *request, CHttpResponse *respo
                         generateNamespace(*ctx, request, serviceName, methodName, ns);
                         getSchemaLocation(*ctx, request, schemaLocation);
 
+                        ctx->setESDLBindingID(m_bindingId.get());
                         m_pESDLService->handleServiceRequest(*ctx, *srvdef, *mthdef, tgtcfg, tgtctx, ns.str(), schemaLocation.str(), reqTree.get(), jsonresp, logdata, ESDL_BINDING_RESPONSE_JSON);
 
                         jsonresp.append("}");
@@ -3119,7 +3242,7 @@ int EsdlBindingImpl::onGetSampleXml(bool isRequest, IEspContext &ctx, CHttpReque
     return 0;
 }
 
-void EsdlBindingImpl::transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyTree* forRoxie )
+void EsdlBindingImpl::transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyTree* forRoxie, const char* altElementName )
 {
     // Do we need to handle 'local FQDN'? It doesn't appear to be in the
     // Gateway element. Not sure where it's set but the RemoteNSClient
@@ -3131,6 +3254,8 @@ void EsdlBindingImpl::transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyT
         cfgIter->first();
         if( cfgIter->isValid() )
         {
+            const char* treeName = (!isEmptyString(altElementName) ? altElementName : "Gateway");
+
             while( cfgIter->isValid() )
             {
                 IPropertyTree& cfgGateway = cfgIter->query();
@@ -3141,11 +3266,11 @@ void EsdlBindingImpl::transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyT
                     cfgGateway.getProp("@name", service);
                     service.toLowerCase();
 
-                    Owned<IPropertyTree> gw = createPTree("Gateway", false);
+                    Owned<IPropertyTree> gw = createPTree(treeName, false);
                     gw->addProp("ServiceName", service.str());
                     gw->addProp("URL", url.str());
 
-                    forRoxie->addPropTree("Gateway", gw.getLink());
+                    forRoxie->addPropTree(treeName, gw.getLink());
                 }
                 else
                 {

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -289,7 +289,7 @@ public:
 
     int onGetSampleXml(bool isRequest, IEspContext &ctx, CHttpRequest* request, CHttpResponse* response, const char *serv, const char *method);
     static void splitURLList(const char* urlList, StringBuffer& protocol,StringBuffer& UserName,StringBuffer& Password, StringBuffer& ipportlistbody, StringBuffer& path, StringBuffer& options);
-    static void transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyTree* forRoxie );
+    static void transformGatewaysConfig( IPropertyTree* srvcfg, IPropertyTree* forRoxie, const char* altElementName = nullptr );
     static bool makeURL( StringBuffer& url, IPropertyTree& cfg );
 
     bool usesESDLDefinition(const char * name, int version);

esp/services/esdl_svc_engine/esdl_store.cpp

@@ -434,6 +434,99 @@ public:
             return configureMethod(bindingId, methodName, configTree, overwrite, message);
     }
 
+    virtual int configureLogTransform(const char* bindingId, const char* logTransformName, IPropertyTree* configTree, bool overwrite, StringBuffer& message) override
+    {
+        if (isEmptyString(bindingId))
+        {
+            message.set("Unable to configure method, binding id must be provided");
+            return -1;
+        }
+        if (!configTree)
+        {
+            message.setf("Unable to configure method '%s', configuration attributes must be provided", logTransformName);
+            return -1;
+        }
+
+        VStringBuffer rxpath("%sBinding[@id='%s']/Definition/LogTransforms[1]", ESDL_BINDINGS_ROOT_PATH, bindingId);
+        Owned<IRemoteConnection> conn;
+
+        try
+        {
+            conn.setown(querySDS().connect(rxpath, myProcessSession(), RTM_LOCK_WRITE, SDS_LOCK_TIMEOUT_DESDL));
+        }
+        catch (ISDSException * e)
+        {
+            StringBuffer msg;
+            e->errorMessage(msg);
+            if (msg.isEmpty())
+                message.setf("Unable to operate on Dali path: %s", rxpath.str());
+            else
+                message.setf("Unable to operate on Dali path: %s. %s", rxpath.str(), msg.str());
+            e->Release();
+            return -1;
+        }
+
+        //Only lock the branch for the target we're interested in.
+        if (!conn)
+            throw MakeStringException(-1, "Unable to connect to %s", rxpath.str());
+
+        Owned<IPropertyTree> root = conn->getRoot();
+        if (!root.get())
+            throw MakeStringException(-1, "Unable to open %s", rxpath.str());
+
+        VStringBuffer xpath("LogTransform[@name='%s']", logTransformName);
+        Owned<IPropertyTree> oldEnvironment = root->getPropTree(xpath.str());
+        if (oldEnvironment.get())
+        {
+            if (overwrite)
+            {
+                message.set("Existing LogTransform configuration overwritten!");
+                root->removeTree(oldEnvironment);
+            }
+            else
+            {
+                message.set("LogTransform configuration exists will not overwrite!");
+                return -1;
+            }
+        }
+
+        root->addPropTree("LogTransform", configTree);
+        conn->commit();
+
+        VStringBuffer changestr("action=update;type=binding;targetId=%s", bindingId);
+        triggerSubscription(changestr.str());
+
+        message.appendf("\nSuccessfully configured Method '%s' for binding '%s'", logTransformName, bindingId);
+        return 0;
+    }
+
+    virtual int configureLogTransform(const char* espProcName, const char* espBindingName, const char* definitionId, const char* logTransformName, IPropertyTree* configTree, bool overwrite, StringBuffer& message) override
+    {
+        if (isEmptyString(espProcName))
+        {
+            message.set("Unable to configure method, ESP Process Name not available");
+            return -1;
+        }
+        if (isEmptyString(espBindingName))
+        {
+            message.set("Unable to configure method, ESP Binding Name not available");
+            return -1;
+        }
+
+        if (isEmptyString(definitionId))
+        {
+            message.set("Unable to configure method, ESDL definition ID not available");
+            return -1;
+        }
+
+        StringBuffer bindingId;
+        getIdFromProcBindingDef(espProcName, espBindingName, definitionId, bindingId, message);
+        if (bindingId.isEmpty())
+            return -1;
+
+        return configureLogTransform(bindingId, logTransformName, configTree, overwrite, message);
+    }
+
     virtual int bindService(const char* bindingName,
                                              IPropertyTree* methodsConfig,
                                              const char* espProcName,
@@ -649,6 +742,7 @@ public:
         else
             esdldeftree->addPropTree("Methods");
 
+        esdldeftree->addPropTree("LogTransforms");
         bindingtree->addPropTree(ESDL_DEF_ENTRY, LINK(esdldeftree));
         bindings->addPropTree(ESDL_BINDING_ENTRY, LINK(bindingtree));
 

esp/services/esdl_svc_engine/esdl_store.hpp

@@ -40,6 +40,8 @@ interface IEsdlStore : public IInterface
     virtual bool addDefinition(const char* definitionName, IPropertyTree* definitionInfo, StringBuffer &newId, unsigned &newSeq, const char* userid, bool deleteprev, StringBuffer & message) = 0;
     virtual int configureMethod(const char* bindingId, const char* methodName, IPropertyTree* configTree, bool overwrite, StringBuffer& message) = 0;
     virtual int configureMethod(const char* espProcName, const char* espBindingName, const char* definitionId, const char* methodName, IPropertyTree* configTree, bool overwrite, StringBuffer& message) = 0;
+    virtual int configureLogTransform(const char* bindingId, const char* logTransformName, IPropertyTree* configTree, bool overwrite, StringBuffer& message) = 0;
+    virtual int configureLogTransform(const char* espProcName, const char* espBindingName, const char* definitionId, const char* logTransformName, IPropertyTree* configTree, bool overwrite, StringBuffer& message) = 0;
     virtual int bindService(const char* bindingName, IPropertyTree* methodsConfig, const char* espProcName, const char* espPort, const char* definitionId,
                        const char* esdlServiceName, StringBuffer& message, bool overwrite, const char* user) = 0;
     virtual bool deleteDefinition(const char* definitionId, StringBuffer& errmsg, StringBuffer* defxml) = 0;

esp/services/esdl_svc_engine/esdl_svc_custom.cpp

@@ -26,7 +26,9 @@ CEsdlCustomTransformChoose::CEsdlCustomTransformChoose(IPropertyTree * choosewhe
 {
     if (choosewhen)
     {
-        IPropertyTree * whentree = choosewhen->queryPropTree("xsdl:when");
+        if (choosewhen->hasProp("@_crtTarget"))
+            crtTarget.set(choosewhen->queryProp("@_crtTarget"));
+        IPropertyTree * whentree = choosewhen->queryPropTree("xsdl:when"); //should support multiple when statements
         if (whentree)
         {
             StringBuffer testatt;
@@ -110,20 +112,20 @@ void CEsdlCustomTransformChoose::processClauses(IEspContext * context, IProperty
     }
 }
 
-void CEsdlCustomTransformChoose::processChildClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext,  bool otherwise)
+void CEsdlCustomTransformChoose::processChildClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext,  bool otherwise, IPropertyTree *origTree)
 {
     if (!otherwise)
     {
         ForEachItemIn(currNestedConditionalIndex, m_childChooseClauses)
         {
-            m_childChooseClauses.item(currNestedConditionalIndex).process(context, request, xpathContext);
+            m_childChooseClauses.item(currNestedConditionalIndex).process(context, request, xpathContext, origTree, nullptr);
         }
     }
     else
     {
         ForEachItemIn(currNestedConditionalIndex, m_childOtherwiseClauses)
         {
-            m_childOtherwiseClauses.item(currNestedConditionalIndex).process(context, request, xpathContext);
+            m_childOtherwiseClauses.item(currNestedConditionalIndex).process(context, request, xpathContext, origTree, nullptr);
         }
     }
 }
@@ -208,14 +210,36 @@ bool CEsdlCustomTransformChoose::evaluate(IXpathContext * xpathContext)
     return evalresp;
 }
 
-void CEsdlCustomTransformChoose::process(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext)
+void CEsdlCustomTransformChoose::process(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext, IPropertyTree *origTree, const char *defaultTarget)
 {
+    StringBuffer xpath;
+
+    if (crtTarget.length())
+        xpath.set(crtTarget.str());
+    else if (defaultTarget && *defaultTarget)
+        xpath.set(defaultTarget);
+
+    IPropertyTree *reqTarget = request;
+
+    if (xpath.length())
+    {
+        //we can use real xpath processing in the future, for now simple substitution is fine
+        xpath.replaceString("{$query}", xpathContext->getVariable("query"));
+        xpath.replaceString("{$method}", xpathContext->getVariable("method"));
+        xpath.replaceString("{$service}", xpathContext->getVariable("service"));
+        xpath.replaceString("{$request}", xpathContext->getVariable("request"));
+
+        reqTarget = origTree->queryPropTree(xpath.str());  //get pointer to the write-able area
+        if (!reqTarget)
+            throw MakeStringException(-1, "EsdlCustomTransformChoose::process error getting request target %s", xpath.str());
+    }
+
     bool result = false;
     try
     {
         bool result = evaluate(xpathContext);
-        processClauses(context, request, xpathContext, !result);
-        processChildClauses(context, request, xpathContext, !result);
+        processClauses(context, reqTarget, xpathContext, !result);
+        processChildClauses(context, reqTarget, xpathContext, !result, origTree);
     }
     catch (...)
     {
@@ -307,6 +331,7 @@ void CEsdlCustomTransform::processTransform(IEspContext * context, IPropertyTree
         xpathContext->addVariable("query", tgtcfg->queryProp("@queryname"));
         xpathContext->addVariable("method", mthdef.queryMethodName());
         xpathContext->addVariable("service", srvdef.queryName());
+        xpathContext->addVariable("request", mthdef.queryRequestType());
 
         auto user = context->queryUser();
         if (user)
@@ -363,24 +388,16 @@ void CEsdlCustomTransform::processTransform(IEspContext * context, IPropertyTree
         }
 
         Owned<IPropertyTree> theroot = createPTreeFromXMLString(request.str());
-        StringBuffer xpath = m_target.str();
-        if (!xpath.length())
-        {
+        StringBuffer defaultTarget;
             //This default gives us backward compatibility with only being able to write to the actual request
-            const char *tgtQueryName = tgtcfg->queryProp("@queryname");
-            xpath.setf("soap:Body/%s/%s", tgtQueryName ? tgtQueryName : mthdef.queryMethodName(), mthdef.queryRequestType());
-        }
-        //we can use real xpath processing in the future, for now simple substitution is fine
-        xpath.replaceString("{$query}", tgtcfg->queryProp("@queryname"));
-        xpath.replaceString("{$method}", mthdef.queryMethodName());
-        xpath.replaceString("{$service}", srvdef.queryName());
-        IPropertyTree *thereq = theroot->queryPropTree(xpath.str());  //get pointer to the write-able area
+        const char *tgtQueryName = tgtcfg->queryProp("@queryname");
+        defaultTarget.setf("soap:Body/%s/%s", tgtQueryName ? tgtQueryName : mthdef.queryMethodName(), mthdef.queryRequestType());
+
         ForEachItemIn(currConditionalIndex, m_customTransformClauses)
-        {
-            m_customTransformClauses.item(currConditionalIndex).process(context, thereq, xpathContext);
-        }
+            m_customTransformClauses.item(currConditionalIndex).process(context, theroot, xpathContext, theroot, defaultTarget);
+
         toXML(theroot, request.clear());
 
-        ESPLOG(LogMax,"MODIFIED REQUEST: %s", request.str());
+        ESPLOG(1,"MODIFIED REQUEST: %s", request.str());
     }
 }

esp/services/esdl_svc_engine/esdl_svc_custom.hpp

@@ -99,6 +99,7 @@ private:
     CIArrayOf<CEsdlCustomTransformRule> m_otherwiseClauses;
     CIArrayOf<CEsdlCustomTransformChoose> m_childChooseClauses;
     CIArrayOf<CEsdlCustomTransformChoose> m_childOtherwiseClauses;
+    StringAttr crtTarget;
 
 public:
     IMPLEMENT_IINTERFACE;
@@ -110,7 +111,7 @@ public:
         DBGLOG("CEsdlCustomTransformClause released!");
 #endif
     }
-    void process(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext);
+    void process(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext, IPropertyTree *origTree, const char *defTarget);
 #if defined(_DEBUG)
     void toDBGLog();
 #endif
@@ -124,7 +125,7 @@ private:
     bool evaluate(IXpathContext * xpathContext);
     void processClauses(IPropertyTree *request, IXpathContext * xpathContext, CIArrayOf<CEsdlCustomTransformRule> & m_fieldTransforms);
     void processClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext, bool otherwise);
-    void processChildClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext,  bool otherwise);
+    void processChildClauses(IEspContext * context, IPropertyTree *request, IXpathContext * xpathContext,  bool otherwise, IPropertyTree *origTree);
 };
 
 interface IEsdlCustomTransform : extends IInterface

esp/services/ws_esdlconfig/ws_esdlconfigservice.cpp

@@ -26,6 +26,12 @@
 #include "esdl_binding.hpp"
 #include "TpWrapper.hpp"
 
+enum ESDLConfigInfoType
+{
+    ESDLMethod = 0,
+    ESDLLogTransform = 1,
+};
+
 IPropertyTree * fetchConfigInfo(const char * config,
                                 StringBuffer & espProcName,
                                 StringBuffer & espBindingName,
@@ -117,9 +123,15 @@ IPropertyTree * fetchConfigInfo(const char * config,
 }
 
 IPropertyTree * fetchConfigInfo(const char * config,
-                                const char* bindingId)
+                                const char* bindingId,
+                                ESDLConfigInfoType infoType = ESDLMethod)
 {
-    IPropertyTree * methodstree = NULL;
+    IPropertyTree * retTree = nullptr;
+    StringAttr treeName;
+    if (infoType == ESDLMethod)
+        treeName.set("Methods");
+    else
+        treeName.set("LogTransforms");
 
     if (!config || !*config)
     {
@@ -127,7 +139,7 @@ IPropertyTree * fetchConfigInfo(const char * config,
     }
     else
     {
-        Owned<IPropertyTree>  configTree = createPTreeFromXMLString(config, ipt_caseInsensitive);
+        Owned<IPropertyTree>  configTree = createPTreeFromXMLString(config, ipt_caseInsensitive | ipt_ordered);
         //Now let's figure out the structure of the configuration passed in...
 
         StringBuffer rootname;
@@ -146,17 +158,15 @@ IPropertyTree * fetchConfigInfo(const char * config,
             deftree = configTree->queryBranch("Definition[1]");
 
         if (deftree)
-        {
-            methodstree = deftree->getBranch("Methods");
-        }
+            retTree = deftree->getBranch(treeName.get());
 
-        if (!methodstree) //if we didn't already find the methods section of the config, let's look at the root
+        if (!retTree) //if we didn't already find the methods or log-transforms section of the config, let's look at the root
         {
-            if (stricmp(rootname.str(), "Methods") == 0)
-                methodstree = configTree.getLink();
+            if (stricmp(rootname.str(), treeName.get()) == 0)
+                retTree = configTree.getLink();
         }
     }
-    return methodstree;
+    return retTree;
 }
 
 void CWsESDLConfigEx::init(IPropertyTree *cfg, const char *process, const char *service)
@@ -949,6 +959,118 @@ bool CWsESDLConfigEx::onConfigureESDLBindingMethod(IEspContext &context, IEspCon
     return true;
 }
 
+bool CWsESDLConfigEx::onConfigureESDLBindingLogTransform(IEspContext &context, IEspConfigureESDLBindingLogTransformRequest &req, IEspConfigureESDLBindingLogTransformResponse &resp)
+{
+    int success = 0;
+    try
+    {
+        if (m_isDetachedFromDali)
+            throw MakeStringException(-1, "Cannot Configure ESDL Binding LogTransform. ESP is currently detached from DALI.");
+
+        context.ensureFeatureAccess(FEATURE_URL, SecAccess_Write, ECLWATCH_ROXIE_QUERY_ACCESS_DENIED, "WsESDLConfigEx::ConfigureESDLBindingLogTransform: Permission denied.");
+
+        StringBuffer username;
+        context.getUserID(username);
+        DBGLOG("CWsESDLConfigEx::onConfigureESDBindingLogTransform User=%s",username.str());
+
+        const char* bindingId = req.getEsdlBindingId();
+        const char* logTransformName = req.getLogTransformName();
+        const char* logTransform = req.getConfig();
+        if (isEmptyString(logTransform))
+            throw MakeStringException(-1, "Config not defined.");
+
+        StringBuffer espProcName;
+        StringBuffer espBindingName;
+        StringBuffer esdlServiceName;
+        StringBuffer esdlDefIdSTR;
+        StringBuffer config;
+        if (strncmp(logTransform, "<LogTransforms>", 15) == 0)
+            config.set(logTransform);
+        else if (isEmptyString(logTransformName))
+            throw MakeStringException(-1, "LogTransformName not defined");
+        else
+        {
+            config.appendf("<LogTransforms><LogTransform name='%s'>", logTransformName);
+            if (req.getEncoded())
+                config.append(logTransform);
+            else
+            {
+                StringBuffer encodedLogTransform;
+                encodeXML(logTransform, encodedLogTransform);
+                config.append(encodedLogTransform.str());
+            }
+            config.append("</LogTransform></LogTransforms>");
+        }
+
+        StringBuffer msg;
+        Owned<IPropertyTree> bindingtree = m_esdlStore->getBindingTree(bindingId, msg);
+        if(!bindingtree)
+            throw MakeStringException(-1, "Can't find esdl binding for id %s", bindingId);
+
+        bindingtree->getProp("@espprocess", espProcName);
+        bindingtree->getProp("@espbinding", espBindingName);
+        bindingtree->getProp("Definition[1]/@esdlservice", esdlServiceName);
+        bindingtree->getProp("Definition[1]/@id", esdlDefIdSTR);
+
+        Owned<IPropertyTree> logTransformTree = fetchConfigInfo(config, bindingId, ESDLLogTransform);
+        if (!logTransformTree || logTransformTree->getCount("LogTransform") <= 0)
+            throw MakeStringException(-1, "Could not find any LogTransform configuration entries.");
+
+        const char *esdlDefId = esdlDefIdSTR.str();
+        if (isEmptyString(esdlDefId))
+            throw MakeStringException(-1, "Can't find esdl definition for binding %s", bindingId);
+
+        StringBuffer esdlDefinitionName;
+        while (esdlDefId && *esdlDefId != '.')
+            esdlDefinitionName.append(*esdlDefId++);
+
+        if (isEmptyString(esdlDefId))
+            throw MakeStringException(-1, "Invalid ESDL Definition ID format detected: '%s'. Expected format: <esdldefname>.<ver>", esdlDefIdSTR.str());
+
+        int esdlver = 0;
+        esdlDefId++;
+        if (esdlDefId)
+            esdlver = atoi(esdlDefId);
+
+        if (esdlver <= 0)
+            throw MakeStringException(-1, "Invalid ESDL Definition version detected: %d", esdlver);
+
+        if (!m_esdlStore->definitionExists(esdlDefIdSTR.str()))
+            throw MakeStringException(-1, "Invalid ESDL Definition: %s", esdlDefIdSTR.str());
+
+        bool override = req.getOverwrite();
+        StringBuffer status;
+        StringBuffer logTransformXPath;
+        if (!isEmptyString(logTransformName))
+            logTransformXPath.appendf("LogTransform[@name='%s']", logTransformName);
+        else
+            logTransformXPath.append("LogTransform");
+        Owned<IPropertyTreeIterator> iter = logTransformTree->getElements(logTransformXPath.str());
+        ForEach(*iter)
+        {
+            IPropertyTree &item = iter->query();
+            const char* ltName = item.queryProp("@name");
+            StringBuffer msg;
+            success = m_esdlStore->configureLogTransform(bindingId, logTransformName, LINK(&item), override, msg);
+            status.appendf("%s: %s; ", ltName, msg.str());
+        }
+        resp.updateStatus().setDescription(status.str());
+
+        resp.setEspProcName(espProcName.str());
+        resp.setEspBindingName(espBindingName.str());
+        resp.setEsdlDefinitionID(esdlDefIdSTR.str());
+        resp.setEsdlServiceName(esdlServiceName.str());
+    }
+    catch(IException* e)
+    {
+       FORWARDEXCEPTION(context, e, -1);
+    }
+
+    resp.updateStatus().setCode(success);
+
+    return true;
+}
+
 int CWsESDLConfigEx::getBindingXML(const char * bindingId, StringBuffer & bindingXml, StringBuffer & msg)
 {
     Owned<IPropertyTree> esdlBinding = m_esdlStore->getBindingTree(bindingId, msg);

esp/services/ws_esdlconfig/ws_esdlconfigservice.hpp

@@ -48,6 +48,7 @@ public:
     bool onPublishESDLDefinition(IEspContext &context, IEspPublishESDLDefinitionRequest &req, IEspPublishESDLDefinitionResponse &resp);
     bool onPublishESDLBinding(IEspContext &context, IEspPublishESDLBindingRequest &req, IEspPublishESDLBindingResponse &resp);
     bool onConfigureESDLBindingMethod(IEspContext &context, IEspConfigureESDLBindingMethodRequest &req, IEspConfigureESDLBindingMethodResponse &resp);
+    bool onConfigureESDLBindingLogTransform(IEspContext &context, IEspConfigureESDLBindingLogTransformRequest &req, IEspConfigureESDLBindingLogTransformResponse &resp);
     bool onDeleteESDLBinding(IEspContext &context, IEspDeleteESDLBindingRequest &req, IEspDeleteESDLRegistryEntryResponse &resp);
     bool onDeleteESDLDefinition(IEspContext &context, IEspDeleteESDLDefinitionRequest &req, IEspDeleteESDLRegistryEntryResponse &resp);
     bool onGetESDLDefinition(IEspContext &context, IEspGetESDLDefinitionRequest&req, IEspGetESDLDefinitionResponse &resp);

esp/src/eclwatch/ActivityWidget.js

@@ -28,6 +28,33 @@ define([
     registry, Button, ToggleButton, ToolbarSeparator, ContentPane, Tooltip,
     selector, tree,
     GridDetailsWidget, ESPRequest, ESPActivity, DelayLoadWidget, ESPUtil, Utility) {
+
+        var DelayedRefresh = declare("DelayedRefresh", [], {
+            _activityWidget: null,
+            _promises: null,
+
+            constructor: function (activityWidget) {
+                this._activityWidget = activityWidget;
+                this._promises = [];
+            },
+
+            push: function (promise) {
+                this._promises.push(promise);
+            },
+
+            refresh: function () {
+                if (this._promises.length) {
+                    var context = this;
+                    Promise.all(this._promises).then(function () {
+                        context._activityWidget.refreshGrid();
+                        setTimeout(function() {
+                            context._activityWidget._refreshActionState();
+                        }, 100);
+                    });
+                }
+            }
+        });
+
         return declare("ActivityWidget", [GridDetailsWidget], {
 
             i18n: nlsHPCC,
@@ -46,174 +73,147 @@ define([
             },
 
             _onPause: function (event, params) {
+                var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfQueue(item)) {
-                        var context = this;
-                        item.pause().then(function (response) {
-                            context._refreshActionState();
-                        });
-                    }
-                    if (item.ServerType === "ECLserver" || "ECLCCserver") {
-                        setTimeout(function () {
-                            context.refreshGrid();
-                        }, 100);
+                        promises.push(item.pause());
                     }
                 }, this);
+                promises.refresh();
             },
 
             _onResume: function (event, params) {
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfQueue(item)) {
-                        var context = this;
-                        item.resume().then(function (response) {
-                            context._refreshActionState();
-                        });
-                    }
-                    if (item.ServerType === "ECLserver" || "ECLCCserver") {
-                        setTimeout(function () {
-                            context.refreshGrid();
-                        }, 100);
+                        promises.push(item.resume());
                     }
                 }, this);
+                promises.refresh();
             },
 
             _onClear: function (event, params) {
-                var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfQueue(item)) {
-                        item.clear();
+                        promises.push(item.clear());
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUPause: function (event, params) {
-                var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfWorkunit(item)) {
-                        item.pause();
+                        promises.push(item.pause());
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUPauseNow: function (event, params) {
-                var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfWorkunit(item)) {
-                        item.pauseNow();
+                        promises.push(item.pauseNow());
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUResume: function (event, params) {
-                var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfWorkunit(item)) {
-                        item.resume();
+                        promises.push(item.resume());
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUAbort: function (event, params) {
-                var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfWorkunit(item)) {
-                        item.abort();
+                        promises.push(item.abort());
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUPriority: function (event, priority) {
-                var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfWorkunit(item)) {
                         var queue = item.get("ESPQueue");
                         if (queue) {
-                            queue.setPriority(item.Wuid, priority);
+                            promises.push(queue.setPriority(item.Wuid, priority));
                         }
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUTop: function (event, params) {
                 var context = this;
+                var promises = new DelayedRefresh(this);
                 var selected = this.grid.getSelected();
                 for (var i = selected.length - 1; i >= 0; --i) {
                     var item = selected[i];
                     if (this.activity.isInstanceOfWorkunit(item)) {
                         var queue = item.get("ESPQueue");
                         if (queue) {
-                            queue.moveTop(item.Wuid);
+                            promises.push(queue.moveTop(item.Wuid));
                         }
                     }
                 }
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUUp: function (event, params) {
                 var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfWorkunit(item)) {
                         var queue = item.get("ESPQueue");
                         if (queue) {
-                            queue.moveUp(item.Wuid);
+                            promises.push(queue.moveUp(item.Wuid));
                         }
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUDown: function (event, params) {
                 var context = this;
+                var promises = new DelayedRefresh(this);
                 var selected = this.grid.getSelected();
                 for (var i = selected.length - 1; i >= 0; --i) {
                     var item = selected[i];
                     if (this.activity.isInstanceOfWorkunit(item)) {
                         var queue = item.get("ESPQueue");
                         if (queue) {
-                            queue.moveDown(item.Wuid);
+                            promises.push(queue.moveDown(item.Wuid));
                         }
                     }
                 }
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             _onWUBottom: function (event, params) {
                 var context = this;
+                var promises = new DelayedRefresh(this);
                 arrayUtil.forEach(this.grid.getSelected(), function (item, idx) {
                     if (this.activity.isInstanceOfWorkunit(item)) {
                         var queue = item.get("ESPQueue");
                         if (queue) {
-                            queue.moveBottom(item.Wuid);
+                            promises.push(queue.moveBottom(item.Wuid));
                         }
                     }
                 }, this);
-                setTimeout(function () {
-                    context.refreshGrid();
-                }, 100);
+                promises.refresh();
             },
 
             doSearch: function (searchText) {
@@ -391,8 +391,8 @@ define([
                             width: 300,
                             sortable: true,
                             shouldExpand: function (row, level, previouslyExpanded) {
-                                if (context.firstLoad === true) {
-                                    return true;
+                                if (level === 0) {
+                                    return previouslyExpanded === undefined ? true : previouslyExpanded;
                                 }
                                 return previouslyExpanded;
                             },

esp/src/eclwatch/GraphsWUWidget.js

@@ -58,6 +58,7 @@ define([
 
                     this.timeline = new hpccEclWatch.WUTimeline()
                         .target(this.id + "TimelinePane")
+                        .maxZoom(Number.MAX_SAFE_INTEGER)
                         .overlapTolerence(1)
                         .baseUrl("")
                         .wuid(params.Wuid)

esp/src/src/ESPActivity.ts

@@ -116,25 +116,39 @@ var Activity = declare([ESPUtil.Singleton, ESPUtil.Monitor], {
         var context = this;
         if (responseTargetClusters) {
             arrayUtil.forEach(responseTargetClusters, function (item, idx) {
-                var queue = null;
-                if (item.ClusterName) {
-                    queue = ESPQueue.GetTargetCluster(item.ClusterName);
+                if (lang.exists("Queues.ServerJobQueue", item)) {
+                    arrayUtil.forEach(item.Queues.ServerJobQueue, function (queueItem) {
+                        context.refreshTargetCluster(item, queueItem, targetClusters, targetClusterMap);
+                    });
                 } else {
-                    queue = ESPQueue.GetServerJobQueue(item.ServerName);
+                    context.refreshTargetCluster(item, undefined, targetClusters, targetClusterMap);
                 }
-                queue.updateData(item);
-                queue.set("DisplayName", queue.getDisplayName());
-                queue.clearChildren();
-                targetClusters.push(queue);
-                targetClusterMap[queue.__hpcc_id] = queue;
-                if (!context._watched[queue.__hpcc_id]) {
-                    context._watched[queue.__hpcc_id] = queue.watch("__hpcc_changedCount", function (name, oldValue, newValue) {
-                        if (oldValue !== newValue) {
-                            if (context.observableStore.get(queue.__hpcc_id)) {
-                                context.observableStore.notify(queue, queue.__hpcc_id);
-                            }
-                        }
-                    });
+            });
+        }
+    },
+
+    refreshTargetCluster: function (item, queueItem, targetClusters, targetClusterMap) {
+        var queue = null;
+        if (item.ClusterName) {
+            queue = ESPQueue.GetTargetCluster(item.ClusterName, true);
+        } else {
+            queue = ESPQueue.GetServerJobQueue(queueItem ? queueItem.QueueName : item.ServerName, true);
+        }
+        queue.updateData(item);
+        if (queueItem) {
+            queue.updateData(queueItem);
+        }
+        queue.set("DisplayName", queue.getDisplayName());
+        queue.clearChildren();
+        targetClusters.push(queue);
+        targetClusterMap[queue.__hpcc_id] = queue;
+        var context = this;
+        if (!this._watched[queue.__hpcc_id]) {
+            this._watched[queue.__hpcc_id] = queue.watch("__hpcc_changedCount", function (name, oldValue, newValue) {
+                if (oldValue !== newValue) {
+                    if (context.observableStore.get(queue.__hpcc_id)) {
+                        context.observableStore.notify(queue, queue.__hpcc_id);
+                    }
                 }
             });
         }
@@ -145,16 +159,23 @@ var Activity = declare([ESPUtil.Singleton, ESPUtil.Monitor], {
             arrayUtil.forEach(responseActiveWorkunits, function (item, idx) {
                 item["__hpcc_id"] = item.Wuid;
                 var queue = null;
-                if (item.ClusterName) {
-                    queue = ESPQueue.GetTargetCluster(item.ClusterName);
-                } else {
-                    queue = ESPQueue.GetServerJobQueue(item.ServerName);
+                if (item.QueueName) {
+                    queue = ESPQueue.GetServerJobQueue(item.QueueName);
+                }
+                if (!queue) {
+                    if (item.ClusterName) {
+                        queue = ESPQueue.GetTargetCluster(item.ClusterName);
+                    } else {
+                        queue = ESPQueue.GetServerJobQueue(item.ServerName);
+                    }
                 }
                 var wu = item.Server === "DFUserver" ? ESPDFUWorkunit.Get(item.Wuid) : ESPWorkunit.Get(item.Wuid);
                 wu.updateData(lang.mixin({
                     __hpcc_id: item.Wuid
                 }, item));
-                queue.addChild(wu);
+                if (!wu.isComplete || !wu.isComplete()) {
+                    queue.addChild(wu);
+                }
             });
         }
     },

esp/src/src/ESPQueue.ts

@@ -1,5 +1,6 @@
 import * as declare from "dojo/_base/declare";
 import * as lang from "dojo/_base/lang";
+import * as arrayUtil from "dojo/_base/array";
 import * as Memory from "dojo/store/Memory";
 
 import * as WsSMC from "./WsSMC";
@@ -57,7 +58,10 @@ var Queue = declare([ESPUtil.Singleton, ESPUtil.Monitor], {
         var context = this;
         return WsSMC.ClearQueue({
             request: {
-                QueueName: this.QueueName
+                QueueName: this.QueueName,
+                ServerType: this.ServerType,
+                NetworkAddress: this.NetworkAddress,
+                Port: this.Port
             }
         }).then(function (response) {
             context.clearChildren();
@@ -200,7 +204,7 @@ var TargetCluster = declare([Queue], {
     },
 
     getDisplayName: function () {
-        return this.ClusterName;
+        return this.ServerType + (this.ClusterName ? " - " + this.ClusterName : "");
     },
 
     isNormal: function () {
@@ -268,15 +272,20 @@ var ServerJobQueue = declare([Queue], {
                 NetworkAddress: this.NetworkAddress
             }
         }).then(function (response) {
-            if (lang.exists("GetStatusServerInfoResponse.StatusServerInfo.ServerInfo", response)) {
-                context.updateData(response.GetStatusServerInfoResponse.StatusServerInfo.ServerInfo);
+            if (lang.exists("GetStatusServerInfoResponse.StatusServerInfo.ServerInfo.Queues.ServerJobQueue", response)) {
+                arrayUtil.forEach(response.GetStatusServerInfoResponse.StatusServerInfo.ServerInfo.Queues.ServerJobQueue, function (queueItem) {
+                    if (queueItem.QueueName === context.QueueName) {
+                        context.updateData(response.GetStatusServerInfoResponse.StatusServerInfo.ServerInfo);
+                        context.updateData(queueItem);
+                    }
+                });
             }
             return response;
         });
     },
 
     getDisplayName: function () {
-        return this.ServerName;
+        return this.ServerName + (this.QueueName ? " - " + this.QueueName : "");
     },
 
     isNormal: function () {
@@ -340,22 +349,22 @@ export function isInstanceOfQueue(obj) {
     return obj && obj.isInstanceOf && obj.isInstanceOf(Queue);
 }
 
-export function GetTargetCluster(name) {
+export function GetTargetCluster(name, createIfMissing = false) {
     var store = GetGlobalQueueStore();
     var id = "TargetCluster::" + name;
     var retVal = store.get(id);
-    if (!retVal) {
+    if (!retVal && createIfMissing) {
         retVal = new TargetCluster(id);
         store.put(retVal);
     }
     return retVal;
 }
 
-export function GetServerJobQueue(name) {
+export function GetServerJobQueue(name, createIfMissing = false) {
     var store = GetGlobalQueueStore();
     var id = "ServerJobQueue::" + name;
     var retVal = store.get(id);
-    if (!retVal) {
+    if (!retVal && createIfMissing) {
         retVal = new ServerJobQueue(id);
         store.put(retVal);
     }

esp/src/src/ESPRequest.ts

@@ -461,9 +461,6 @@ export const Store = declare(null, {
             } else {
                 deferredResults.total.resolve(0);
             }
-            if (deferredResults.isResolved()) {
-                debugger;
-            }
             deferredResults.resolve(items);
             return response;
         });

esp/src/src/ESPUtil.ts

@@ -69,7 +69,7 @@ var SingletonData = declare([Stateful], {
     updateData: function (response) {
         var changed = false;
         for (var key in response) {
-            if (response[key] !== undefined || response[key] !== null) {
+            if (response[key] !== undefined && response[key] !== null) {
                 var jsonStr = json.stringify(response[key]);
                 if (this.__hpcc_changedCache[key] !== jsonStr) {
                     this.__hpcc_changedCache[key] = jsonStr;

esp/src/src/ESPWorkunit.ts

@@ -622,8 +622,12 @@ var Workunit = declare([ESPUtil.Singleton, ESPUtil.Monitor], {  // jshint ignore
         if (this.Archived) {
             return "iconArchived";
         }
+        if (this.isComplete()) {
+            return "iconCompleted";
+        }
         switch (this.StateID) {
             case 1:
+                return "iconSubmitted";
             case 3:
                 return "iconCompleted";
             case 2:
@@ -654,9 +658,12 @@ var Workunit = declare([ESPUtil.Singleton, ESPUtil.Monitor], {  // jshint ignore
         if (this.Archived) {
             return "workunit_archived.png";
         }
+        if (this.isComplete()) {
+            return "workunit_completed.png";
+        }
         switch (this.StateID) {
             case 1:
-                return "workunit_completed.png";
+                return "workunit_submitted.png";
             case 2:
                 return "workunit_running.png";
             case 3:

esp/src/src/Timings.ts

@@ -28,6 +28,7 @@ export class Timings {
     private wu: Workunit;
 
     private timeline = new WUTimeline()
+        .maxZoom(Number.MAX_SAFE_INTEGER)
         .overlapTolerence(1)
         .baseUrl("")
         .request({
@@ -295,7 +296,7 @@ export class Timings {
             this.chart._columnsMetric = {};
             const columns = ["id", ...this._metricSelectValues.map(mv => {
                 const retVal = `${mv}(${this._rawColumns[mv].Measure})`;
-                this.chart._columnsMetric[retVal] = this._rawColumns[mv]; 
+                this.chart._columnsMetric[retVal] = this._rawColumns[mv];
                 return retVal;
             })];
 

esp/src/src/WUScopeController.ts

@@ -306,8 +306,11 @@ export class WUScopeController {
     }
 
     createVertex(vertex: ScopeVertex): VertexType {
-        let v = this.verticesMap[vertex._.Id];
         const attrs = vertex._.rawAttrs();
+        attrs["ID"] = vertex._.Id;
+        attrs["Parent ID"] = vertex.parent && vertex.parent._.Id;
+        attrs["Scope"] = vertex._.ScopeName;
+        let v = this.verticesMap[vertex._.Id];
         if (!v) {
             if (vertex._.ScopeType === "dummy") {
                 const parent = this.subgraphsMap[vertex.parent._.Id];
@@ -376,9 +379,12 @@ export class WUScopeController {
     }
 
     createEdge(edge: ScopeEdge): Edge | undefined {
+        const attrs = edge._.rawAttrs();
+        attrs["ID"] = edge._.Id;
+        attrs["Parent ID"] = edge.parent && edge.parent._.Id;
+        attrs["Scope"] = edge._.ScopeName;
         let e = this.edgesMap[edge._.Id];
         if (!e) {
-            const attrs = edge._.rawAttrs();
             const sourceV = this.verticesMap[edge.source._.Id];
             const targetV = this.verticesMap[edge.target._.Id];
             if (sourceV && targetV) {
@@ -411,6 +417,10 @@ export class WUScopeController {
                 this.rEdgesMap[e.id()] = edge;
             }
         }
+        if (e instanceof Edge) {
+            const label = this.format(this.edgeLabelTpl(), attrs);
+            e.text(label);
+        }
         return e;
     }
 
@@ -438,9 +448,6 @@ export class WUScopeController {
         const e = this.createEdge(edge);
         if (e) {
             const attrs = edge._.rawAttrs();
-            const formattedAttrs = edge._.formattedAttrs();
-            const label = this.format(this.edgeLabelTpl(), formattedAttrs);
-            e.text(label)
             const numSlaves = parseInt(attrs["NumSlaves"]);
             const numStarts = parseInt(attrs["NumStarts"]);
             const numStops = parseInt(attrs["NumStops"]);

esp/src/tsconfig.json

@@ -17,7 +17,7 @@
         "lib": [
             "dom",
             "es5",
-            "es2015.promise"
+            "es2015"
         ],
         "typeRoots": [],
         "types": []

initfiles/componentfiles/configxml/@temp/wslogserviceespagent.xsl

@@ -36,8 +36,7 @@ xmlns:set="http://exslt.org/sets">
         <xsl:if test="string($loggingServerUrl) = ''">
             <xsl:message terminate="yes">Logging Server URL is undefined for <xsl:value-of select="$agentName"/>!</xsl:message>
         </xsl:if>
-        <xsl:variable name="logDataXPath" select="$agentNode/LogDataXPath"/>
-        <xsl:if test="not($logDataXPath)">
+        <xsl:if test="not($agentNode/LogDataItem[1]) and not($agentNode/LogInfo[1])">
             <xsl:message terminate="yes">Log Data XPath is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
         </xsl:if>
 
@@ -73,44 +72,11 @@ xmlns:set="http://exslt.org/sets">
             </xsl:call-template>
                 
             <LogDataXPath>
-                <xsl:if test="string($logDataXPath/@IP) != ''">
-                    <IP><xsl:value-of select="$logDataXPath/@IP"/></IP>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@UserName) != ''">
-                    <UserName><xsl:value-of select="$logDataXPath/@UserName"/></UserName>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@ServiceName) != ''">
-                    <ServiceName><xsl:value-of select="$logDataXPath/@ServiceName"/></ServiceName>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@RecordCount) != ''">
-                    <RecordCount><xsl:value-of select="$logDataXPath/@RecordCount"/></RecordCount>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@DomainName) != ''">
-                    <DomainName><xsl:value-of select="$logDataXPath/@DomainName"/></DomainName>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@GUID) != ''">
-                    <GUID><xsl:value-of select="$logDataXPath/@GUID"/></GUID>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@BlindLogging) != ''">
-                    <BlindLogging><xsl:value-of select="$logDataXPath/@BlindLogging"/></BlindLogging>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@EncryptedLogging) != ''">
-                    <EncryptedLogging><xsl:value-of select="$logDataXPath/@EncryptedLogging"/></EncryptedLogging>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@ForwardLog) != ''">
-                    <ForwardLog><xsl:value-of select="$logDataXPath/@ForwardLog"/></ForwardLog>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@RawLogInformation) != ''">
-                    <RawLogInformation><xsl:value-of select="$logDataXPath/@RawLogInformation"/></RawLogInformation>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@CompressBlobs) != ''">
-                    <CompressBlobs><xsl:value-of select="$logDataXPath/@CompressBlobs"/></CompressBlobs>
-                </xsl:if>
-                <xsl:if test="string($logDataXPath/@WorkunitID) != ''">
-                    <WorkunitID><xsl:value-of select="$logDataXPath/@WorkunitID"/></WorkunitID>
-                </xsl:if>
+                <xsl:for-each select="$agentNode/LogDataItem">
+                    <LogDataItem name="{current()/@name}" XPath="{current()/@xpath}" xsl="{current()/@xsl}" default="{current()/@default}"/>
+                </xsl:for-each>
                 <xsl:for-each select="$agentNode/LogInfo">
-                    <LogInfo name="{current()/@name}" valueXPath="{current()/@valueXPath}" dataXPath="{current()/@dataXPath}" multipleValue="{current()/@multipleValue}" multipleData="{current()/@multipleData}" encodeValue="{current()/@encodeValue}" encodeData="{current()/@encodeData}"/>
+                    <LogInfo name="{current()/@name}" default="{current()/@default}" XPath="{current()/@xpath}" xsl="{current()/@xsl}" multiple="{current()/@multiple}" encode="{current()/@encode}" type="{current()/@type}"/>
                 </xsl:for-each>
             </LogDataXPath>
         </LogAgent>

lib2/CMakeLists.txt

@@ -78,7 +78,7 @@ foreach(dylib ${DYLIBS})
         install(CODE "
            file(GLOB files \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${EXEC_DIR}/*\" \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/${LIB_DIR}/*.dylib\" \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/plugins/*.dylib\" \"\$ENV{DESTDIR}\${CMAKE_INSTALL_PREFIX}/lib2/*.dylib\")
            foreach(file \${files})
-               execute_process(COMMAND bash \"-c\"  \"otool -L \\\"\${file}\\\" | egrep \\\"/${dylib_name_only}(.[0-9]{1,})*.dylib\\\" | sed \\\"s/^[[:space:]]//g\\\" | cut -d' ' -f1\"
+               execute_process(COMMAND bash \"-c\"  \"otool -L \\\"\${file}\\\" | egrep \\\"(\\\\s|/)${dylib_name_only}(.[0-9]{1,})*.dylib\\\" | sed \\\"s/^[[:space:]]//g\\\" | cut -d' ' -f1\"
                   OUTPUT_VARIABLE otoolOut
                   ERROR_VARIABLE  otoolErr
                   OUTPUT_STRIP_TRAILING_WHITESPACE

system/jhtree/jhtree.cpp

@@ -276,6 +276,12 @@ void SegMonitorList::finish(unsigned keyedSize)
         {
             unsigned idx = segMonitors.length();
             size32_t offset = recInfo.getFixedOffset(idx);
+            if (offset == keyedSize)
+            {
+                DBGLOG("SegMonitor record does not match key");  // Can happen when reading older indexes that don't save key information in metadata properly
+                keySegCount - segMonitors.length();
+                break;
+            }
             size32_t size = recInfo.getFixedOffset(idx+1) - offset;
             segMonitors.append(*createWildKeySegmentMonitor(idx, offset, size));
         }

thorlcr/activities/keyedjoin/thkeyedjoinslave.cpp

@@ -741,23 +741,26 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
     {
         typedef CLookupHandler PARENT;
     protected:
-        Owned<const ITranslator> translator;
+        std::vector<Owned<const ITranslator>> translators;
 
-        void setupTranslation(unsigned partNo, IKeyManager &keyManager)
+        void setupTranslation(unsigned partNo, unsigned selected, IKeyManager &keyManager)
         {
-            IPartDescriptor &part = activity.allIndexParts.item(partNo);
-            IPropertyTree &props = part.queryOwner().queryProperties();
-            unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
-            Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
-            unsigned expectedFormatCrc = helper->getIndexFormatCrc();
-            unsigned projectedFormatCrc = helper->getProjectedIndexFormatCrc();
-            IOutputMetaData *projectedFormat = helper->queryProjectedIndexRecordSize();
+            if (!translators[selected])
+            {
+                IPartDescriptor &part = activity.allIndexParts.item(partNo);
+                IPropertyTree &props = part.queryOwner().queryProperties();
+                unsigned publishedFormatCrc = (unsigned)props.getPropInt("@formatCrc", 0);
+                Owned<IOutputMetaData> publishedFormat = getDaliLayoutInfo(props);
+                unsigned expectedFormatCrc = helper->getIndexFormatCrc();
+                unsigned projectedFormatCrc = helper->getProjectedIndexFormatCrc();
+                IOutputMetaData *projectedFormat = helper->queryProjectedIndexRecordSize();
 
-            RecordTranslationMode translationMode = getTranslationMode(activity);
-            OwnedRoxieString fname = helper->getIndexFileName();
-            translator.setown(getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode));
-            if (translator)
-                keyManager.setLayoutTranslator(&translator->queryTranslator());
+                RecordTranslationMode translationMode = getTranslationMode(activity);
+                OwnedRoxieString fname = helper->getIndexFileName();
+                translators[selected].setown(getTranslators(fname, expectedFormatCrc, helper->queryIndexRecordSize(), publishedFormatCrc, publishedFormat, projectedFormatCrc, projectedFormat, translationMode));
+            }
+            if (translators[selected])
+                keyManager.setLayoutTranslator(&translators[selected]->queryTranslator());
         }
     public:
         CKeyLookupLocalBase(CKeyedJoinSlave &_activity) : CLookupHandler(_activity, _activity.keyLookupRowWithJGRowIf)
@@ -847,6 +850,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
         {
             PARENT::addPartNum(partNum);
             keyManagers.push_back(nullptr);
+            translators.push_back(nullptr);
         }
         virtual void process(CThorExpandingRowArray &processing, unsigned selected) override
         {
@@ -858,7 +862,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
             {
                 keyManager = activity.createPartKeyManager(partNo, copy);
                 // NB: potentially translation per part could be different if dealing with superkeys
-                setupTranslation(partNo, *keyManager);
+                setupTranslation(partNo, selected, *keyManager);
             }
             processRows(processing, partNo, keyManager);
         }
@@ -872,6 +876,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
         CKeyLookupMergeHandler(CKeyedJoinSlave &_activity) : CKeyLookupLocalBase(_activity)
         {
             limiter = &activity.lookupThreadLimiter;
+            translators.push_back(nullptr);
         }
         virtual void process(CThorExpandingRowArray &processing, unsigned __unused) override
         {
@@ -886,7 +891,7 @@ class CKeyedJoinSlave : public CSlaveActivity, implements IJoinProcessor
                     partKeySet->addIndex(keyIndex.getClear());
                 }
                 keyManager.setown(createKeyMerger(helper->queryIndexRecordSize()->queryRecordAccessor(true), partKeySet, 0, nullptr, helper->hasNewSegmentMonitors()));
-                setupTranslation(0, *keyManager);
+                setupTranslation(0, 0, *keyManager);
             }
             processRows(processing, 0, keyManager);
         }

tools/esdlcmd/esdl-publish.cpp

@@ -1095,6 +1095,314 @@ public:
     }
 };
 
+class EsdlBindLogTransformCmd : public EsdlBindServiceCmd
+{
+protected:
+    StringAttr optLogTransform;
+    StringAttr optBindingId;
+    bool       optEncoded;
+
+public:
+    int processCMD()
+    {
+        Owned<IClientWsESDLConfig> esdlConfigClient = EsdlCmdHelper::getWsESDLConfigSoapService(optWSProcAddress, optWSProcPort, optUser, optPass);
+        Owned<IClientConfigureESDLBindingLogTransformRequest> request = esdlConfigClient->createConfigureESDLBindingLogTransformRequest();
+
+        fprintf(stdout,"\nAttempting to configure LogTransform : '%s' for binding '%s'\n", optLogTransform.get(), optBindingId.get());
+        request->setEsdlBindingId(optBindingId.get());
+        request->setLogTransformName(optLogTransform.get());
+        request->setConfig(optInput);
+        request->setOverwrite(optOverWrite);
+        request->setEncoded(optEncoded);
+
+        if (optVerbose)
+            fprintf(stdout,"\nLogTransform config: %s\n", optInput.get());
+
+        Owned<IClientConfigureESDLBindingLogTransformResponse> resp = esdlConfigClient->ConfigureESDLBindingLogTransform(request);
+
+        if (resp->getExceptions().ordinality()>0)
+        {
+            EsdlCmdHelper::outputMultiExceptions(resp->getExceptions());
+            return 1;
+        }
+
+        outputWsStatus(resp->getStatus().getCode(), resp->getStatus().getDescription());
+
+        return 0;
+    }
+
+    void usage()
+    {
+        printf( "\nUsage:\n\n"
+                "esdl bind-log-transform <TargetESDLBindingID> [TargetLogTransformName] --config <file|xml> [command options]\n\n"
+                "   TargetESDLBindingID                              The id of the target ESDL binding (must exist in dali)\n"
+                "   TargetLogTransformName                           The name of the target LogTransform (required when config is xsl transform)\n"
+                "   --config <file|xml>                              Configuration XML for all LogTransforms associated with the target Service\n"
+
+                "\nOptions (use option flag followed by appropriate value):\n"
+                "   --overwrite                                      Overwrite LogTransform if it already exists\n"
+                "   --encoded                                        The LogTransform in config has been encoded.\n");
+
+                EsdlPublishCmdCommon::usage();
+
+        printf( "\n Use this command to publish ESDL Service based bindings.\n"
+                "   To bind a ESDL Service, provide the id of the esdl binding and the LogTransform name you're going to configure.\n"
+                "   Optionally provide configuration information either directly, or via a\n"
+                "   configuration file in the following syntax:\n"
+                "     <LogTransforms>\n"
+                "     \t<LogTransform name=\"myLogTransform\">escaped transform script</LogTransform>\n"
+                "     \t<LogTransform name=\"myLogTransform2\">escaped transform script</LogTransform>\n"
+                "     </LogTransforms>\n"
+                "   or\n"
+                "     one transform script\n"
+                );
+
+        printf("\nExample:\n"
+                ">esdl bind-log-transform myesp.8003.EsdlExample myLogTransform --config /myService/log-transforms.xml\n"
+                "or:\n"
+                ">esdl bind-log-transform myesp.8003.EsdlExample myLogTransform --config /myService/log-transform.xsl\n"
+                );
+    }
+
+    bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        //First 2 parameter order is fixed.
+        for (int cur = 0; cur < 2 && !iter.done(); cur++)
+        {
+           const char *arg = iter.query();
+           if (*arg != '-')
+           {
+               switch (cur)
+               {
+                case 0:
+                    optBindingId.set(arg);
+                    break;
+                case 1:
+                    optLogTransform.set(arg);
+                    break;
+               }
+           }
+           else
+           {
+               fprintf(stderr, "\noption detected before required arguments: %s\n", arg);
+               usage();
+               return false;
+           }
+
+           iter.next();
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            if (parseCommandLineOption(iter))
+                continue;
+
+            if (matchCommandLineOption(iter, true)!=EsdlCmdOptionMatch)
+                return false;
+        }
+
+        return true;
+    }
+
+    bool parseCommandLineOption(ArgvIterator &iter)
+    {
+        if (iter.matchFlag(optEncoded, ESDL_OPTION_ENCODED) )
+            return true;
+
+        if (EsdlBindServiceCmd::parseCommandLineOption(iter))
+            return true;
+
+        return false;
+    }
+
+    bool finalizeOptions(IProperties *globals)
+    {
+        if (optInput.length())
+        {
+            const char *in = optInput.get();
+            while (*in && isspace(*in)) in++;
+            if (*in!='<')
+            {
+                StringBuffer content;
+                content.loadFile(in);
+                optInput.set(content.str());
+            }
+        }
+
+        if (optBindingId.isEmpty())
+            throw MakeStringException( 0, "ESDLBindingID must be provided!" );
+
+        if (optLogTransform.isEmpty() && (strncmp(optInput.str(), "<LogTransforms>", 15) != 0))
+            throw MakeStringException( 0, "Name of ESDL based LogTransform must be provided" );
+
+        return EsdlPublishCmdCommon::finalizeOptions(globals);
+    }
+};
+
+class EsdlUnBindLogTransformCmd : public EsdlBindLogTransformCmd
+{
+public:
+    int processCMD()
+    {
+        int success = -1;
+        Owned<IClientWsESDLConfig> esdlConfigClient = EsdlCmdHelper::getWsESDLConfigSoapService(optWSProcAddress, optWSProcPort, optUser, optPass);
+        Owned<IClientGetESDLBindingRequest> getrequest = esdlConfigClient->createGetESDLBindingRequest();
+        if (optVerbose)
+            fprintf(stdout,"\nFetching current ESDL binging configuration for (%s)\n", optBindingId.get());
+        getrequest->setEsdlBindingId(optBindingId.get());
+
+        Owned<IClientGetESDLBindingResponse> getresp = esdlConfigClient->GetESDLBinding(getrequest);
+
+        if (getresp->getExceptions().ordinality()>0)
+        {
+            EsdlCmdHelper::outputMultiExceptions(getresp->getExceptions());
+            return success;
+        }
+
+        if (getresp->getStatus().getCode()!=0)
+        {
+            fprintf(stderr, "\n Failed to retrieve ESDL Binding configuration for %s: %s.\n", optBindingId.get(), getresp->getStatus().getDescription());
+            return success;
+        }
+
+        const char * currentconfig = getresp->getConfigXML();
+        if (currentconfig && *currentconfig)
+        {
+            Owned<IPropertyTree> currconfigtree = createPTreeFromXMLString(currentconfig, ipt_caseInsensitive | ipt_ordered);
+            if (currconfigtree)
+            {
+                VStringBuffer xpath("Definition[1]/LogTransforms/LogTransform[@name='%s']", optLogTransform.get());
+
+                if (currconfigtree->hasProp(xpath.str()))
+                {
+                    if (currconfigtree->removeProp(xpath.str()))
+                    {
+                        StringBuffer newconfig;
+                        toXML(currconfigtree, newconfig);
+
+                        Owned<IClientPublishESDLBindingRequest> request = esdlConfigClient->createPublishESDLBindingRequest();
+
+                        if (optVerbose)
+                            fprintf(stdout,"\nAttempting to remove LogTransform '%s' from esdl binding '%s'\n", optLogTransform.get(), optBindingId.get());
+
+                        request->setEsdlDefinitionID(getresp->getESDLBinding().getDefinition().getId());
+                        request->setEsdlServiceName(getresp->getServiceName());
+                        request->setEspProcName(getresp->getEspProcName());
+                        request->setEspPort(getresp->getEspPort());
+                        request->setConfig(newconfig.str());
+                        request->setOverwrite(true);
+
+                        Owned<IClientPublishESDLBindingResponse> resp = esdlConfigClient->PublishESDLBinding(request);
+
+                        if (resp->getExceptions().ordinality() > 0)
+                        {
+                            EsdlCmdHelper::outputMultiExceptions(resp->getExceptions());
+                            return success;
+                        }
+
+                        if (resp->getStatus().getCode() == 0)
+                        {
+                            fprintf(stdout, "\nSuccessfully unbound LogTransform %s from ESDL Binding %s.\n", optLogTransform.get(), optBindingId.get());
+                            success = 0;
+                        }
+                        else
+                            fprintf(stderr, "\nCould not unbound LogTransform %s from ESDL Binding %s: %s\n", optLogTransform.get(), optBindingId.get(), resp->getStatus().getDescription());
+                    }
+                    else
+                        fprintf(stderr, "\n Could not remove LogTransform %s from ESDL Binding %s configuration.\n", optLogTransform.get(), optBindingId.get());
+                }
+                else
+                    fprintf(stderr, "\n LogTransform %s doesn't seem to be associated with ESDL Binding %s.\n", optLogTransform.get(), optBindingId.get());
+            }
+            else
+                fprintf(stderr, "\n Could not interpret configuration for ESDL Binding %s :  %s.\n", optBindingId.get(), currentconfig );
+        }
+        else
+            fprintf(stderr, "\n Received empty configuration for ESDL Binding %s.\n", optBindingId.get());
+
+        return success;
+    }
+
+    void usage()
+    {
+        printf( "\nUsage:\n\n"
+                "esdl unbind-log-transform <ESDLBindingID> <LogTransformName> [\n\n"
+                "   ESDLBindingID                              The id of the esdl binding associated with the target LogTransform\n"
+                "   LogTransformName                           The name of the target LogTransform (must exist in the service ESDL definition)\n");
+
+                EsdlPublishCmdCommon::usage();
+
+        printf( "\n   Use this command to unbind a LogTransform configuration currently associated with a given ESDL binding.\n"
+                "   To unbind a LogTransform, provide the target esdl binding id and the name of the LogTransform to unbind\n");
+
+        printf("\nExample:"
+                ">esdl unbind-log-transform myesp.8003.WsMyService myLogTransform\n"
+                );
+    }
+    bool parseCommandLineOptions(ArgvIterator &iter)
+    {
+        if (iter.done())
+        {
+            usage();
+            return false;
+        }
+
+        //First 2 parameter order is fixed.
+        for (int cur = 0; cur < 2 && !iter.done(); cur++)
+        {
+           const char *arg = iter.query();
+           if (*arg != '-')
+           {
+               switch (cur)
+               {
+                case 0:
+                    optBindingId.set(arg);
+                    break;
+                case 1:
+                    optLogTransform.set(arg);
+                    break;
+               }
+           }
+           else
+           {
+               fprintf(stderr, "\noption detected before required arguments: %s\n", arg);
+               usage();
+               return false;
+           }
+
+           iter.next();
+        }
+
+        for (; !iter.done(); iter.next())
+        {
+            if (parseCommandLineOption(iter))
+                continue;
+
+            if (matchCommandLineOption(iter, true)!=EsdlCmdOptionMatch)
+                return false;
+        }
+
+        return true;
+    }
+
+    bool finalizeOptions(IProperties *globals)
+    {
+        if(optBindingId.isEmpty())
+            throw MakeStringException( 0, "Name of Target ESDL Binding must be provided" );
+
+        if (optLogTransform.isEmpty())
+            throw MakeStringException( 0, "Name of ESDL based LogTransform must be provided" );
+
+        return EsdlPublishCmdCommon::finalizeOptions(globals);
+    }
+};
+
 class EsdlGetCmd : public EsdlPublishCmdCommon
 {
     protected:

tools/esdlcmd/esdlcmd_common.hpp

@@ -103,6 +103,7 @@ typedef IEsdlCommand *(*EsdlCommandFactory)(const char *cmdname);
 #define ESDL_OPTION_ROLLUP              "--rollup"
 #define ESDL_OPTION_ECL_INCLUDE_LIST    "--ecl-imports"
 #define ESDL_OPTION_ECL_HEADER_BLOCK    "--ecl-header"
+#define ESDL_OPTION_ENCODED             "--encoded"
 
 #define ESDLOPT_INCLUDE_PATH            "--include-path"
 #define ESDLOPT_INCLUDE_PATH_S          "-I"

tools/esdlcmd/esdlcmd_core.cpp

@@ -835,6 +835,10 @@ IEsdlCommand *createCoreEsdlCommand(const char *cmdname)
         return new EsdlListESDLDefCmd();
     if (strieq(cmdname, "LIST-BINDINGS"))
         return new EsdlListESDLBindingsCmd();
+    if (strieq(cmdname, "BIND-LOG-TRANSFORM"))
+        return new EsdlBindLogTransformCmd();
+    if (strieq(cmdname, "UNBIND-LOG-TRANSFORM"))
+        return new EsdlUnBindLogTransformCmd();
     if (strieq(cmdname, "MONITOR"))
         return createEsdlMonitorCommand(cmdname);
     if (strieq(cmdname, "MONITOR-TEMPLATE"))

tools/esdlcmd/esdlcmd_shell.cpp

@@ -197,6 +197,8 @@ void EsdlCMDShell::usage()
            "   bind-method       Configure method associated with existing ESDL binding.\n"
            "   unbind-method     Remove method associated with existing ESDL binding.\n"
            "   get-binding       Get ESDL binding.\n"
+           "   bind-log-transform  	Configure log transform associated with existing ESDL binding.\n"
+           "   unbind-log-transform	Remove log transform associated with existing ESDL binding.\n"
            "   monitor           Generate ECL code for result monitoring / differencing\n"
            "   monitor-template  Generate a template for use with 'monitor' command\n"
            ""