Merge branch 'candidate-7.10.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

cmake_modules/dependencies/el8.cmake

@@ -0,0 +1,6 @@
+# Centos 8
+SET_DEPENDENCIES ( CPACK_RPM_PACKAGE_REQUIRES m4 libtool gcc-c++ openssh-server openssh-clients expect rsync zip psmisc curl )
+
+if(SPARK)
+    SET_DEPENDENCIES ( CPACK_RPM_PACKAGE_REQUIRES java-1.8.0-openjdk-devel )
+endif(SPARK)

common/roxiecommlib/roxiecommunicationclient.cpp

@@ -24,6 +24,7 @@
 
 
 #define GET_LOCK_FAILURE            1100
+#define EMPTY_RESULT_FAILURE        1200
 
 //////////////////////////////////////////
 class CRoxieCommunicationClient: implements IRoxieCommunicationClient, public CInterface
@@ -80,7 +81,11 @@ protected:
             _WINREV(sendlen);
             sock->read(response.reserveTruncate(sendlen), sendlen);
         }
+        if (response.isEmpty())
+            throw MakeStringException(EMPTY_RESULT_FAILURE, "CRoxieCommunicationClient empty response for control request(%s)", xml);
         Owned<IPropertyTree> ret = createPTreeFromXMLString(response.str());
+        if (!ret)
+            throw MakeStringException(EMPTY_RESULT_FAILURE, "CRoxieCommunicationClient empty result tree for control request(%s)", xml);
         Owned<IMultiException> me = MakeMultiException();
         Owned<IPropertyTreeIterator> endpoints = ret->getElements("Endpoint");
         ForEach(*endpoints)

dali/dalidiag/dalidiag.cpp

@@ -71,6 +71,7 @@ void usage(const char *exe)
     printf("-cleartracetransactions  -- stop tracing dali transactions\n");
     printf("-setldapflags <val>      -- set LDAP flags\n");
     printf("-getldapflags            -- get LDAP flags\n");
+    printf("-save                    -- force Dali to save (will block all clients whilst saving)\n");
 
 }
 

ecl/ecl-package/ecl-package.cpp

@@ -918,7 +918,6 @@ public:
             outputLfnCategoryTree(querynode, "Optional");
         }
     }
-
     virtual int processCMD()
     {
         Owned<IClientWsPackageProcess> packageProcessClient = getWsPackageSoapService(optServer, optPort, optUsername, optPassword);
@@ -948,7 +947,36 @@ public:
         int ret = outputMultiExceptionsEx(resp->getExceptions());
         if (ret != 0)
             validateMessages = true;
-        StringArray &errors = resp->getErrors();
+        const char *pmid = resp->getPMID();
+        if (!isEmptyString(pmid))
+        { //server version < 1.04
+            processValidatePackageResponse(nullptr, nullptr, resp->getErrors(), resp->getWarnings(),
+                resp->getQueries().getUnmatched(), resp->getPackages().getUnmatched(), resp->getFiles(), validateMessages);
+        }
+        else
+        {
+            IArrayOf<IConstValidateResult> &results = resp->getResults();
+            ForEachItemIn(i, results)
+            {
+                IConstValidateResult &result = results.item(i);
+                processValidatePackageResponse(result.getTarget(), result.getPMID(), result.getErrors(), result.getWarnings(),
+                    result.getQueries().getUnmatched(), result.getPackages().getUnmatched(), result.getFiles(), validateMessages);
+            }
+        }
+
+        if (!validateMessages)
+            fputs("   Validation was successful\n", stdout);
+
+        return ret;
+    }
+
+    bool processValidatePackageResponse(const char *target, const char *pmid, StringArray &errors, StringArray &warnings,
+        StringArray &unmatchedQueries, StringArray &unusedPackages, IConstValidatePackageFiles &files,
+        bool &validateMessages)
+    {
+        if (!isEmptyString(target) && !isEmptyString(pmid))
+            fprintf(stderr, "   Target: %s, PMID: %s :\n", target, pmid);
+
         if (errors.ordinality()>0)
         {
             validateMessages = true;
@@ -957,7 +985,6 @@ public:
             ForEachItemIn(i, errors)
                 fprintf(stderr, "      %s\n", errors.item(i));
         }
-        StringArray &warnings = resp->getWarnings();
         if (warnings.ordinality()>0)
         {
             validateMessages = true;
@@ -965,7 +992,6 @@ public:
             ForEachItemIn(i, warnings)
                 fprintf(stderr, "      %s\n", warnings.item(i));
         }
-        StringArray &unmatchedQueries = resp->getQueries().getUnmatched();
         if (unmatchedQueries.ordinality()>0)
         {
             validateMessages = true;
@@ -973,7 +999,6 @@ public:
             ForEachItemIn(i, unmatchedQueries)
                 fprintf(stderr, "      %s\n", unmatchedQueries.item(i));
         }
-        StringArray &unusedPackages = resp->getPackages().getUnmatched();
         if (unusedPackages.ordinality()>0)
         {
             validateMessages = true;
@@ -981,7 +1006,7 @@ public:
             ForEachItemIn(i, unusedPackages)
                 fprintf(stderr, "      %s\n", unusedPackages.item(i));
         }
-        StringArray &unusedFiles = resp->getFiles().getUnmatched();
+        StringArray &unusedFiles = files.getUnmatched();
         if (unusedFiles.ordinality()>0)
         {
             validateMessages = true;
@@ -1006,7 +1031,7 @@ public:
             outputLfnTree(filetree);
         }
 
-        StringArray &notInDFS = resp->getFiles().getNotInDFS();
+        StringArray &notInDFS = files.getNotInDFS();
         if (notInDFS.ordinality()>0)
         {
             validateMessages = true;
@@ -1015,10 +1040,9 @@ public:
                 fprintf(stderr, "      %s\n", notInDFS.item(i));
         }
 
-        if (!validateMessages)
-            fputs("   Validation was successful\n", stdout);
-
-        return ret;
+        if (!isEmptyString(target) && !isEmptyString(pmid))
+            fprintf(stderr, "\n   Target: %s, PMID: %s done\n\n", target, pmid);
+        return validateMessages;
     }
 
     virtual void usage()

esp/clients/roxiecontrol.cpp

@@ -20,10 +20,12 @@
 
 const unsigned roxieQueryRoxieTimeOut = 60000;
 
-void checkRoxieControlExceptions(IPropertyTree *msg)
+#define EMPTY_RESULT_FAILURE 1200
+
+static void checkRoxieControlExceptions(IPropertyTree *response)
 {
     Owned<IMultiException> me = MakeMultiException();
-    Owned<IPropertyTreeIterator> endpoints = msg->getElements("Endpoint");
+    Owned<IPropertyTreeIterator> endpoints = response->getElements("Endpoint");
     ForEach(*endpoints)
     {
         IPropertyTree &endp = endpoints->query();
@@ -65,8 +67,12 @@ IPropertyTree *sendRoxieControlQuery(ISocket *sock, const char *msg, unsigned wa
         if (size_read<len)
             throw MakeStringException(-1, "Error reading roxie control message response");
     }
-
+    if (resp.isEmpty())
+        throw MakeStringException(EMPTY_RESULT_FAILURE, "Empty response string for roxie control request(%s) wait(%d)", msg, wait);
     Owned<IPropertyTree> ret = createPTreeFromXMLString(resp.str());
+    if (!ret)
+        throw MakeStringException(EMPTY_RESULT_FAILURE, "Empty result tree for roxie control request(%s) wait(%d)", msg, wait);
+
     checkRoxieControlExceptions(ret);
     return ret.getClear();
 }

esp/scm/ws_packageprocess.ecm

@@ -212,10 +212,10 @@ ESPstruct ValidatePackageFiles
     ESParray<string, File> NotInDFS;
 };
 
-ESPresponse [exceptions_inline] ValidatePackageResponse
+ESPstruct ValidateResult
 {
+    string Target;
     string PMID;
-    ESPstruct BasePackageStatus status;
     ESParray<string> Warnings;
     ESParray<string> Errors;
     ESPstruct ValidatePackageInfo packages;
@@ -223,6 +223,18 @@ ESPresponse [exceptions_inline] ValidatePackageResponse
     ESPstruct ValidatePackageFiles files;
 };
 
+ESPresponse [exceptions_inline] ValidatePackageResponse
+{
+    [depr_ver("1.04")] string PMID;
+    [depr_ver("1.04")] ESPstruct BasePackageStatus status;
+    [depr_ver("1.04")] ESParray<string> Warnings;
+    [depr_ver("1.04")] ESParray<string> Errors;
+    [depr_ver("1.04")] ESPstruct ValidatePackageInfo packages;
+    [depr_ver("1.04")] ESPstruct ValidatePackageQueries queries;
+    [depr_ver("1.04")] ESPstruct ValidatePackageFiles files;
+    [min_ver("1.04")] ESParray<ESPstruct ValidateResult, Result> Results;
+};
+
 ESPrequest GetQueryFileMappingRequest
 {
     string Target;
@@ -316,7 +328,7 @@ ESPresponse [exceptions_inline] GetPartFromPackageMapResponse
     string Content;
 };
 
-ESPservice [auth_feature("NONE"), version("1.03"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
+ESPservice [auth_feature("NONE"), version("1.04"), exceptions_inline("./smc_xslt/exceptions.xslt")] WsPackageProcess
 {
     ESPmethod Echo(EchoRequest, EchoResponse);
     ESPmethod [auth_feature("PackageMapAccess:WRITE")] AddPackage(AddPackageRequest, AddPackageResponse);

esp/services/ws_packageprocess/ws_packageprocessService.cpp

@@ -28,6 +28,7 @@
 #include "referencedfilelist.hpp"
 #include "package.h"
 #include "eclwatch_errorlist.hpp"
+#include "exception_util.hpp"
 
 #define SDS_LOCK_TIMEOUT (5*60*1000) // 5mins, 30s a bit short
 
@@ -287,7 +288,9 @@ public:
             target.set(_target);
             if (target.isEmpty())
                 throw MakeStringExceptionDirect(PKG_MISSING_PARAM, "Target cluster parameter required");
-            ensureClusterInfo();
+
+            if (!streq(target.get(), "*"))
+                ensureClusterInfo();
             pmid.append(target).append("::");
         }
         pmid.append(name);
@@ -362,7 +365,19 @@ public:
     }
     void cloneDfsInfo(unsigned updateFlags, StringArray &filesNotFound, IPropertyTree *pt)
     {
-        cloneFileInfoToDali(updateFlags, filesNotFound, pt, daliIP, ensureClusterInfo(), srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
+        if (!streq(target.get(), "*"))
+            cloneFileInfoToDali(updateFlags, filesNotFound, pt, daliIP, ensureClusterInfo(), srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
+        else
+        {
+            CConstWUClusterInfoArray clusters;
+            getEnvironmentClusterInfo(clusters);
+            ForEachItemIn(i, clusters)
+            {
+                IConstWUClusterInfo &cluster = clusters.item(i);
+                if (cluster.getPlatform() == RoxieCluster)
+                    cloneFileInfoToDali(updateFlags, filesNotFound, pt, daliIP, &cluster, srcCluster, prefix, userdesc, checkFlag(PKGADD_ALLOW_FOREIGN));
+            }
+        }
     }
     void cloneDfsInfo(unsigned updateFlags, StringArray &filesNotFound)
     {
@@ -1084,102 +1099,150 @@ bool CWsPackageProcessEx::onGetPackageMapById(IEspContext &context, IEspGetPacka
 
 bool CWsPackageProcessEx::onValidatePackage(IEspContext &context, IEspValidatePackageRequest &req, IEspValidatePackageResponse &resp)
 {
-    StringArray warnings;
-    StringArray errors;
-    StringArray unmatchedQueries;
-    StringArray unusedPackages;
-    StringArray unmatchedFiles;
+    try
+    {
+        const char *target = req.getTarget();
+        if (isEmptyString(target))
+            throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
+
+        Owned<IPropertyTree> packageMapTree;
+        const char *pmID = req.getPMID();
+        const char *packageMapString = req.getInfo();
+        if (!req.getActive() && isEmptyString(pmID))
+        {
+            if (isEmptyString(packageMapString))
+                throw makeStringException(ECLWATCH_INVALID_INPUT, "Please specify PMID or Package Map content");
 
-    Owned<IHpccPackageSet> set;
-    Owned<IPropertyTree> mapTree;
+            packageMapTree.setown(createPTreeFromXMLString(packageMapString));
+            if (!packageMapTree)
+                throw makeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error processing package file content");
+            fixPackageMapFileIds(packageMapTree, false);
+        }
 
-    const char *target = req.getTarget();
-    if (!target || !*target)
-        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster required");
-    Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
-    if (!clusterInfo)
-        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster not found");
+        StringArray &queriesToIgnore = req.getQueriesToIgnore();
+        StringArray &queriesToVerify = req.getQueriesToVerify();
+        const char *queryid = req.getQueryIdToVerify();
+        if (!isEmptyString(queryid))
+            queriesToVerify.appendUniq(queryid);
+
+        double version = context.getClientVersion();
+        IArrayOf<IEspValidateResult> results;
+        if (!streq(target, "*"))
+        {
+            Owned<IConstWUClusterInfo> clusterInfo = getTargetClusterInfo(target);
+            if (!clusterInfo)
+                throw makeStringException(PKG_TARGET_NOT_DEFINED, "Target cluster not found");
+            validatePackage(context, req, packageMapTree, clusterInfo, queriesToVerify, queriesToIgnore,
+                (version < 1.04) ? &resp : nullptr, results);
+        }
+        else
+        {
+            CConstWUClusterInfoArray clusters;
+            getEnvironmentClusterInfo(clusters);
+            ForEachItemIn(i, clusters)
+            {
+                IConstWUClusterInfo &cluster = clusters.item(i);
+                if (cluster.getPlatform() == RoxieCluster)
+                {
+                    validatePackage(context, req, packageMapTree, &cluster, queriesToVerify, queriesToIgnore,
+                        (version < 1.04) ? &resp : nullptr, results);
+                    if (version < 1.04)
+                        break;
+                }
+            }
+        }
+        if (version >= 1.04)
+            resp.setResults(results);
+    }
+    catch(IException *e)
+    {
+        FORWARDEXCEPTION(context, e, ECLWATCH_INTERNAL_ERROR);
+    }
+    return true;
+}
+
+void CWsPackageProcessEx::validatePackage(IEspContext &context, IEspValidatePackageRequest &req, IPropertyTree *packageMapTree,
+    IConstWUClusterInfo *clusterInfo, StringArray &queriesToVerify, StringArray &queriesToIgnore,
+    IEspValidatePackageResponse *resp, IArrayOf<IEspValidateResult>& results)
+{
     SCMStringBuffer process;
     clusterInfo->getRoxieProcess(process);
     if (!process.length())
-        throw MakeStringException(PKG_TARGET_NOT_DEFINED, "Roxie process not found");
+        throw makeStringException(PKG_TARGET_NOT_DEFINED, "Roxie process not found");
 
+    SCMStringBuffer clusterName;
+    clusterInfo->getName(clusterName);
+    const char *target = clusterName.str();
     const char* pmID = req.getPMID();
-    const char* info = req.getInfo();
 
+    Owned<IPropertyTree> mapTree;
     if (req.getActive()) //validate active map
     {
         mapTree.setown(resolveActivePackageMap(process.str(), target, true));
         if (!mapTree)
-            throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
-    }
-    else if (pmID && *pmID)
-    {
-        mapTree.setown(getPackageMapById(req.getGlobalScope() ? NULL : target, pmID, true));
-        if (!mapTree)
-            throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Package map %s not found", pmID);
+            throw makeStringException(PKG_PACKAGEMAP_NOT_FOUND, "Active package map not found");
     }
-    else if (info && *info)
+    else if (!isEmptyString(pmID))
     {
-        mapTree.setown(createPTreeFromXMLString(info));
+        mapTree.setown(getPackageMapById(req.getGlobalScope() ? nullptr : target, pmID, true));
         if (!mapTree)
-            throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error processing package file content");
-        fixPackageMapFileIds(mapTree, false);
+            throw makeStringExceptionV(PKG_PACKAGEMAP_NOT_FOUND, "Package map %s not found", pmID);
     }
-    else
+    else if (packageMapTree)
     {
-        throw MakeStringException(PKG_PACKAGEMAP_NOT_FOUND, "package map not specified");
+        mapTree.setown(LINK(packageMapTree));
     }
 
+    StringArray notInDFS;
     if (req.getCheckDFS())
     {
         Owned<IReferencedFileList> pmfiles = createReferencedFileList(context.queryUserId(), context.queryPassword(), true, false);
         pmfiles->addFilesFromPackageMap(mapTree);
-        pmfiles->resolveFiles(process.str(), NULL, NULL, NULL, true, false, false);
+        pmfiles->resolveFiles(process.str(), nullptr, nullptr, nullptr, true, false, false);
         Owned<IReferencedFileIterator> files = pmfiles->getFiles();
-        StringArray notInDFS;
         ForEach(*files)
         {
             IReferencedFile &file = files->query();
             if (file.getFlags() & RefFileNotFound)
                 notInDFS.append(file.getLogicalName());
         }
-        resp.updateFiles().setNotInDFS(notInDFS);
     }
 
     const char *id = mapTree->queryProp("@id");
     Owned<IHpccPackageMap> map = createPackageMapFromPtree(mapTree, target, id);
     if (!map)
-        throw MakeStringException(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", id);
+        throw makeStringExceptionV(PKG_LOAD_PACKAGEMAP_FAILED, "Error loading package map %s", id);
 
-    StringArray queriesToVerify;
-    const char *queryid = req.getQueryIdToVerify();
-    if (queryid && *queryid)
-        queriesToVerify.append(queryid);
-    ForEachItemIn(i, req.getQueriesToVerify())
-    {
-        queryid = req.getQueriesToVerify().item(i);
-        if (queryid && *queryid)
-            queriesToVerify.appendUniq(queryid);
-    }
-    StringArray queriesToIgnore;
-    ForEachItemIn(i2, req.getQueriesToIgnore())
+    StringArray warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles;
+    map->validate(queriesToVerify, queriesToIgnore, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles, req.getIgnoreOptionalFiles());
+
+    if (resp)
     {
-        queryid = req.getQueriesToIgnore().item(i2);
-        if (queryid && *queryid)
-            queriesToIgnore.appendUniq(queryid);
+        resp->setPMID(map->queryPackageId());
+        if (!req.getIgnoreWarnings())
+            resp->setWarnings(warnings);
+        resp->setErrors(errors);
+        resp->updateQueries().setUnmatched(unmatchedQueries);
+        resp->updatePackages().setUnmatched(unusedPackages);
+        resp->updateFiles().setUnmatched(unmatchedFiles);
+        resp->updateStatus().setCode(0);
+        if (notInDFS.length())
+            resp->updateFiles().setNotInDFS(notInDFS);
+        return;
     }
-    map->validate(queriesToVerify, queriesToIgnore, warnings, errors, unmatchedQueries, unusedPackages, unmatchedFiles, req.getIgnoreOptionalFiles());
 
-    resp.setPMID(map->queryPackageId());
+    Owned<IEspValidateResult> result = createValidateResult();
+    result->setTarget(target);
+    result->setPMID(map->queryPackageId());
     if (!req.getIgnoreWarnings())
-        resp.setWarnings(warnings);
-    resp.setErrors(errors);
-    resp.updateQueries().setUnmatched(unmatchedQueries);
-    resp.updatePackages().setUnmatched(unusedPackages);
-    resp.updateFiles().setUnmatched(unmatchedFiles);
-    resp.updateStatus().setCode(0);
-    return true;
+        result->setWarnings(warnings);
+    result->setErrors(errors);
+    result->updateQueries().setUnmatched(unmatchedQueries);
+    result->updatePackages().setUnmatched(unusedPackages);
+    result->updateFiles().setUnmatched(unmatchedFiles);
+    if (notInDFS.length())
+        result->updateFiles().setNotInDFS(notInDFS);
+    results.append(*result.getClear());
 }
 
 bool CWsPackageProcessEx::onGetPackageMapSelectOptions(IEspContext &context, IEspGetPackageMapSelectOptionsRequest &req, IEspGetPackageMapSelectOptionsResponse &resp)

esp/services/ws_packageprocess/ws_packageprocessService.hpp

@@ -20,6 +20,7 @@
 
 #include "ws_packageprocess_esp.ipp"
 #include "dasds.hpp"
+#include "environment.hpp"
 
 #define THORCLUSTER "thor"
 #define HTHORCLUSTER "hthor"
@@ -151,6 +152,8 @@ class CWsPackageProcessEx : public CWsPackageProcess
     void getPkgInfoById(const char *target, const char *packageMapId, IPropertyTree* tree);
     void getPkgInfoById(const char *packageMapId, IPropertyTree* tree);
     void deletePackage(const char *packageMap, const char *target, const char *process, bool globalScope, StringBuffer &returnMsg, int &returnCode);
+    void validatePackage(IEspContext &context, IEspValidatePackageRequest &req, IPropertyTree *packageMapTree, IConstWUClusterInfo *clusterInfo,
+        StringArray &queriesToVerify, StringArray &queriesToIgnore, IEspValidatePackageResponse *resp, IArrayOf<IEspValidateResult>& results);
 public:
     IMPLEMENT_IINTERFACE;
     virtual ~CWsPackageProcessEx()

esp/src/package.json

@@ -12,7 +12,7 @@
     "copy-res-dojox": "cpx \"./node_modules/dojox/widget/ColorPicker/images/**/*.{png,jpg,gif}\" ./build/eclwatch/img/",
     "copy-res-TopoJSON": "cpx \"./node_modules/@hpcc-js/map/TopoJSON/**/*\" ./build/dist/TopoJSON/",
     "copy-res-font-awesome": "cpx \"./node_modules/font-awesome/**/*\" ./build/dist/font-awesome/",
-    "copy-res-wasm": "cpx \"node_modules/@hpcc-js/wasm/dist/graphvizlib.wasm\" ./build/dist/",
+    "copy-res-wasm": "cpx \"./node_modules/@hpcc-js/graph/dist/graphvizlib.wasm\" ./build/dist/",
     "copy-res-stub_htm": "cpx \"./stub.htm\" ./build/",
     "copy-res": "run-s copy-res-es6-promise copy-res-eclwatch-img copy-res-eclwatch-ecl copy-res-dojo copy-res-dojox copy-res-TopoJSON copy-res-font-awesome copy-res-stub_htm copy-res-wasm",
     "compile": "tsc",

esp/src/src/GraphTree7Widget.ts

@@ -47,8 +47,6 @@ import "hpcc/TableContainer";
 
 declare const dojoConfig;
 
-window["__hpcc_wasmFolder"] = dojoConfig.urlInfo.fullPath + "/dist";
-
 class DataGraph {
     private _controller: WUScopeController = new WUScopeController();
     private _widget: GraphWidget = new GraphWidget();
@@ -67,7 +65,9 @@ class DataGraph {
 
 class DataGraph2 {
     private _controller: WUScopeController8 = new WUScopeController8();
-    private _widget: Graph2Widget = new Graph2Widget();
+    private _widget: Graph2Widget = new Graph2Widget()
+        .wasmFolder(dojoConfig.urlInfo.fullPath + "/dist")
+        ;
 
     constructor() {
     }

rtl/eclrtl/rtldynfield.cpp

@@ -1595,11 +1595,13 @@ private:
                 {
                     size32_t defaultSize = (initializer && !isVirtualInitializer(initializer)) ? type->size(initializer, nullptr) : type->getMinSize();
                     fixedDelta -= defaultSize;
+#ifdef TRACE_TRANSLATION
+                    DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
+#endif
                 }
                 if ((field->flags & RFTMispayloadfield) == 0)
                     matchFlags |= match_keychange;
                 defaulted++;
-                //DBGLOG("Decreasing fixedDelta size by %d to %d for defaulted field %d (%s)", defaultSize, fixedDelta, idx, destRecInfo.queryName(idx));
             }
             else
             {
@@ -1720,7 +1722,9 @@ private:
                                 info.matchType = match_truncate;
                                 if (((sourceFlags|destFlags) & RFTMinifblock) == 0)
                                     fixedDelta += sourceType->getMinSize()-type->getMinSize();
-                                //DBGLOG("Increasing fixedDelta size by %d to %d for truncated field %d (%s)", sourceType->getMinSize()-type->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
+#ifdef TRACE_TRANSLATION
+                                DBGLOG("Increasing fixedDelta size by %d to %d for truncated field %d (%s)", sourceType->getMinSize()-type->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
+#endif
                             }
                         }
                         else
@@ -1730,7 +1734,9 @@ private:
                                 info.matchType = match_extend;
                                 if (((sourceFlags|destFlags) & RFTMinifblock) == 0)
                                     fixedDelta += sourceType->getMinSize()-type->getMinSize();
-                                //DBGLOG("Decreasing fixedDelta size by %d to %d for truncated field %d (%s)", type->getMinSize()-sourceType->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
+#ifdef TRACE_TRANSLATION
+                                DBGLOG("Decreasing fixedDelta size by %d to %d for truncated field %d (%s)", type->getMinSize()-sourceType->getMinSize(), fixedDelta, idx, destRecInfo.queryName(idx));
+#endif
                             }
                         }
                     }
@@ -1739,7 +1745,7 @@ private:
                     info.matchType = match_typecast;
                 if (deblob)
                     info.matchType |= match_deblob;
-                if (sourceFlags & RFTMinifblock)
+                if (sourceFlags & RFTMinifblock || field->flags & RFTMinifblock)
                     info.matchType |= match_inifblock;  // Avoids incorrect commoning up of adjacent matches
                 // MORE - could note the highest interesting fieldnumber in the source and not bother filling in offsets after that
                 // Not sure it would help much though - usually need to know the total record size anyway in real life
@@ -1772,7 +1778,9 @@ private:
                         const RtlTypeInfo *type = field->type;
                         if (type->isFixedSize() && (field->flags & RFTMinifblock)==0)
                         {
-                            //DBGLOG("Reducing estimated size by %d for (fixed size) omitted field %s", (int) type->getMinSize(), field->name);
+#ifdef TRACE_TRANSLATION
+                            DBGLOG("Reducing estimated size by %d for (fixed size) omitted field %s", (int) type->getMinSize(), field->name);
+#endif
                             fixedDelta += type->getMinSize();
                         }
                         else
@@ -1781,21 +1789,29 @@ private:
                     allUnmatched.append(idx);
                 }
             }
-            //DBGLOG("Delta from fixed-size fields is %d bytes", fixedDelta);
+#ifdef TRACE_TRANSLATION
+            DBGLOG("Delta from fixed-size fields is %d bytes", fixedDelta);
+#endif
         }
     }
     size32_t estimateNewSize(const RtlRow &sourceRow) const
     {
-        //DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
+#ifdef TRACE_TRANSLATION
+        DBGLOG("Source record size is %d", (int) sourceRow.getRecordSize());
+#endif
         size32_t expectedSize = sourceRow.getRecordSize();
         assertex((int) expectedSize >= fixedDelta);
         expectedSize -= fixedDelta;
-        //DBGLOG("Source record size without fixed delta is %d", expectedSize);
+#ifdef TRACE_TRANSLATION
+        DBGLOG("Source record size without fixed delta is %d", expectedSize);
+#endif
         ForEachItemIn(i, variableUnmatched)
         {
             unsigned fieldNo = variableUnmatched.item(i);
             expectedSize -= sourceRow.getSize(fieldNo);
-            //DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
+#ifdef TRACE_TRANSLATION
+            DBGLOG("Reducing estimated size by %d to %d for omitted field %d (%s)", (int) sourceRow.getSize(fieldNo), expectedSize, fieldNo, sourceRecInfo.queryName(fieldNo));
+#endif
         }
         if (matchFlags & ~(match_perfect|match_link|match_none|match_virtual|match_extend|match_truncate))
         {
@@ -1804,29 +1820,34 @@ private:
                 const MatchInfo &match = matchInfo[idx];
                 const RtlTypeInfo *type = destRecInfo.queryType(idx);
                 unsigned matchField = match.matchIdx;
-                switch (match.matchType)
+                if ((match.matchType & match_inifblock) == 0)
                 {
-                case match_perfect:
-                case match_link:
-                case match_none:
-                case match_virtual:
-                case match_extend:
-                case match_truncate:
-                    // These ones were already included in fixedDelta
-                    break;
-                default:
-                    // This errs on the side of small - i.e. it assumes that all typecasts end up at minimum size
-                    // We could do better in some cases e.g. variable string <-> variable unicode we can assume factor of 2,
-                    // uft8 <-> string we could calculate here - but unlikely to be worth the effort.
-                    // But it's fine for fixed size output fields, including truncate/extend
-                    // We could also precalculate the expected delta if all omitted fields are fixed size - but not sure how likely/worthwhile that is.
-                    auto minSize = type->getMinSize();
-                    auto sourceSize = sourceRow.getSize(matchField);
-                    expectedSize += minSize;
-                    assertex(expectedSize >= sourceSize);
-                    expectedSize -= sourceSize;
-                    //DBGLOG("Adjusting estimated size by (%d - %d) to %d for translated field %d (%s)", (int) sourceSize, minSize, expectedSize, matchField, sourceRecInfo.queryName(matchField));
-                    break;
+                    switch (match.matchType)
+                    {
+                    case match_perfect:
+                    case match_link:
+                    case match_none:
+                    case match_virtual:
+                    case match_extend:
+                    case match_truncate:
+                        // These ones were already included in fixedDelta
+                        break;
+                    default:
+                        // This errs on the side of small - i.e. it assumes that all typecasts end up at minimum size
+                        // We could do better in some cases e.g. variable string <-> variable unicode we can assume factor of 2,
+                        // uft8 <-> string we could calculate here - but unlikely to be worth the effort.
+                        // But it's fine for fixed size output fields, including truncate/extend
+                        // We could also precalculate the expected delta if all omitted fields are fixed size - but not sure how likely/worthwhile that is.
+                        auto minSize = type->getMinSize();
+                        auto sourceSize = sourceRow.getSize(matchField);
+                        expectedSize += minSize;
+                        assertex(expectedSize >= sourceSize);
+                        expectedSize -= sourceSize;
+    #ifdef TRACE_TRANSLATION
+                        DBGLOG("Adjusting estimated size by (%d - %d) to %d for translated field %d (%s)", (int) sourceSize, minSize, expectedSize, matchField, sourceRecInfo.queryName(matchField));
+    #endif
+                        break;
+                    }
                 }
             }
         }