Merge branch 'candidate-5.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

docs/ECLProgrammersGuide/CMakeLists.txt

@@ -17,8 +17,9 @@
 
 DOCBOOK_TO_PDF( ${FO_XSL} PrGd-Includer.xml "ECLProgrammersGuide" "PRG_Mods")
 
-set(zip_out_dir ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs)
-ADD_CUSTOM_COMMAND(
+IF(MAKE_DOCS)
+  set(zip_out_dir ${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/docs)
+  ADD_CUSTOM_COMMAND(
 	COMMAND mkdir -p ${zip_out_dir}
 	COMMAND mkdir -p ECL_Code_Files
 	COMMAND rm -rf ECL_Code_Files/*
@@ -27,5 +28,5 @@ ADD_CUSTOM_COMMAND(
         OUTPUT ${zip_out_dir}/ECL_Code_Files.zip
 	)
 
-ADD_CUSTOM_TARGET(ECL_Code_Files ALL DEPENDS  ${zip_out_dir}/ECL_Code_Files.zip)
-
+  ADD_CUSTOM_TARGET(ECL_Code_Files ALL DEPENDS  ${zip_out_dir}/ECL_Code_Files.zip)
+ENDIF(MAKE_DOCS)

docs/Installing_and_RunningTheHPCCPlatform/Inst-Mods/hpcc_ldap.xml

@@ -600,28 +600,39 @@
     <link linkend="configuring-a-multi-node-system"><emphasis>Configuring a
     Multi-Node System</emphasis></link> for more info about configuring your
     system. With the correct environment.xml file in place, you must then run
-    the <emphasis role="bold">initldap</emphasis> utility that installs the
-    initial security components and the default users.</para>
+    the <emphasis role="bold">initldap</emphasis> utility that initializes the
+    security components and the default users.</para>
 
     <sect3>
       <title>The initldap Utility</title>
 
-      <para>The initldap utility is used to create the HPCC Administrator's
-      user account and the HPCC OUs. The initldap utility uses the settings in
-      the LDAPServer component(s) in the environment.xml bound to the
-      configured ESPs.</para>
+      <para>The initldap utility creates the HPCC Administrator's user account
+      and the HPCC OUs for a newly defined LDAP server. The initldap utility
+      extracts these settings from the LDAPServer component(s) in the
+      environment.xml bound to the configured ESPs.</para>
 
-      <para>Once you complete your configuration with LDAP components enabled
-      and have distributed your environment.xml file to all nodes, you must
-      then run the <emphasis role="bold">initldap</emphasis> utility.</para>
+      <para>You run the <emphasis role="bold">initldap</emphasis> utility once
+      you complete your configuration with LDAP components enabled and have
+      distributed your environment.xml file to all nodes.</para>
 
       <programlisting>sudo /opt/HPCCSystems/bin/initldap</programlisting>
 
       <para>The <emphasis role="bold">initldap</emphasis> utility prompts you
       for LDAP Administrator credentials. Enter the appropriate values when
-      prompted. The initldap utility uses those values to initialize the LDAP
-      server and create the HPCC System User. It also creates all of the
-      required OUs.</para>
+      prompted. </para>
+
+      <para>The following example of initldap for a 389DirectoryServer
+      deployment. </para>
+
+      <para><programlisting>Enter the '389DirectoryServer' LDAP Admin User name on '10.123.456.78'...Directory Manager        
+Enter the LDAP Admin user 'Directory Manager' password...********
+
+Ready to initialize HPCC LDAP Environment, using the following settings
+       LDAP Server     : 10.123.456.78
+       LDAP Type       : 389DirectoryServer
+       HPCC Admin User : HPCCAdmin389
+Proceed?  y/n 
+</programlisting></para>
     </sect3>
 
     <sect3>

ecl/hqlcpp/hqlttcpp.cpp

@@ -3003,44 +3003,47 @@ IHqlExpression * ThorHqlTransformer::normalizeJoinOrDenormalize(IHqlExpression *
         }
     }
 
-    //Sort,Sort->join is O(NlnN) lookup join using a hash table is O(N) =>convert for hthor/roxie
-    if (!isThorCluster(targetClusterType) &&
-        !expr->hasAttribute(_normalized_Atom) && !expr->hasAttribute(smartAtom) && !expr->hasAttribute(streamedAtom))
+    if (!isThorCluster(targetClusterType) && !expr->hasAttribute(_normalized_Atom))
     {
-        bool createLookup = false;
-        if ((op == no_join) && options.convertJoinToLookup)
+        if (!expr->hasAttribute(streamedAtom) && !expr->hasAttribute(smartAtom))
         {
-            if ((targetClusterType == RoxieCluster) || hasFewRows(rightDs))
-                if (!isFullJoin(expr) && !isRightJoin(expr) && !expr->hasAttribute(partitionRightAtom))
-                    createLookup = !expr->hasAttribute(_lightweight_Atom);
-        }
+            //Sort,Sort->join is O(NlnN) lookup join using a hash table is O(N) =>convert for hthor/roxie
+            bool createLookup = false;
+            if ((op == no_join) && options.convertJoinToLookup)
+            {
+                if ((targetClusterType == RoxieCluster) || hasFewRows(rightDs))
+                    if (!isFullJoin(expr) && !isRightJoin(expr) && !expr->hasAttribute(partitionRightAtom))
+                        createLookup = !expr->hasAttribute(_lightweight_Atom);
+            }
 
-        if (joinInfo.hasOptionalEqualities())
-            createLookup = false;           //doesn't support it yet
-        else if (createLookup && joinInfo.queryLeftSort().ordinality())
-        {
-            //Check this isn't going to generate a between join - if it is that takes precedence.
-            if ((joinInfo.slidingMatches.ordinality() != 0) && (joinInfo.queryLeftSort().ordinality() == joinInfo.slidingMatches.ordinality()))
-                createLookup = false;
-        }
+            if (joinInfo.hasOptionalEqualities())
+                createLookup = false;           //doesn't support it yet
+            else if (createLookup && joinInfo.queryLeftSort().ordinality())
+            {
+                //Check this isn't going to generate a between join - if it is that takes precedence.
+                if ((joinInfo.slidingMatches.ordinality() != 0) && (joinInfo.queryLeftSort().ordinality() == joinInfo.slidingMatches.ordinality()))
+                    createLookup = false;
+            }
 
-        if (createLookup)
-        {
-            IHqlExpression * lhs = expr->queryChild(0);
-            HqlExprArray args;
-            if (isGrouped(lhs))
+            if (createLookup)
             {
-                OwnedHqlExpr ungroup = createDataset(no_group, LINK(lhs));
-                args.append(*cloneInheritedAnnotations(expr, ungroup));
+                IHqlExpression * lhs = expr->queryChild(0);
+                HqlExprArray args;
+                if (isGrouped(lhs))
+                {
+                    OwnedHqlExpr ungroup = createDataset(no_group, LINK(lhs));
+                    args.append(*cloneInheritedAnnotations(expr, ungroup));
+                }
+                else
+                    args.append(*LINK(lhs));
+                unwindChildren(args, expr, 1);
+                args.append(*createAttribute(manyAtom));
+                args.append(*createAttribute(lookupAtom));
+                return expr->clone(args);
             }
-            else
-                args.append(*LINK(lhs));
-            unwindChildren(args, expr, 1);
-            args.append(*createAttribute(manyAtom));
-            args.append(*createAttribute(lookupAtom));
-            return expr->clone(args);
         }
 
+        //Ensure that inputs to the activities in hthor/roxie are sorted and grouped.  (Should really be done in the engines)
         OwnedHqlExpr newLeft = getNonThorSortedJoinInput(expr, leftDs, joinInfo.queryLeftSort(), options.implicitSubSort);
         OwnedHqlExpr newRight = getNonThorSortedJoinInput(expr, rightDs, joinInfo.queryRightSort(), options.implicitSubSort);
         try

esp/src/eclwatch/DFUQueryWidget.js

@@ -51,6 +51,7 @@ define([
     "hpcc/TargetSelectWidget",
     "hpcc/FilterDropDownWidget",
     "hpcc/SelectionGridWidget",
+    "hpcc/WsTopology",
 
     "put-selector/put",
 
@@ -77,7 +78,7 @@ define([
 ], function (declare, lang, i18n, nlsHPCC, arrayUtil, dom, domAttr, domConstruct, domClass, domForm, date, on, topic,
                 registry, Dialog, Menu, MenuItem, MenuSeparator, PopupMenuItem, Textarea, ValidationTextBox,
                 editor, selector, tree,
-                _TabContainerWidget, WsDfu, FileSpray, ESPUtil, ESPLogicalFile, ESPDFUWorkunit, DelayLoadWidget, TargetSelectWidget, FilterDropDownWidget, SelectionGridWidget,
+                _TabContainerWidget, WsDfu, FileSpray, ESPUtil, ESPLogicalFile, ESPDFUWorkunit, DelayLoadWidget, TargetSelectWidget, FilterDropDownWidget, SelectionGridWidget, WsTopology,
                 put,
                 template) {
     return declare("DFUQueryWidget", [_TabContainerWidget, ESPUtil.FormHelper], {
@@ -102,6 +103,7 @@ define([
             this.desprayForm = registry.byId(this.id + "DesprayForm");
             this.desprayTargetSelect = registry.byId(this.id + "DesprayTargetSelect");
             this.desprayGrid = registry.byId(this.id + "DesprayGrid");
+            this.remoteCopyReplicateCheckbox = registry.byId(this.id + "RemoteCopyReplicate");
         },
 
         startup: function (args) {
@@ -297,6 +299,11 @@ define([
             this.importTargetSelect.init({
                 Groups: true
             });
+
+            this.importTargetSelect.on('change', function (value){
+                context.checkReplicate(value, context.remoteCopyReplicateCheckbox);
+            });
+
             this.copyTargetSelect.init({
                 Groups: true
             });
@@ -394,6 +401,25 @@ define([
             pMenu.startup();
         },
 
+        checkReplicate: function (value, checkBoxValue) {
+            WsTopology.TpGroupQuery({
+                request: {}
+            }).then(function (response) {
+                if (lang.exists("TpGroupQueryResponse.TpGroups.TpGroup", response)) {
+                    var arr = response.TpGroupQueryResponse.TpGroups.TpGroup;
+                    for (var index in arr) {
+                        if (arr[index].Name === value && arr[index].ReplicateOutputs === true) {
+                            checkBoxValue.set("disabled", false);
+                            break;
+                        } else if (arr[index].Name === value) {
+                            checkBoxValue.set("disabled", true);
+                            break;
+                        }
+                    }
+                }
+            });
+        },
+
         initWorkunitsGrid: function () {
             var context = this;
             this.listStore = this.params.searchResults ? this.params.searchResults : new ESPLogicalFile.CreateLFQueryStore();

esp/src/eclwatch/LZBrowseWidget.js

@@ -105,6 +105,11 @@ define([
             this.dropZoneFolderSelect = registry.byId(this.id + "DropZoneFolderSelect");
             this.fileListDialog = registry.byId(this.id + "FileListDialog");
             this.overwriteCheckbox = registry.byId(this.id + "FileOverwriteCheckbox");
+            this.fixedSprayReplicateCheckbox = registry.byId(this.id + "FixedSprayReplicate");
+            this.delimitedSprayReplicateCheckbox = registry.byId(this.id + "DelimitedSprayReplicate");
+            this.xmlSprayReplicateCheckbox = registry.byId(this.id + "XMLSprayReplicate");
+            this.variableSprayReplicateCheckbox = registry.byId(this.id + "VariableSprayReplicate");
+            this.blobSprayReplicateCheckbox = registry.byId(this.id + "BlobSprayReplicate");
 
             var context = this;
             this.connect(this.uploader, "onComplete", function (response) {
@@ -469,6 +474,45 @@ define([
                 DropZoneFolders: true,
                 includeBlank: true
             });
+
+            this.sprayFixedDestinationSelect.on('change', function (value){
+                context.checkReplicate(value, context.fixedSprayReplicateCheckbox);
+            });
+
+            this.sprayDelimitedDestinationSelect.on('change', function (value){
+                context.checkReplicate(value, context.delimitedSprayReplicateCheckbox);
+            });
+
+            this.sprayXmlDestinationSelect.on('change', function (value){
+                context.checkReplicate(value, context.xmlSprayReplicateCheckbox);
+            });
+
+            this.sprayVariableDestinationSelect.on('change', function (value){
+                context.checkReplicate(value, context.variableSprayReplicateCheckbox);
+            });
+
+            this.sprayBlobDestinationSelect.on('change', function (value){
+                context.checkReplicate(value, context.blobSprayReplicateCheckbox);
+            });
+        },
+
+        checkReplicate: function (value, checkBoxValue) {
+            FileSpray.GetSprayTargets({
+                request: {}
+            }).then(function (response) {
+                if (lang.exists("GetSprayTargetsResponse.GroupNodes.GroupNode", response)) {
+                    var arr = response.GetSprayTargetsResponse.GroupNodes.GroupNode;
+                    for (var index in arr) {
+                        if (arr[index].Name === value && arr[index].ReplicateOutputs === true) {
+                            checkBoxValue.set("disabled", false);
+                            break;
+                        } else if (arr[index].Name === value) {
+                            checkBoxValue.set("disabled", true);
+                            break;
+                        }
+                    }
+                }
+            });
         },
 
         initTab: function () {

esp/src/eclwatch/templates/DFUQueryWidget.html

@@ -35,7 +35,7 @@
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Wrap}:" name="Wrap" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}RemoteCopyReplicate"title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.RetainSuperfileStructure}:" name="superCopy" data-dojo-type="dijit.form.CheckBox" />
                                     </div>
                                 </div>

esp/src/eclwatch/templates/LZBrowseWidget.html

@@ -45,7 +45,7 @@
                                     <legend>${i18n.Options}</legend>
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}FixedSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
@@ -91,7 +91,7 @@
                                         <input id="${id}SprayDelimitedTerminators" title="${i18n.LineTerminators}:" name="sourceCsvTerminate" style="width: 95%;" value="\n,\r\n" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'\\n,\\r\\n'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input id="${id}SprayDelimitedQuote" title="${i18n.Quote}:" style="width: 95%;" name="sourceCsvQuote" value='"' data-data-dojo-props="trim: true, placeHolder:'\''" colspan="2" data-dojo-type="dijit.form.TextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}DelimitedSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
@@ -133,7 +133,7 @@
                                         </select>
                                         <input id="${id}SprayXmlMaxRecordLength" title="${i18n.MaxRecordLength}:" value="8192" style="width: 95%;" name="sourceMaxRecordSize" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}XMLSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
@@ -173,7 +173,7 @@
                                         </select>
                                         <input id="${id}SprayJsonMaxRecordLength" title="${i18n.MaxRecordLength}:" value="8192" style="width: 95%;" name="sourceMaxRecordSize" required="true" colspan="2" data-dojo-props="trim: true, placeHolder:'8192'" data-dojo-type="dijit.form.ValidationTextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}JSONSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
@@ -208,7 +208,7 @@
                                             <option value="variablebigendian">${i18n.VariableBigendian}</option>
                                         </select>
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}VariableSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />
@@ -238,7 +238,7 @@
                                     <div data-dojo-props="cols:2" data-dojo-type="hpcc.TableContainer">
                                         <input title="${i18n.BlobPrefix}:" style="width: 95%;" name="prefix" data-dojo-props="trim: true, placeHolder:'${i18n.PrefixPlaceholder}'" colspan="2" data-dojo-type="dijit.form.TextBox" />
                                         <input title="${i18n.Overwrite}:" name="overwrite" data-dojo-type="dijit.form.CheckBox" />
-                                        <input title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
+                                        <input id="${id}BlobSprayReplicate" title="${i18n.Replicate}:" name="replicate" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.NoSplit}:" name="nosplit" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.Compress}:" name="compress" data-dojo-type="dijit.form.CheckBox" />
                                         <input title="${i18n.FailIfNoSourceFile}:" name="failIfNoSourceFile" data-dojo-type="dijit.form.CheckBox" />

system/jlib/jdebug.cpp

@@ -892,7 +892,7 @@ unsigned getAffinityCpus()
 {
     unsigned numCpus = 0;
     DWORD ProcessAffinityMask, SystemAffinityMask;
-    if (GetProcessAffinityMask(GetCurrentProcess(), &ProcessAffinityMask, &SystemAffinityMask))
+    if (GetProcessAffinityMask(GetCurrentProcess(), (PDWORD_PTR)&ProcessAffinityMask, (PDWORD_PTR)&SystemAffinityMask))
     {
         unsigned i = 0;
         while (ProcessAffinityMask)

system/security/LdapSecurity/ldapconnection.cpp

@@ -2745,12 +2745,12 @@ public:
             m_ldapconfig->getLdapHost(server);
             fullserver.append(server.str());
             LPWSTR whost = (LPWSTR)alloca((fullserver.length() +1) * sizeof(WCHAR));
-            ConvertCToW(whost, fullserver.str());
+            ConvertCToW((unsigned short *)whost, fullserver.str());
 
             LPWSTR wusername = (LPWSTR)alloca((strlen(username) + 1) * sizeof(WCHAR));
-            ConvertCToW(wusername, username);
+            ConvertCToW((unsigned short *)wusername, username);
             LPWSTR wnewpasswd = (LPWSTR)alloca((strlen(newPassword) + 1) * sizeof(WCHAR));
-            ConvertCToW(wnewpasswd, newPassword);
+            ConvertCToW((unsigned short *)wnewpasswd, newPassword);
             usriSetPassword.usri1003_password  = wnewpasswd;
             nStatus = NetUserSetInfo(whost, wusername,  1003, (LPBYTE)&usriSetPassword, NULL);