Browse Source

Merge remote-tracking branch 'origin/candidate-3.8.x'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>
Richard Chapman 13 years ago
parent
commit
0d4ea125c0
51 changed files with 338 additions and 188 deletions
  1. 38 5
      docs/Installing_and_RunningTheHPCCPlatform/user_Sect.xml
  2. 14 69
      docs/UsingConfigManager/UsingConfigManager.xml
  3. BIN
      docs/images/CM-img08-1.jpg
  4. BIN
      docs/images/CM-img08-2.jpg
  5. BIN
      docs/images/CM-img08-3.jpg
  6. BIN
      docs/images/CM-img08-4.jpg
  7. BIN
      docs/images/CM-img08-5.jpg
  8. BIN
      docs/images/CM-img08.jpg
  9. BIN
      docs/images/ECLWMyAccount.jpg
  10. BIN
      docs/images/GS_1311.jpg
  11. BIN
      docs/images/LDAP_005.jpg
  12. BIN
      docs/images/Permissions001.jpg
  13. BIN
      docs/images/Permissions002.jpg
  14. BIN
      docs/images/Permissions003.jpg
  15. BIN
      docs/images/Permissions004.jpg
  16. BIN
      docs/images/Permissions005.jpg
  17. BIN
      docs/images/VM1000.jpg
  18. 3 1
      ecl/hql/hqlexpr.cpp
  19. 0 2
      ecl/hql/hqlgram.y
  20. 2 0
      ecl/hqlcpp/hqlttcpp.cpp
  21. 28 0
      ecl/regress/packed3.ecl
  22. 1 1
      esp/bindings/http/platform/httpbinding.cpp
  23. 0 1
      esp/services/ecldirect/EclDirectService.hpp
  24. 6 6
      esp/services/ws_workunits/ws_workunitsHelpers.cpp
  25. 2 2
      esp/services/ws_workunits/ws_workunitsHelpers.hpp
  26. 2 2
      esp/services/ws_workunits/ws_workunitsService.cpp
  27. 1 0
      esp/services/ws_workunits/ws_workunitsService.hpp
  28. 1 1
      initfiles/componentfiles/configxml/dali.xsd
  29. 29 0
      initfiles/componentfiles/configxml/thor.xsl
  30. 2 1
      initfiles/componentfiles/thor/CMakeLists.txt
  31. 2 2
      initfiles/componentfiles/thor/start_slave
  32. 28 0
      initfiles/componentfiles/thor/stop_slaves
  33. 24 13
      initfiles/componentfiles/thor/stop_thor
  34. 1 0
      initfiles/etc/DIR_NAME/environment.xml.in
  35. 1 1
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/ECLEngine.java
  36. 5 10
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCColumnMetaData.java
  37. 25 26
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCDatabaseMetaData.java
  38. 8 2
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCDriver.java
  39. 28 0
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCQuery.java
  40. 3 3
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCResultSetMetadata.java
  41. 11 0
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCVersionTracker.java
  42. 16 10
      plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/SQLParser.java
  43. 1 1
      system/security/LdapSecurity/ldapconnection.cpp
  44. 5 5
      thorlcr/activities/filter/thfilterslave.cpp
  45. 1 1
      thorlcr/activities/rollup/throllupslave.cpp
  46. 13 8
      thorlcr/master/thgraphmanager.cpp
  47. 1 1
      thorlcr/master/thgraphmanager.hpp
  48. 5 3
      thorlcr/master/thmastermain.cpp
  49. 12 2
      thorlcr/slave/slavmain.cpp
  50. 18 8
      thorlcr/slave/thslavemain.cpp
  51. 1 1
      thorlcr/thorutil/thmem.cpp

+ 38 - 5
docs/Installing_and_RunningTheHPCCPlatform/user_Sect.xml

@@ -81,13 +81,46 @@
     <para>Access to the permission settings for each of these areas is
     available using the <emphasis role="bold">Users/Permissions</emphasis>
     area in ECL Watch.</para>
+
+    <sect3 role="brk">
+      <title>Information about your account</title>
+
+      <para>To find out more information about your account, in ECL Watch
+      click on the <emphasis role="bold">My Account</emphasis> link.</para>
+
+      <para><graphic fileref="../images/ECLWMyAccount.jpg" /> <orderedlist>
+          <listitem>
+            <para>Click on the <emphasis role="bold">My Account</emphasis>
+            link.</para>
+
+            <para>Information about your account is displayed.</para>
+          </listitem>
+
+          <listitem>
+            <para>Confirm the User Name that you are logged in as.</para>
+
+            <para>Note that Administrator rights are needed to manage users
+            and permissions.</para>
+
+            <para>Ensure you are using an account with Administrator rights if
+            you intend to manage users or permissions.</para>
+          </listitem>
+
+          <listitem>
+            <para>Verify the password expiration date, or if password is set
+            to expire.</para>
+          </listitem>
+        </orderedlist></para>
+
+      <para></para>
+    </sect3>
   </sect2>
 
   <sect2>
     <title>Security Administration using ECL Watch</title>
 
-    <para>Administrator rights are needed to manager permissions. Once you
-    have administrator access rights, open ECL Watch in your browser using the
+    <para>Administrator rights are needed to manage permissions. Once you have
+    administrator access rights, open ECL Watch in your browser using the
     following URL:</para>
 
     <itemizedlist>
@@ -298,7 +331,7 @@
             </listitem>
 
             <listitem>
-              <para>Click OK to confirm. </para>
+              <para>Click OK to confirm.</para>
 
               <para>Confirmation of the request is shown.</para>
             </listitem>
@@ -327,7 +360,7 @@
             </listitem>
 
             <listitem>
-              <para>Click <emphasis role="bold">Submit</emphasis>. </para>
+              <para>Click <emphasis role="bold">Submit</emphasis>.</para>
 
               <para>Confirmation of the request is shown.</para>
             </listitem>
@@ -347,7 +380,7 @@
             </listitem>
 
             <listitem>
-              <para>Click <emphasis role="bold">Delete</emphasis>. </para>
+              <para>Click <emphasis role="bold">Delete</emphasis>.</para>
 
               <para>Confirmation of the request is shown.</para>
             </listitem>

+ 14 - 69
docs/UsingConfigManager/UsingConfigManager.xml

@@ -799,7 +799,7 @@ sudo configmgr</programlisting>
           <sect4>
             <title>DaliServer LDAP options</title>
 
-            <para>This section describes the DaliServer LDAP tab. </para>
+            <para>This section describes the DaliServer LDAP tab.</para>
 
             <para><graphic fileref="images/CM-img04-4.jpg" /></para>
 
@@ -1761,6 +1761,12 @@ sudo configmgr</programlisting>
 
                       <entry>Provides Web access to published queries</entry>
                     </row>
+
+                    <row>
+                      <entry><emphasis>myecldirect</emphasis></entry>
+
+                      <entry>Provides Ecl direct access</entry>
+                    </row>
                   </tbody>
                 </tgroup>
               </informaltable></para>
@@ -1791,12 +1797,6 @@ sudo configmgr</programlisting>
                     </row>
 
                     <row>
-                      <entry><emphasis>AccurintSecurity</emphasis></entry>
-
-                      <entry>the AccurintSecurity pass...</entry>
-                    </row>
-
-                    <row>
                       <entry><emphasis>ldapAuthMethod</emphasis></entry>
 
                       <entry>the LDAP Authentication Method</entry>
@@ -1819,6 +1819,13 @@ sudo configmgr</programlisting>
 
                       <entry>the method...</entry>
                     </row>
+
+                    <row>
+                      <entry><emphasis>passwordExpirationWarningDays</emphasis></entry>
+
+                      <entry>the number of days advance warning of password
+                      expiration</entry>
+                    </row>
                   </tbody>
                 </tgroup>
               </informaltable></para>
@@ -1947,68 +1954,6 @@ sudo configmgr</programlisting>
           </sect4>
 
           <sect4>
-            <title>Esp - myesp Protocol XTab</title>
-
-            <para>This section describes the Esp - myesp Protocol X
-            tab.</para>
-
-            <para><graphic fileref="images/CM-img08-5.jpg" /></para>
-
-            <para><informaltable colsep="0" frame="none" rowsep="0">
-                <tgroup cols="2">
-                  <colspec align="left" colwidth="122.40pt" />
-
-                  <colspec />
-
-                  <tbody>
-                    <row>
-                      <entry>Attribute name</entry>
-
-                      <entry>Definition</entry>
-                    </row>
-
-                    <row>
-                      <entry><emphasis>Authentication
-                      Timeout</emphasis></entry>
-
-                      <entry>The Authentication timeout value...</entry>
-                    </row>
-
-                    <row>
-                      <entry><emphasis>Default Time out </emphasis></entry>
-
-                      <entry>The default timeout value...</entry>
-                    </row>
-
-                    <row>
-                      <entry><emphasis>ldle Timeout</emphasis></entry>
-
-                      <entry>The timeout value...</entry>
-                    </row>
-
-                    <row>
-                      <entry><emphasis>Maximum Timeout</emphasis></entry>
-
-                      <entry>The maximum timeout value...</entry>
-                    </row>
-
-                    <row>
-                      <entry><emphasis>Minimum Timeout</emphasis></entry>
-
-                      <entry>The minimum timeout value...</entry>
-                    </row>
-
-                    <row>
-                      <entry><emphasis>Thread Count</emphasis></entry>
-
-                      <entry>the number of threads...</entry>
-                    </row>
-                  </tbody>
-                </tgroup>
-              </informaltable></para>
-          </sect4>
-
-          <sect4>
             <title>EspProcess Notes</title>
 
             <para>This tab allows you to add any notes pertinent to the

BIN
docs/images/CM-img08-1.jpg


BIN
docs/images/CM-img08-2.jpg


BIN
docs/images/CM-img08-3.jpg


BIN
docs/images/CM-img08-4.jpg


BIN
docs/images/CM-img08-5.jpg


BIN
docs/images/CM-img08.jpg


BIN
docs/images/ECLWMyAccount.jpg


BIN
docs/images/GS_1311.jpg


BIN
docs/images/LDAP_005.jpg


BIN
docs/images/Permissions001.jpg


BIN
docs/images/Permissions002.jpg


BIN
docs/images/Permissions003.jpg


BIN
docs/images/Permissions004.jpg


BIN
docs/images/Permissions005.jpg


BIN
docs/images/VM1000.jpg


+ 3 - 1
ecl/hql/hqlexpr.cpp

@@ -9643,7 +9643,9 @@ protected:
                     }
 
                     HqlDummyLookupContext dummyctx(ctx.errors);
-                    return newModule->queryScope()->lookupSymbol(selectedName, makeLookupFlags(true, expr->hasProperty(ignoreBaseAtom), false), dummyctx);
+                    IHqlScope * newScope = newModule->queryScope();
+                    assertex(newScope);
+                    return newScope->lookupSymbol(selectedName, makeLookupFlags(true, expr->hasProperty(ignoreBaseAtom), false), dummyctx);
                 }
                 break;
             }

+ 0 - 2
ecl/hql/hqlgram.y

@@ -3976,8 +3976,6 @@ endrecord
                             record->Link();     // link should be in startrecord, but can only link after closeExpr()
                             parser->popSelfScope();
                             OwnedHqlExpr newRecord = record->closeExpr();
-                            if (newRecord->hasProperty(packedAtom))
-                                newRecord.setown(getPackedRecord(newRecord));
                             $$.setExpr(newRecord.getClear());
                             parser->popLocale();
                             $$.setPosition($1);

+ 2 - 0
ecl/hqlcpp/hqlttcpp.cpp

@@ -11021,6 +11021,8 @@ IHqlExpression * HqlTreeNormalizer::createTransformedBody(IHqlExpression * expr)
     case no_record:
         {
             OwnedHqlExpr transformed = completeTransform(expr);
+            if (transformed->hasProperty(packedAtom))
+                transformed.setown(getPackedRecord(transformed));
 
             if (options.ensureRecordsHaveSymbols)
             {

+ 28 - 0
ecl/regress/packed3.ecl

@@ -0,0 +1,28 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+rec := RECORD,packed
+  STRING name;
+  unsigned1 flag;
+  real value;
+END;
+
+
+ds := DATASET([{'Gavin',1,100.0},{'James',2,98.6}], rec);
+
+output(ds);

+ 1 - 1
esp/bindings/http/platform/httpbinding.cpp

@@ -568,7 +568,7 @@ bool EspHttpBinding::basicAuth(IEspContext* ctx)
             if(access < required)
             {
                 const char *desc=curres->getDescription();
-                ESPLOG(LogMax, "Access denied to: %s. Access=%d, Required=%d\n", desc?desc:"<no-desc>", access, required);
+                ESPLOG(LogMin, "Access for user '%s' denied to: %s. Access=%d, Required=%d", user->getName(), desc?desc:"<no-desc>", access, required);
                 ctx->AuditMessage(AUDIT_TYPE_ACCESS_FAILURE, "Authorization", "Access Denied: Not Authorized", "Resource: %s [%s]", curres->getName(), (desc) ? desc : "");
                 authorized = false;
                 break;

+ 0 - 1
esp/services/ecldirect/EclDirectService.hpp

@@ -65,7 +65,6 @@ public:
     {
         IPropertyTree *folder = ensureNavFolder(data, "ECL", "Run Ecl code and review Ecl workunits", NULL, false, 2);
         ensureNavLink(*folder, "Run Ecl", "/EclDirect/RunEclEx/Form", "Submit ECL text for execution", NULL, NULL, 3);
-        ensureNavLink(*folder, "ECL Playground", "/esp/files/ECLPlayground.htm", "ECL Editor, Executor, Graph and Result Viewer", NULL, NULL, 4);
     }
 
     virtual int onGet(CHttpRequest* request, CHttpResponse* response);

+ 6 - 6
esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -1402,7 +1402,7 @@ bool WsWuInfo::getResultViews(StringArray &viewnames, unsigned flags)
     return false;
 }
 
-void appendIOStreamContent(MemoryBuffer &mb, IFileIOStream *ios)
+void appendIOStreamContent(MemoryBuffer &mb, IFileIOStream *ios, bool forDownload)
 {
     StringBuffer line;
     bool eof = false;
@@ -1424,7 +1424,7 @@ void appendIOStreamContent(MemoryBuffer &mb, IFileIOStream *ios)
         }
 
         mb.append(line.length(), line.str());
-        if (mb.length() > 640000)
+        if (!forDownload && (mb.length() > 640000))
             break;
     }
 }
@@ -1538,7 +1538,7 @@ void WsWuInfo::getWorkunitThorLog(MemoryBuffer& buf)
     }
 }
 
-void WsWuInfo::getWorkunitThorSlaveLog(const char *slaveip, MemoryBuffer& buf)
+void WsWuInfo::getWorkunitThorSlaveLog(const char *slaveip, MemoryBuffer& buf, bool forDownload)
 {
    if (isEmpty(slaveip))
       throw MakeStringException(ECLWATCH_INVALID_INPUT,"ThorSlave IP not specified.");
@@ -1569,7 +1569,7 @@ void WsWuInfo::getWorkunitThorSlaveLog(const char *slaveip, MemoryBuffer& buf)
         throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read file %s.",logdir.str());
 
     OwnedIFileIOStream ios = createBufferedIOStream(rIO);
-    appendIOStreamContent(buf, ios.get());
+    appendIOStreamContent(buf, ios.get(), forDownload);
 }
 
 void WsWuInfo::getWorkunitResTxt(MemoryBuffer& buf)
@@ -1621,7 +1621,7 @@ void WsWuInfo::getWorkunitXml(const char* plainText, MemoryBuffer& buf)
     buf.append(xml.length(), xml.str());
 }
 
-void WsWuInfo::getWorkunitCpp(const char *cppname, const char* description, const char* ipAddress, MemoryBuffer& buf)
+void WsWuInfo::getWorkunitCpp(const char *cppname, const char* description, const char* ipAddress, MemoryBuffer& buf, bool forDownload)
 {
     if (isEmpty(description))
         throw MakeStringException(ECLWATCH_INVALID_INPUT, "File not specified.");
@@ -1644,7 +1644,7 @@ void WsWuInfo::getWorkunitCpp(const char *cppname, const char* description, cons
     OwnedIFileIOStream ios = createBufferedIOStream(rIO);
     if (!ios)
         throw MakeStringException(ECLWATCH_CANNOT_READ_FILE,"Cannot read %s.", description);
-    appendIOStreamContent(buf, ios.get());
+    appendIOStreamContent(buf, ios.get(), forDownload);
 }
 
 

+ 2 - 2
esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -166,12 +166,12 @@ public:
 
     void getWorkunitEclAgentLog(MemoryBuffer& buf);
     void getWorkunitThorLog(MemoryBuffer& buf);
-    void getWorkunitThorSlaveLog(const char *slaveip, MemoryBuffer& buf);
+    void getWorkunitThorSlaveLog(const char *slaveip, MemoryBuffer& buf, bool forDownload);
     void getWorkunitResTxt(MemoryBuffer& buf);
     void getWorkunitArchiveQuery(MemoryBuffer& buf);
     void getWorkunitDll(StringBuffer &name, MemoryBuffer& buf);
     void getWorkunitXml(const char* plainText, MemoryBuffer& buf);
-    void getWorkunitCpp(const char* cppname, const char* description, const char* ipAddress, MemoryBuffer& buf);
+    void getWorkunitCpp(const char* cppname, const char* description, const char* ipAddress, MemoryBuffer& buf, bool forDownload);
     void getEventScheduleFlag(IEspECLWorkunit &info);
 
 public:

+ 2 - 2
esp/services/ws_workunits/ws_workunitsService.cpp

@@ -2404,7 +2404,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             }
             else if (strieq(File_Cpp,req.getType()) && notEmpty(req.getName()))
             {
-                winfo.getWorkunitCpp(req.getName(), req.getDescription(), req.getIPAddress(),mb);
+                winfo.getWorkunitCpp(req.getName(), req.getDescription(), req.getIPAddress(),mb, opt > 0);
                 openSaveFile(context, opt, req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_DLL,req.getType()))
@@ -2426,7 +2426,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             }
             else if (strieq(File_ThorSlaveLog,req.getType()))
             {
-                winfo.getWorkunitThorSlaveLog(req.getSlaveIP(), mb);
+                winfo.getWorkunitThorSlaveLog(req.getSlaveIP(), mb, opt > 0);
                 openSaveFile(context, opt, "ThorSlave.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
             else if (strieq(File_EclAgentLog,req.getType()))

+ 1 - 0
esp/services/ws_workunits/ws_workunitsService.hpp

@@ -131,6 +131,7 @@ public:
             IPropertyTree *folder = ensureNavFolder(data, "ECL", "Run Ecl code and review Ecl workunits", NULL, false, 2);
             ensureNavLink(*folder, "Search Workunits", "/WsWorkunits/WUQuery?form_", "Search Workunits", NULL, NULL, 1);
             ensureNavLink(*folder, "Browse Workunits", "/WsWorkunits/WUQuery", "Browse Workunits", NULL, NULL, 2);
+            ensureNavLink(*folder, "ECL Playground", "/esp/files/ECLPlayground.htm", "ECL Editor, Executor, Graph and Result Viewer", NULL, NULL, 4);
 
             IPropertyTree *folderQueryset = ensureNavFolder(data, "Query Sets", NULL, NULL, false, 3);
             ensureNavLink(*folderQueryset, "Browse", "/WsWorkunits/WUQuerySets", "Browse Published Queries");

+ 1 - 1
initfiles/componentfiles/configxml/dali.xsd

@@ -328,7 +328,7 @@
         </xs:restriction>
       </xs:simpleType>
     </xs:attribute>
-    <xs:attribute name="filesBasedn" type="xs:string" use="optional" default="ou=Files,ou=ecl">
+    <xs:attribute name="filesBasedn" type="xs:string" use="required" default="ou=Files,ou=ecl">
       <xs:annotation>
         <xs:appinfo>
           <tooltip>base location for ldap file scopes</tooltip>

+ 29 - 0
initfiles/componentfiles/configxml/thor.xsl

@@ -104,6 +104,35 @@
   </xsl:template>
 
   <xsl:template match="ThorCluster">
+    <xsl:variable name="masterport" select="@masterport"/>
+    <xsl:variable name="slaveport" select="@slaveport"/>
+    <xsl:variable name="masternode" select="ThorMasterProcess/@computer"/>
+    <xsl:for-each select="/Environment/Software/ThorCluster[@name!=string($process)]">
+      <xsl:variable name="thismasterport" select="@masterport"/>
+      <xsl:variable name="thisthor" select="@name"/>
+        <xsl:for-each select="ThorMasterProcess[@computer=$masternode]">
+          <xsl:if test="string($thismasterport)=string($masterport)">
+            <xsl:message terminate="yes">
+              There cannot be more than one ThorCluster ('<xsl:value-of select="$process"/>' and '<xsl:value-of select="$thisthor"/>') with the same thor master '"<xsl:value-of select="$masternode"/>' and same thor master port '<xsl:value-of select="$masterport"/>!
+            </xsl:message>
+          </xsl:if>
+        </xsl:for-each>
+    </xsl:for-each>
+
+    <xsl:for-each select="ThorSlaveProcess">
+      <xsl:variable name="slavenode" select="@computer"/>
+      <xsl:for-each select="/Environment/Software/ThorCluster[@name!=string($process)]">
+        <xsl:variable name="thisslaveport" select="@slaveport"/>
+        <xsl:if test="string($thisslaveport)=string($slaveport)">
+            <xsl:if test="count(ThorSlaveProcess[@computer=$slavenode]) > 0">
+              <xsl:message terminate="yes">
+                There cannot be more than one ThorCluster ('<xsl:value-of select="$process"/>' and '<xsl:value-of select="@name"/>') with the same thor slave '"<xsl:value-of select="$slavenode"/>' and thor slave port '<xsl:value-of select="$slaveport"/>'!
+              </xsl:message>
+          </xsl:if>
+        </xsl:if>
+      </xsl:for-each>
+    </xsl:for-each>
+
     <Thor>
       <xsl:attribute name="name">
         <xsl:value-of select="@name"/>

+ 2 - 1
initfiles/componentfiles/thor/CMakeLists.txt

@@ -17,6 +17,7 @@ FOREACH( iFILES
     ${CMAKE_CURRENT_SOURCE_DIR}/start_thor
     ${CMAKE_CURRENT_SOURCE_DIR}/run_thor
     ${CMAKE_CURRENT_SOURCE_DIR}/stop_thor
+    ${CMAKE_CURRENT_SOURCE_DIR}/stop_slaves
 )
     Install ( PROGRAMS ${iFILES} DESTINATION ${OSSDIR}/bin )
-ENDFOREACH ( iFILES )
+ENDFOREACH ( iFILES )

+ 2 - 2
initfiles/componentfiles/thor/start_slave

@@ -69,8 +69,8 @@ ln -s -f `which $prog` $slaveproc
 
 echo "slave starting `date`"
 
-echo $instancedir/$slaveproc master=$master slave=.:$slaveport logDir=$logpth
-$instancedir/$slaveproc master=$master slave=.:$slaveport logDir=$logpth 2>/dev/null 1>/dev/null &
+echo $instancedir/$slaveproc master=$master slave=.:$slaveport slavenum=$slavenum logDir=$logpth
+$instancedir/$slaveproc master=$master slave=.:$slaveport slavenum=$slavenum logDir=$logpth 2>/dev/null 1>/dev/null &
 slavepid=$!
 echo $slavepid > $PID_NAME
 if [ "$slavepid" -eq "0" ]; then

+ 28 - 0
initfiles/componentfiles/thor/stop_slaves

@@ -0,0 +1,28 @@
+#!/bin/bash
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+hpcc_compname=$1
+hpcc_setenv=$2
+
+source ${hpcc_setenv}
+
+for file in $PID/${hpcc_compname}_slave_*.pid
+do
+  kill -9 `cat ${file}` # > /dev/null
+done
+

+ 24 - 13
initfiles/componentfiles/thor/stop_thor

@@ -20,6 +20,9 @@ PATH_PRE=`type -path hpcc_setenv`
 INSTALL_DIR=`dirname ${PATH_PRE}`/..
 source  ${INSTALL_DIR}/etc/init.d/hpcc_common
 
+source ${PATH_PRE} ""
+export MASTER_PID_NAME="$PID/`basename $PWD`_master.pid" ## this needed when we use bash_init_system
+
 which_pidof
 
 if [ $# -lt 1 ]; then
@@ -44,23 +47,31 @@ if [ "$#" -lt "2" ] || [ "$2" != "keep_sentinel" ]; then
     sleep 1
 fi
 
-masterproc="$instancedir/thormaster_$THORNAME"
-while [ "`${PIDOF} $masterproc`" != "" ]
-do
-  echo --------------------------
-  echo stopping thormaster $THORMASTER 
-  killall $masterproc
-  sleep 8
-  killall -9 $masterproc
-  sleep 1
-done
+masterpid=`cat ${MASTER_PID_NAME} 2> /dev/null`
+if [ ! -z $masterpid ]; then
+  while :
+  do
+    kill -0 $masterpid >& /dev/null
+    masterRunning=$(( $? == 0 ? 1 : 0 ))
+    if [ 0 == $masterRunning ]; then
+      break
+    fi
+    echo --------------------------
+    echo stopping thormaster $THORMASTER 
+    kill $masterpid >& /dev/null
+    sleep 8
+    kill -9 $masterpid >& /dev/null
+    sleep 1
+  done
+fi
 
 echo --------------------------
 echo stopping thor slaves 
 
-slaveproc="$instancedir/thorslave_$THORNAME"
+SLAVE_PID_NAME="$PID/${hpcc_compname}_slave_${slavenum}.pid"
 if [ "$localthor" = "true" ]; then
-    killall -9 $slaveproc
+    slavepid=`cat ${SLAVE_PID_NAME} 2> /dev/null`
+    kill -9 $slavepid > /dev/null
 else
     # use 20 threads
     # timeout of 60 seconds (in case slave busy)
@@ -78,7 +89,7 @@ else
             exit 1
         fi
     fi
-    $deploydir/frunssh $instancedir/slaves "/bin/sh -c 'killall -9 $slaveproc'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries 2>&1 | egrep -v "no process killed"
+    $deploydir/frunssh uniq $instancedir/slaves "/bin/sh -c '$deploydir/stop_slaves ${hpcc_compname} $PATH_PRE'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries 2>&1 | egrep -v "no process killed"
     echo slaves stopped
 fi
 

+ 1 - 0
initfiles/etc/DIR_NAME/environment.xml.in

@@ -384,6 +384,7 @@
   <DaliServerProcess build="${projname}_${version}-${stagever}"
                      buildSet="dali"
                      environment="${CONFIG_DIR}/${ENV_XML_FILE}"
+                     filesBasedn="ou=Files,ou=ecl"
                      name="mydali"
                      recoverFromIncErrors="true">
    <Instance computer="localhost"

+ 1 - 1
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/ECLEngine.java

@@ -228,7 +228,7 @@ public class ECLEngine
 									eclEnteties.put("SCALAROUTNAME", col.getColumnName());
 							}
 
-							col.setSQLType(java.sql.Types.NUMERIC);
+							col.setSqlType(java.sql.Types.NUMERIC);
 						}
 						else if (col.getColumnName().equalsIgnoreCase("MAX"))
 						{

+ 5 - 10
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCColumnMetaData.java

@@ -87,16 +87,6 @@ public class HPCCColumnMetaData {
 		return constantValue;
 	}
 
-	public int getSQLType()
-	{
-		return sqlType;
-	}
-
-	public void setSQLType(int type)
-	{
-		this.sqlType = type;
-	}
-
 	public String getColumnName()
 	{
 		return columnName;
@@ -152,6 +142,11 @@ public class HPCCColumnMetaData {
 	// return (String)this.restrictions.get(restriction);
 	// }
 
+	public void setSqlType(int type)
+	{
+		sqlType = type;
+	}
+
 	public int getSqlType()
 	{
 		return sqlType;

+ 25 - 26
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCDatabaseMetaData.java

@@ -895,7 +895,7 @@ public class HPCCDatabaseMetaData implements DatabaseMetaData {
 
 	@Override
 	public boolean supportsOuterJoins() throws SQLException {
-		return true;
+		return false;
 	}
 
 	@Override
@@ -1214,6 +1214,7 @@ public class HPCCDatabaseMetaData implements DatabaseMetaData {
 		if (!isQuerySetMetaDataCached())
 			setQuerySetMetaDataCached(fetchHPCCQueriesInfo());
 
+		boolean wildprocsearch = procedureNamePattern == null || procedureNamePattern.length()==0 || procedureNamePattern.contains("*") || procedureNamePattern.contains("%");
 		System.out.println("ECLDATABASEMETADATA GETPROCS catalog: " + catalog +", schemaPattern: " + schemaPattern + ", procedureNamePattern: " + procedureNamePattern);
 
 		metacols.add(new HPCCColumnMetaData("PROCEDURE_CAT", 1, java.sql.Types.VARCHAR));
@@ -1231,19 +1232,22 @@ public class HPCCDatabaseMetaData implements DatabaseMetaData {
 			HPCCQuery query = (HPCCQuery) queries.nextElement();
 			String queryname = query.getName();
 
-				//if(!wildproceduresearch && !procedureNamePattern.equalsIgnoreCase(queryname))
-				//	continue;
+			if(!wildprocsearch && !query.isQueryNameMatch(procedureNamePattern))
+				continue;
 
-				ArrayList rowValues = new ArrayList();
-				procedures.add(rowValues);
-				rowValues.add("");
-				rowValues.add(query.getQuerySet());
-				rowValues.add(query.getQuerySet()+"::"+queryname);
-				rowValues.add("");
-				rowValues.add("");
-				rowValues.add("");
-				rowValues.add("QuerySet: " + query.getQuerySet());
-				rowValues.add(procedureResultUnknown);
+			ArrayList rowValues = new ArrayList();
+			procedures.add(rowValues);
+			rowValues.add("");
+			rowValues.add(query.getQuerySet());
+			rowValues.add(query.getQuerySet()+"::"+queryname);
+			rowValues.add("");
+			rowValues.add("");
+			rowValues.add("");
+			rowValues.add("QuerySet: " + query.getQuerySet());
+			rowValues.add(procedureResultUnknown);
+
+			if(!wildprocsearch)
+				break;
 		}
 
 		return new HPCCResultSet(procedures, metacols, "Procedures");
@@ -1260,8 +1264,6 @@ public class HPCCDatabaseMetaData implements DatabaseMetaData {
 		if (!isQuerySetMetaDataCached())
 			setQuerySetMetaDataCached(fetchHPCCQueriesInfo());
 
-		System.out.println("ECLDATABASEMETADATA GETPROCS catalog: " + catalog +", schemaPattern: " + schemaPattern + ", procedureNamePattern: " + procedureNamePattern);
-
 		boolean wildprocsearch = procedureNamePattern == null || procedureNamePattern.length()==0 || procedureNamePattern.contains("*") || procedureNamePattern.contains("%");
 		boolean wildcolumnsearch = columnNamePattern == null || columnNamePattern.length()==0 || columnNamePattern.contains("*") || columnNamePattern.contains("%");
 
@@ -1285,31 +1287,29 @@ public class HPCCDatabaseMetaData implements DatabaseMetaData {
 		while(queries.hasMoreElements())
 		{
 			HPCCQuery query = (HPCCQuery)queries.nextElement();
-			String eclqueryname = query.getName();
-			if(!wildprocsearch && !procedureNamePattern.equalsIgnoreCase(eclqueryname))
-				continue;
 
-			String tablename = query.getDefaultTableName();
+			if(!wildprocsearch && !query.isQueryNameMatch(procedureNamePattern))
+				continue;
 
-			Iterator<HPCCColumnMetaData> e = query.getAllFields().iterator();
+			Iterator<HPCCColumnMetaData> queryfields = query.getAllFields().iterator();
 
-			while (e.hasNext())
+			while (queryfields.hasNext())
 			{
-				HPCCColumnMetaData col = (HPCCColumnMetaData)e.next();
+				HPCCColumnMetaData col = (HPCCColumnMetaData)queryfields.next();
 				String fieldname = col.getColumnName();
 				if(!wildcolumnsearch && !columnNamePattern.equalsIgnoreCase(fieldname))
 					continue;
 
-				coltype = getProcFieldType(eclqueryname,tablename, fieldname);
+				coltype = col.getSqlType();
 
-				System.out.println("Proc col Found: "  + eclqueryname+"#"+tablename+"."+fieldname + " of type: " + coltype + "("+convertSQLtype2JavaClassName(coltype)+")");
+				System.out.println("Proc col Found: "  + query.getName() + "." + fieldname + " of type: " + coltype + "("+convertSQLtype2JavaClassName(coltype)+")");
 
 				ArrayList rowValues = new ArrayList();
 				procedurecols.add(rowValues);
 
 				/* 1*/rowValues.add(catalog);
 				/* 2*/rowValues.add(schemaPattern);
-				/* 3*/rowValues.add(query.getName());
+				/* 3*/rowValues.add(query.getQuerySet()+"::"+query.getName());
 				/* 4*/rowValues.add(fieldname);
 				/* 5*/rowValues.add(col.getParamType());
 				/* 6*/rowValues.add(coltype);
@@ -1517,7 +1517,6 @@ public class HPCCDatabaseMetaData implements DatabaseMetaData {
 		while(files.hasMoreElements())
 		{
 			DFUFile file = (DFUFile)files.nextElement();
-			//String filename = file.getFileName();
 			String filename = file.getFullyQualifiedName();
 			if(!wildtablesearch && !tableNamePattern.equalsIgnoreCase(filename))
 				continue;

+ 8 - 2
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCDriver.java

@@ -135,7 +135,7 @@ public class HPCCDriver implements Driver
 			//"select 1 as ONE"
 			//"call myroxie::fetchpeoplebyzipservice(33445)"
 			//"call fetchpeoplebyzipservice(33445)"
-			"call fetchpeoplebyzipservice()"
+			"call fetchpeoplebyzipservice(33445)"
 			//"select MIN(zip), city from tutorial::rp::tutorialperson where zip  > '33445'"
 
 			//"select tbl.* from progguide::exampledata::peopleaccts tbl"
@@ -285,8 +285,14 @@ public class HPCCDriver implements Driver
 			ResultSet procs = conn.getMetaData().getProcedures(null, null, null);
 
 			System.out.println("procs found: ");
-			while (procs.next()) {
+			while (procs.next())
+			{
 				System.out.println("   " + procs.getString("PROCEDURE_NAME"));
+				ResultSet proccols = conn.getMetaData().getProcedureColumns(null,null,procs.getString("PROCEDURE_NAME"),"%");
+				while (proccols.next())
+				{
+					System.out.println(proccols.getString("PROCEDURE_NAME")+"."+ proccols.getString("COLUMN_NAME"));
+				}
 			}
 
 			/*

+ 28 - 0
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCQuery.java

@@ -261,4 +261,32 @@ public class HPCCQuery
 	{
 		QuerySet = qsname;
 	}
+
+	public boolean isQueryNameMatch(String fullname)
+	{
+		String querysplit [] = fullname.split("::");
+		String inname ="";
+		String inqs ="";
+		if(querysplit.length > 2)
+		{
+			for (int i = 1; i < querysplit.length; i++)
+				inname += "::" + querysplit[i];
+			inqs = querysplit[0];
+		}
+		else if (querysplit.length == 2)
+		{
+			inqs = querysplit[0];
+			inname = querysplit[1];
+		}
+		else if (querysplit.length == 1)
+			inname = querysplit[0];
+		else
+			return false;
+
+		if (!inname.equalsIgnoreCase(this.Name) ||
+			(inqs.length() > 0 && !inqs.equalsIgnoreCase(this.QuerySet)))
+			return false;
+
+		return true;
+	}
 }

+ 3 - 3
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCResultSetMetadata.java

@@ -105,7 +105,7 @@ public class HPCCResultSetMetadata implements ResultSetMetaData{
     public int getColumnType(int column) throws SQLException
     {
 		if (column >= 1 && column <= columnList.size())
-			return columnList.get(column - 1).getSQLType();
+			return columnList.get(column - 1).getSqlType();
 		else
 			throw new SQLException("Invalid Column Index = " + column);
     }
@@ -113,7 +113,7 @@ public class HPCCResultSetMetadata implements ResultSetMetaData{
     public String getColumnTypeName(int column) throws SQLException
     {
 		if (column >= 1 && column <= columnList.size())
-			return HPCCDatabaseMetaData.getFieldName(columnList.get(column - 1).getSQLType());
+			return HPCCDatabaseMetaData.getFieldName(columnList.get(column - 1).getSqlType());
 		else
 			throw new SQLException("Invalid Column Index = " + column);
     }
@@ -137,7 +137,7 @@ public class HPCCResultSetMetadata implements ResultSetMetaData{
     {
     	if (column >= 1 && column <= columnList.size())
     	{
-    		return HPCCDatabaseMetaData.convertSQLtype2JavaClassName(columnList.get(column - 1).getSQLType());
+			return HPCCDatabaseMetaData.convertSQLtype2JavaClassName(columnList.get(column - 1).getSqlType());
 		}
 		else
 		{

+ 11 - 0
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/HPCCVersionTracker.java

@@ -0,0 +1,11 @@
+package com.hpccsystems.jdbcdriver;
+
+public class HPCCVersionTracker
+{
+	static final String HPCCProject 	= "PLACEHOLDER";
+	static final int HPCCMajor 			= 0;
+	static final int HPCCMinor 			= 0;
+	static final int HPCCPoint 			= 0;
+	static final String HPCCPMaturity 	= "PLACEHOLDER";
+	static final int HPCCSequence 		= 0;
+}

+ 16 - 10
plugins/dbconnectors/hpccjdbc/src/com/hpccsystems/jdbcdriver/SQLParser.java

@@ -229,18 +229,22 @@ public class SQLParser
 			}
 
 			String splittablefromalias [] = fullTableName.split("\\s+(?i)as(\\s+|$)");
-			if (splittablefromalias != null && splittablefromalias.length > 0)
+			if (splittablefromalias.length == 1)
 			{
-				if (!splittablefromalias[0].contains(" "))
-					tableName = splittablefromalias[0].trim();
-				else
+				String splittablebyblank [] = splittablefromalias[0].trim().split("\\s+");
+				tableName = splittablebyblank[0];
+				if (splittablebyblank.length==2)
+					tableAlias = splittablebyblank[1].trim();
+				else if (splittablebyblank.length>2)
 					throw new SQLException("Invalid SQL: " + splittablefromalias[0]);
 			}
-			else
-				throw new SQLException("Invalid SQL: Missing table name.");
-
-			if (splittablefromalias.length > 1)
+			else if (splittablefromalias.length == 2)
+			{
+				tableName = splittablefromalias[0].trim();
 				tableAlias = splittablefromalias[1].trim();
+			}
+			else
+				throw new SQLException("Invalid SQL: " + fullTableName);
 
 			if (fromstrpos <= 7)
 				throw new SQLException("Invalid SQL: Missing select column(s).");
@@ -301,6 +305,8 @@ public class SQLParser
 						colmetadata = new HPCCColumnMetaData(funcname, sqlcolpos++, funccols);
 					else
 						throw new SQLException("Function " + funcname + " does not map to ECL as written");
+
+					colmetadata.setSqlType(func.getReturnType().getSqlType());
 				}
 				else if(col.contains("."))
 				{
@@ -760,7 +766,7 @@ public class SQLParser
 				{
 					column.setColumnName("ConstStr"+ column.getIndex());
 					column.setEclType("STRING");
-					column.setSQLType(java.sql.Types.VARCHAR);
+					column.setSqlType(java.sql.Types.VARCHAR);
 					column.setColumnType(HPCCColumnMetaData.COLUMN_TYPE_CONSTANT);
 					column.setConstantValue(fieldName);
 				}
@@ -768,7 +774,7 @@ public class SQLParser
 				{
 					column.setColumnName("ConstNum" + column.getIndex());
 					column.setEclType("INTEGER");
-					column.setSQLType(java.sql.Types.NUMERIC);
+					column.setSqlType(java.sql.Types.NUMERIC);
 					column.setColumnType(HPCCColumnMetaData.COLUMN_TYPE_CONSTANT);
 					column.setConstantValue(fieldName);
 				}

+ 1 - 1
system/security/LdapSecurity/ldapconnection.cpp

@@ -1180,7 +1180,7 @@ public:
                     rc = LdapUtils::LdapBind(user_ld, m_ldapconfig->getDomain(), username, password, userdnbuf.str(), m_ldapconfig->getServerType(), m_ldapconfig->getAuthMethod());
                     ldap_unbind(user_ld);
                 }
-                DBGLOG("finished LdapBind for user %s", username);
+                DBGLOG("finished LdapBind for user %s, rc=%d", username, rc);
                 if(!LdapServerDown(rc) || retries > LDAPSEC_MAX_RETRIES)
                     break;
                 sleep(LDAPSEC_RETRY_WAIT);

+ 5 - 5
thorlcr/activities/filter/thfilterslave.cpp

@@ -309,11 +309,11 @@ public:
 
         if (groupStream)
         {
-			loop
-			{
-				OwnedConstThorRow row = groupStream->nextRow();
-				if (!row)
-					break;
+            loop
+            {
+                OwnedConstThorRow row = groupStream->nextRow();
+                if (!row)
+                break;
                 if (stepCompare->docompare(row, seek, numFields) >= 0)
                 {
                     dataLinkIncrement();

+ 1 - 1
thorlcr/activities/rollup/throllupslave.cpp

@@ -141,7 +141,7 @@ public:
         }
         return NULL;
     }
-	virtual void stop() { }
+    virtual void stop() { }
 };
 
 class CDedupRollupBaseActivity : public CSlaveActivity, implements IStopInput

+ 13 - 8
thorlcr/master/thgraphmanager.cpp

@@ -46,7 +46,7 @@
 #include "thdemonserver.hpp"
 #include "thgraphmanager.hpp"
 
-class CJobManager: public CSimpleInterface, implements IJobManager, implements IExceptionHandler
+class CJobManager : public CSimpleInterface, implements IJobManager, implements IExceptionHandler
 {
     bool stopped, handlingConversation;
     Owned<IConversation> conversation;
@@ -59,6 +59,7 @@ class CJobManager: public CSimpleInterface, implements IJobManager, implements I
     Owned<IDeMonServer> demonServer;
     atomic_t            activeTasks;
     StringAttr          currentWuid;
+    ILogMsgHandler *logHandler;
 
     bool executeGraph(IConstWorkUnit &workunit, const char *graphName, const SocketEndpoint &agentEp);
     void addJob(CJobMaster &job) { CriticalBlock b(jobCrit); jobs.append(job); }
@@ -67,7 +68,7 @@ class CJobManager: public CSimpleInterface, implements IJobManager, implements I
 public:
     IMPLEMENT_IINTERFACE_USING(CSimpleInterface);
 
-    CJobManager();
+    CJobManager(ILogMsgHandler *logHandler);
     ~CJobManager();
 
     bool doit(IConstWorkUnit *workunit, const char *graphName, const SocketEndpoint &agentep);
@@ -89,7 +90,7 @@ public:
 
 // CJobManager impl.
 
-CJobManager::CJobManager()
+CJobManager::CJobManager(ILogMsgHandler *_logHandler) : logHandler(_logHandler)
 {
     stopped = handlingConversation = false;
     addThreadExceptionHandler(this);
@@ -643,7 +644,10 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName,
         Owned<IWorkUnit> wu = &workunit.lock();
         wu->setTracingValue("ThorBuild", BUILD_TAG);
         // expect there to be 1 or 2 of these, so scan/check if log exists already and add if not
-        const char *nLog = globals->queryProp("@logURL");
+        StringBuffer log, logUrl;
+        logHandler->getLogName(log);
+        createUNCFilename(log, logUrl, false);
+        const char *nLog = logUrl.str();
         Owned<IStringIterator> siter = &wu->getDebugValues("ThorLog*");
         unsigned last=0;
         bool found=false;
@@ -718,7 +722,7 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName,
     SCMStringBuffer user, eclstr;
     workunit.getUser(user);
 
-    LOG(MCdebugProgress, thorJob, "Started wuid=%s, user=%s, graph=%s\n**", wuid.str(), user.str(), graphName);
+    PROGLOG("Started wuid=%s, user=%s, graph=%s\n", wuid.str(), user.str(), graphName);
 
     PROGLOG("Query %s loaded", compoundPath.str());
     Owned<IConstWUGraph> graph = workunit.getGraph(graphName);
@@ -767,7 +771,8 @@ bool CJobManager::executeGraph(IConstWorkUnit &workunit, const char *graphName,
         throw;
     }
     job.clear();
-    LOG(MCdebugProgress, thorJob, "Finished wuid=%s", wuid.str());
+    PROGLOG("Finished wuid=%s, graph=%s", wuid.str(), graphName);
+
     setWuid(NULL);
     return allDone;
 }
@@ -852,7 +857,7 @@ void closeThorServerStatus()
     }
 }
 
-void thorMain()
+void thorMain(ILogMsgHandler *logHandler)
 {
     aborting = 0;
     unsigned multiThorMemoryThreshold = globals->getPropInt("@multiThorMemoryThreshold")*0x100000;
@@ -876,7 +881,7 @@ void thorMain()
         CThorResourceMaster masterResource;
         setIThorResource(masterResource);
 
-        Owned<CJobManager> jobManager = new CJobManager();
+        Owned<CJobManager> jobManager = new CJobManager(logHandler);
         try {
             LOG(MCdebugProgress, thorJob, "Listening for graph");
             jobManager->run();

+ 1 - 1
thorlcr/master/thgraphmanager.hpp

@@ -24,7 +24,7 @@ interface IException;
 CSDSServerStatus &queryServerStatus();
 CSDSServerStatus &openThorServerStatus();
 void closeThorServerStatus();
-void thorMain();
+void thorMain(ILogMsgHandler *logHandler);
 void abortThor(IException *e=NULL, bool abortCurrentJob=true);
 void setExitCode(int code);
 int queryExitCode();

+ 5 - 3
thorlcr/master/thmastermain.cpp

@@ -486,13 +486,15 @@ int main( int argc, char *argv[]  )
     StringBuffer nodeGroup, logUrl;
     Owned<IPerfMonHook> perfmonhook;
 
-    try {
+    ILogMsgHandler *logHandler;
+    try
+    {
         {
             Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(globals, "thor");
             lf->setName("thormaster");//override default filename
             lf->setCreateAliasFile(false);
             lf->setMsgFields(MSGFIELD_timeDate | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code);
-            lf->beginLogging();
+            logHandler = lf->beginLogging();
             createUNCFilename(lf->queryLogFileSpec(), logUrl, false);
         }
         LOG(MCdebugProgress, thorJob, "Opened log file %s", logUrl.toCharArray());
@@ -738,7 +740,7 @@ int main( int argc, char *argv[]  )
             LOG(daliAuditLogCat, ",Progress,Thor,Startup,%s,%s,%s,%s",nodeGroup.str(),thorname,queueName.str(),logUrl.str());
             auditStartLogged = true;
 
-            thorMain();
+            thorMain(logHandler);
             LOG(daliAuditLogCat, ",Progress,Thor,Terminate,%s,%s,%s",thorname,nodeGroup.str(),queueName.str());
         }
         else

+ 12 - 2
thorlcr/slave/slavmain.cpp

@@ -229,8 +229,14 @@ public:
                         }
                         else
                             soPath.append(remoteSoPath);
-                        PROGLOG("Using query: %s", soPath.str());
+
                         Owned<IPropertyTree> workUnitInfo = createPTree(msg);
+                        StringBuffer user;
+                        workUnitInfo->getProp("user", user);
+
+                        PROGLOG("Started wuid=%s, user=%s, graph=%s\n", wuid.get(), user.str(), graphName.get());
+
+                        PROGLOG("Using query: %s", soPath.str());
                         Owned<CJobSlave> job = new CJobSlave(watchdog, workUnitInfo, graphName, soPath.str(), mptag, slaveMsgTag);
                         jobs.replace(*LINK(job));
 
@@ -245,11 +251,15 @@ public:
                     {
                         StringAttr key;
                         msg.read(key);
-                        PROGLOG("QueryDone, removing %s from jobs", key.get());
                         CJobSlave *job = jobs.find(key.get());
+                        StringAttr wuid = job->queryWuid();
+                        StringAttr graphName = job->queryGraphName();
+
+                        PROGLOG("QueryDone, removing %s from jobs", key.get());
                         jobs.removeExact(job);
                         PROGLOG("QueryDone, removed %s from jobs", key.get());
 
+                        PROGLOG("Finished wuid=%s, graph=%s", wuid.get(), graphName.get());
                         msg.clear();
                         msg.append(false);
                         break;

+ 18 - 8
thorlcr/slave/thslavemain.cpp

@@ -67,6 +67,7 @@ USE_JLIB_ALLOC_HOOK;
 #endif
 
 static SocketEndpoint slfEp;
+static unsigned mySlaveNum;
 
 static char **cmdArgs;
 void mergeCmdParams(IPropertyTree *props)
@@ -113,11 +114,18 @@ static bool RegisterSelf(SocketEndpoint &masterEp)
         setClusterGroup(group);
 
         SocketEndpoint myEp = queryMyNode()->endpoint();
-        if (RANK_NULL == group->rank(queryMyNode()))
+        rank_t groupPos = group->rank(queryMyNode());
+        if (RANK_NULL == groupPos)
         {
             replyError("Node not part of thorgroup");
             return false;
         }
+        if (globals->hasProp("@SLAVENUM") && (mySlaveNum != (unsigned)groupPos))
+        {
+            VStringBuffer errStr("Slave group rank[%d] does not match provided cmd line slaveNum[%d]", mySlaveNum, (unsigned)groupPos);
+            replyError(errStr.str());
+            return false;
+        }
         globals->Release();
         globals = createPTree(msg);
         mergeCmdParams(globals); // cmd line 
@@ -207,13 +215,10 @@ public:
 
 void startSlaveLog()
 {
-    StringBuffer fileName("thorslave.");
-    SocketEndpoint ep;
-    ep.setLocalHost(0);
-    ep.getUrlStr(fileName);
-    fileName.append("_").append(getMachinePortBase());
-
+    StringBuffer fileName("thorslave");
     Owned<IComponentLogFileCreator> lf = createComponentLogFileCreator(globals->queryProp("@logDir"), "thor");
+    StringBuffer slaveNumStr;
+    lf->setPostfix(slaveNumStr.append(mySlaveNum).str());
     lf->setCreateAliasFile(false);
     lf->setName(fileName.str());//override default filename
     lf->setMsgFields(MSGFIELD_timeDate | MSGFIELD_msgID | MSGFIELD_process | MSGFIELD_thread | MSGFIELD_code);
@@ -280,7 +285,12 @@ int main( int argc, char *argv[]  )
             localHostToNIC(slfEp);
         }
         else 
-            slfEp.setLocalHost(0); 
+            slfEp.setLocalHost(0);
+
+        if (globals->hasProp("@SLAVENUM"))
+            mySlaveNum = atoi(globals->queryProp("@SLAVENUM"));
+        else
+            mySlaveNum = slfEp.port; // shouldn't happen, provided by script
 
         setMachinePortBase(slfEp.port);
         slfEp.port = getMachinePortBase();

+ 1 - 1
thorlcr/thorutil/thmem.cpp

@@ -573,7 +573,7 @@ void CThorExpandingRowArray::transferFrom(CThorExpandingRowArray &donor)
 
 void CThorExpandingRowArray::transferFrom(CThorSpillableRowArray &donor)
 {
-	transferFrom((CThorExpandingRowArray &)donor);
+    transferFrom((CThorExpandingRowArray &)donor);
 }
 
 void CThorExpandingRowArray::removeRows(rowidx_t start, rowidx_t n)