Merge branch 'candidate-5.0.0' into closedown-5.0.x

Conflicts:
	version.cmake

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

esp/services/ws_smc/ws_smcService.cpp

@@ -456,73 +456,64 @@ void CWsSMCEx::sortTargetClusters(IArrayOf<IEspTargetCluster>& clusters, const c
         clusters.sort(descending ? sortTargetClustersBySizeDescending : sortTargetClustersBySizeAscending);
 }
 
-void CWsSMCEx::setClusterQueueStatus(CWsSMCTargetCluster& targetCluster)
+void CWsSMCEx::getClusterQueueStatus(const CWsSMCTargetCluster& targetCluster, ClusterStatusType& queueStatusType, StringBuffer& queueStatusDetails)
 {
-    CWsSMCQueue& jobQueue = targetCluster.clusterQueue;
+    const CWsSMCQueue* jobQueue = &targetCluster.clusterQueue;
     if (targetCluster.clusterType != ThorLCRCluster)
-        jobQueue = targetCluster.agentQueue;
-    if (!jobQueue.queueName.length())
+        jobQueue = &targetCluster.agentQueue;
+    if (!jobQueue->queueName.length())
         return;
 
-    targetCluster.clusterStatusDetails.appendf("%s: ", jobQueue.queueName.str());
-
     bool queuePausedOrStopped = false;
-    unsigned countRunningJobs = jobQueue.countRunningJobs;
-    unsigned countQueuedJobs = jobQueue.countQueuedJobs;
-    if (targetCluster.clusterType == ThorLCRCluster)
+    //get queueStatusDetails
+    if (targetCluster.clusterStatusDetails.length())
+        queueStatusDetails.set(targetCluster.clusterStatusDetails.str());
+    if (jobQueue->queueState.length())
     {
-        countRunningJobs += targetCluster.agentQueue.countRunningJobs;
-        countQueuedJobs += targetCluster.agentQueue.countQueuedJobs;
-    }
+        const char* queueState = jobQueue->queueState.str();
+        queueStatusDetails.appendf("%s: queue %s; ", jobQueue->queueName.str(), queueState);
+        if (jobQueue->queueStateDetails.length())
+            queueStatusDetails.appendf(" %s;", jobQueue->queueStateDetails.str());
 
-    if (jobQueue.queueState.length())
-    {
-        const char* queueState = jobQueue.queueState.str();
-        const char* queueStateDetails = jobQueue.queueStateDetails.str();
-        if (queueStateDetails && *queueStateDetails)
-            targetCluster.clusterStatusDetails.appendf("queue %s; %s;", queueState, queueStateDetails);
-        else
-            targetCluster.clusterStatusDetails.appendf("queue %s; ", queueState);
         if (strieq(queueState,"stopped") || strieq(queueState,"paused"))
             queuePausedOrStopped = true;
     }
 
-    if (!jobQueue.foundQueueInStatusServer)
+    //get queueStatusType
+    if (!jobQueue->foundQueueInStatusServer)
     {
         if (queuePausedOrStopped)
-            jobQueue.statusType = QueuePausedOrStoppedNotFound;
+            queueStatusType = QueuePausedOrStoppedNotFound;
         else
-            jobQueue.statusType = QueueRunningNotFound;
+            queueStatusType = QueueRunningNotFound;
     }
+    else if (!queuePausedOrStopped)
+        queueStatusType = RunningNormal;
+    else if (jobQueue->countRunningJobs > 0)
+        queueStatusType = QueuePausedOrStoppedWithJobs;
     else
-    {
-        if (queuePausedOrStopped)
-        {
-            if (jobQueue.countRunningJobs > 0)
-                jobQueue.statusType = QueuePausedOrStoppedWithJobs;
-            else
-                jobQueue.statusType = QueuePausedOrStoppedWithNoJob;
-        }
-    }
+        queueStatusType = QueuePausedOrStoppedWithNoJob;
+
+    return;
 }
 
-void CWsSMCEx::setClusterStatus(IEspContext& context, CWsSMCTargetCluster& targetCluster, IEspTargetCluster* returnCluster)
+void CWsSMCEx::setClusterStatus(IEspContext& context, const CWsSMCTargetCluster& targetCluster, IEspTargetCluster* returnCluster)
 {
-    setClusterQueueStatus(targetCluster);
+    ClusterStatusType queueStatusType = RunningNormal;
+    StringBuffer queueStatusDetails;
+    getClusterQueueStatus(targetCluster, queueStatusType, queueStatusDetails);
 
-    int statusType = (targetCluster.clusterQueue.statusType > targetCluster.agentQueue.statusType) ? targetCluster.clusterQueue.statusType
-        : targetCluster.agentQueue.statusType;
-    returnCluster->setClusterStatus(statusType);
+    returnCluster->setClusterStatus(queueStatusType);
     //Set 'Warning' which may be displayed beside cluster name
-    if (statusType == QueueRunningNotFound)
+    if (queueStatusType == QueueRunningNotFound)
         returnCluster->setWarning("Cluster not attached");
-    else if (statusType == QueuePausedOrStoppedNotFound)
+    else if (queueStatusType == QueuePausedOrStoppedNotFound)
         returnCluster->setWarning("Queue paused or stopped - Cluster not attached");
-    else if (statusType != RunningNormal)
+    else if (queueStatusType != RunningNormal)
         returnCluster->setWarning("Queue paused or stopped");
     //Set 'StatusDetails' which may be displayed when a mouse is moved over cluster icon
-    if (targetCluster.clusterStatusDetails.length())
-        returnCluster->setStatusDetails(targetCluster.clusterStatusDetails.str());
+    if (queueStatusDetails.length())
+        returnCluster->setStatusDetails(queueStatusDetails.str());
 }
 
 void CWsSMCEx::getWUsNotOnTargetCluster(IEspContext &context, IPropertyTree* serverStatusRoot, IArrayOf<IEspServerJobQueue>& serverJobQueues,
@@ -1162,7 +1153,7 @@ void CWsSMCEx::setActivityResponse(IEspContext &context, ActivityInfo* activityI
     return;
 }
 
-void CWsSMCEx::setESPTargetClusters(IEspContext& context, CIArrayOf<CWsSMCTargetCluster>& targetClusters, IArrayOf<IEspTargetCluster>& respTargetClusters)
+void CWsSMCEx::setESPTargetClusters(IEspContext& context, const CIArrayOf<CWsSMCTargetCluster>& targetClusters, IArrayOf<IEspTargetCluster>& respTargetClusters)
 {
     ForEachItemIn(i, targetClusters)
     {

esp/services/ws_smc/ws_smcService.hpp

@@ -171,8 +171,8 @@ private:
     void readWUsAndStateFromJobQueue(IEspContext& context, CWsSMCTargetCluster& targetCluster, CWsSMCQueue& queue, const char* listQueue, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
     void addToTargetClusterList(IArrayOf<IEspTargetCluster>& clusters, IEspTargetCluster* cluster, const char* sortBy, bool descending);
     bool findQueueInStatusServer(IEspContext& context, IPropertyTree* serverStatusRoot, const char* serverName, const char* queueName);
-    void setClusterQueueStatus(CWsSMCTargetCluster& targetCluster);
-    void setClusterStatus(IEspContext& context, CWsSMCTargetCluster& targetCluster, IEspTargetCluster* returnCluster);
+    void getClusterQueueStatus(const CWsSMCTargetCluster& targetCluster, ClusterStatusType& queueStatusType, StringBuffer& queueStatusDetails);
+    void setClusterStatus(IEspContext& context, const CWsSMCTargetCluster& targetCluster, IEspTargetCluster* returnCluster);
     void getTargetClusterAndWUs(IEspContext& context, CConstWUClusterInfoArray& clusters, IConstWUClusterInfo& cluster,
          IPropertyTree* serverStatusRoot, IPropertyTreeIterator* itStatusECLagent, IEspTargetCluster* returnCluster, IArrayOf<IEspActiveWorkunit>& aws);
     void getWUsNotOnTargetCluster(IEspContext &context, IPropertyTree* serverStatusRoot, IArrayOf<IEspServerJobQueue>& serverJobQueues, IArrayOf<IEspActiveWorkunit>& aws);
@@ -197,7 +197,7 @@ private:
             BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
     void readWUsAndStateFromJobQueue(IEspContext& context, CWsSMCTargetCluster& targetCluster, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
     void readWUsAndStateFromJobQueue(IEspContext& context, CIArrayOf<CWsSMCTargetCluster>& targetClusters, BoolHash& uniqueWUIDs, IArrayOf<IEspActiveWorkunit>& aws);
-    void setESPTargetClusters(IEspContext& context, CIArrayOf<CWsSMCTargetCluster>& targetClusters, IArrayOf<IEspTargetCluster>& respTargetClusters);
+    void setESPTargetClusters(IEspContext& context, const CIArrayOf<CWsSMCTargetCluster>& targetClusters, IArrayOf<IEspTargetCluster>& respTargetClusters);
     ActivityInfo* createActivityInfo(IEspContext &context);
     void clearActivityInfoCache();
     ActivityInfo* getActivityInfo(IEspContext &context);

initfiles/bash/etc/init.d/hpcc-init.in

@@ -113,6 +113,8 @@ set_environmentvars
 envfile=$configs/$environment
 
 # Know HPCC user after set_environmentvars
+log_dir_owner=$(ls -ld $LOG_DIR | awk '{print $3}')
+[ "log_dir_owner" != "${user}" ] && chown ${user}:${user} $LOG_DIR
 chown ${user}:${user} $HPCC_INIT_LOG
 
 #Sourcing the hpcc environment

initfiles/bash/etc/init.d/hpcc_common.in

@@ -960,3 +960,24 @@ run_cluster() {
    echo ""
 }
 
+##
+## Usage: cluster_tools_init 
+##    Initialization for cluster tools
+##
+cluster_tools_init() {
+
+   set_environmentvars
+
+   _cmd_prefix=
+   [ "$(id -u)" != "0" ] && _cmd_prefix=sudo
+
+   # Check and set log directory
+   CLUSTER_LOG_DIR=${LOG_DIR}/cluster
+   [ ! -d $LOG_DIR ]  && ${_cmd_prefix} /etc/init.d/hpcc-init status > /dev/null 2>&1
+
+   [ ! -d $CLUSTER_LOG_DIR ] && mkdir -p $CLUSTER_LOG_DIR
+   # workaround inconsistency of stat command
+   cluster_log_dir_owner=$(ls -ld $CLUSTER_LOG_DIR | awk '{print $3}')
+   [ "$cluster_log_dir_owner" != "${user}" ] && ${_cmd_prefix} chown ${user}:${user} $CLUSTER_LOG_DIR
+   
+}

initfiles/sbin/deploy-java-files.sh.in

@@ -380,6 +380,8 @@ setUser() {
 # MAIN
 #
 ######################################################################
+cluster_tools_init
+
 source=
 target=
 update_classpath=0
@@ -459,7 +461,6 @@ fi
 
 [ -n "$source" ] && setUser
 
-set_environmentvars
 HPCC_CONFIG=${CONFIG_DIR}/${ENV_CONF_FILE}
 
 

initfiles/sbin/hpcc-push.sh.in

@@ -76,14 +76,15 @@ chmod +x ${SCRIPT_FILE}
 # MAIN
 #
 ############################################
-if [ "${USER}" != "root" ]; then
+cluster_tools_init
+
+if [ "${USER}" != "root" ] && [ "${USER}" != "${user}" ]; then
    echo ""
-   echo "The script must run as root or sudo."
+   echo "The script must run as root, $user or sudo."
    echo ""
    exit 1
 fi
 
-set_environmentvars
 
 source=
 target=

initfiles/sbin/hpcc-run.sh.in

@@ -244,15 +244,15 @@ end() {
 # MAIN
 #
 ############################################
-if [ "${USER}" != "root" ]; then
+cluster_tools_init
+
+if [ "${USER}" != "root" ] && [ "${USER}" != "${user}" ]; then
    echo ""
-   echo "The script must run as root or sudo."
+   echo "The script must run as root, $user or sudo."
    echo ""
    exit 1
 fi
 
-
-set_environmentvars
 envfile=$configs/$environment
 configfile=${CONFIG_DIR}/${ENV_CONF_FILE}
 

initfiles/sbin/install-cluster.sh.in

@@ -122,6 +122,14 @@ removePayload(){
     rm /tmp/remote_install.tgz
 }
 
+
+######################################################################
+#
+# MAIN
+#
+######################################################################
+cluster_tools_init
+
 if [ "${USER}" != "root" ]; then
    echo ""
    echo "The script must run as root or sudo."