Big-Data-HPC-Platform/plugins/spark/sparkthor.sh.in

#!/bin/bash
################################################################################
#    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems®.
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#       http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.
################################################################################


_command=${1:-start}
_component=${2:-mysparkthor}

#### Uncomment following lines for verbose debugging
#exec 3>&2 2>/@LOG_PATH@/${_component}/sparkthor.sh.debug
#set -x

export logfile="@LOG_PATH@/${_component}/sparkthor.log"
source @INSTALL_DIR@/sbin/hpcc_setenv
source @INSTALL_DIR@/etc/init.d/hpcc_common
source ./spark-hpcc-env.sh

# update slaves file in case state of environment has been altered since last run
errorMessage=$( @EXEC_PATH@/daliadmin server=$DALISERVER clusternodes ${NODEGROUP} @RUNTIME_PATH@/${_component}/slaves 2>&1 )
errcode=$?
if [[ 0 != ${errcode} ]]; then
    log "failed to lookup dali group for ${component}"
    log "$errorMessage"
    exit 1
fi

log "${_command} master"
@INSTALL_DIR@/externals/spark-hadoop/sbin/${_command}-master.sh

if [[ -f "@RUNTIME_PATH@/${_component}/slaves" ]]; then
    log "Starting sparkthor-workers with ./frunssh @RUNTIME_PATH@/${_component}/slaves"
    clusternodes=$(cat @RUNTIME_PATH@/${_component}/slaves | wc -l)
    @EXEC_PATH@/frunssh @RUNTIME_PATH@/${_component}/slaves "/bin/sh -c '@ADMIN_PATH@/sparkthor-worker.sh ${_command} ${_component}'" -i:$SSHidentityfile -u:$SSHusername -pe:$SSHpassword -t:$SSHtimeout -a:$SSHretries -n:$clusternodes 2>&1
    FRUNSSH_RC=$?
    if [[ ${FRUNSSH_RC} -gt 0 ]]; then
        log "Error ${FRUNSSH_RC} in frunssh"
        log "Please check @LOG_PATH@/frunssh for more details"
        exit 255
    else
        log "Frunssh successful"
    fi
else
    @ADMIN_PATH@/sparkthor-worker.sh ${_command} ${_component}
fi

exit 0