HPCC-26127 Add ElasticStack support to Start/stopall scripts

- Adds support for deploying ES via starall.sh
- Adds support for uninstalling ES via stopall.sh

Signed-off-by: Rodrigo Pastrana <Rodrigo.Pastrana@lexisnexisrisk.com>

dockerfiles/README.md

@@ -39,8 +39,8 @@ Bash scripts
 buildall.sh - Used to create and publish a docker container corresponding to a github tag
 clean.sh    - Clean up old docker images (if disk gets full)
 incr.sh     - Build local images for testing (delta from a published image)
-startall.sh - Start a local k8s cluster
-stopall.sh  - Stop a local k8s cluster
+startall.sh - Start a local k8s cluster, and optional Elastic Stack for log processing purposes
+stopall.sh  - Stop a local k8s cluster, and optional Elastic Stack
 
 ---
 

dockerfiles/startall.sh

@@ -26,6 +26,31 @@ restArgs=()
 CLUSTERNAME=mycluster
 PVFILE=$scriptdir/../helm/examples/local/hpcc-localfile/values.yaml
 
+dependency_check () {
+
+  if [ -z "$1" ]
+  then
+      CHART_SUBPATH="hpcc"
+  else
+      CHART_SUBPATH=$1
+  fi
+
+  missingDeps=0
+  while IFS= read -r line
+  do
+    echo "${line}"
+    if echo "${line}" | egrep -q 'missing$'; then
+      let "missingDeps++"
+    fi
+  done < <(helm dependency list ${scriptdir}/../helm/${CHART_SUBPATH} | grep -v WARNING)
+  if [[ ${missingDeps} -gt 0 ]]; then
+    echo "Some of the chart dependencies are missing."
+    echo "Either issue a 'helm dependency update ${scriptdir}/../helm/${CHART_SUBPATH}' to fetch them,"
+    echo "or rerun $0 with option -c to auto update them."
+    exit 0
+  fi
+}
+
 CMD="install"
 DEVELOPER_OPTIONS="--set global.privileged=true"
 while [ "$#" -gt 0 ]; do
@@ -62,6 +87,7 @@ while [ "$#" -gt 0 ]; do
          echo "    -c                 Update chart dependencies"
          echo "    -p <location>      Use local persistent data"
          echo "    -pv <yamlfile>     Override dataplane definitions for local persistent data"
+         echo "    -e                 Deploy light-weight Elastic Stack for component log processing"
          exit
          ;;
       t) CMD="template"
@@ -70,6 +96,8 @@ while [ "$#" -gt 0 ]; do
       # vanilla install - for testing system in the same way it will normally be used
       v) DEVELOPER_OPTIONS=""
          ;;
+      e) DEPLOY_ES=true
+         ;;
       *) restArgs+=(${arg})
          ;;
     esac
@@ -79,26 +107,14 @@ while [ "$#" -gt 0 ]; do
   shift
 done
 
+
 if [[ -n "${DEP_UPDATE_ARG}" ]]; then
   if [[ "${CMD}" = "upgrade" ]]; then
     echo "Chart dependencies cannot be updated whilst performing a helm upgrade"
     DEP_UPDATE_ARG=""
   fi
 else
-  missingDeps=0
-  while IFS= read -r line
-  do
-    echo "${line}"
-    if echo "${line}" | egrep -q 'missing$'; then
-      let "missingDeps++"
-    fi
-  done < <(helm dependency list ${scriptdir}/../helm/hpcc | grep -v WARNING)
-  if [[ ${missingDeps} -gt 0 ]]; then
-    echo "Some of the chart dependencies are missing."
-    echo "Either issue a 'helm dependency update ${scriptdir}/../helm/hpcc' to fetch them,"
-    echo "or rerun $0 with option -c to auto update them."
-    exit 0
-  fi
+  dependency_check "hpcc"
 fi
 
 [[ -n ${INPUT_DOCKER_REPO} ]] && DOCKER_REPO=${INPUT_DOCKER_REPO}
@@ -118,6 +134,14 @@ else
   helm ${CMD} $CLUSTERNAME $scriptdir/../helm/hpcc/ --set global.image.root="${DOCKER_REPO}" --set global.image.version=$LABEL $DEVELOPER_OPTIONS $DEP_UPDATE_ARG ${restArgs[@]}
 fi
 
+if [[ $DEPLOY_ES ]] ; then
+  echo -e "\n\nDeploying "myelastic4hpcclogs" - light-weight Elastic Stack:"
+  if [[ -z "${DEP_UPDATE_ARG}" ]]; then
+    dependency_check "managed/logging/elastic"
+  fi
+  helm ${CMD} myelastic4hpcclogs $scriptdir/../helm/managed/logging/elastic $DEP_UPDATE_ARG ${restArgs[@]}
+fi
+
 if [ ${CMD} != "template" ] ; then
   sleep 1
   kubectl get pods

dockerfiles/stopall.sh

@@ -29,8 +29,11 @@ while [ "$#" -gt 0 ]; do
       -n) shift
          CLUSTERNAME=$1
          ;;
+      -e) UNINSTALL_ELK=1
+         ;;
       *) echo "Usage: stoptall.sh [options]"
          echo "    -w  Wait for all pods to terminate"
+         echo "    -e  Uninstall light-weight Elastic Stack"
          exit
          ;;
     esac
@@ -39,8 +42,14 @@ done
 
 helm uninstall $CLUSTERNAME
 helm uninstall localfile
-kubectl delete jobs --all 
-kubectl delete networkpolicy --all 
+kubectl delete jobs --all
+kubectl delete networkpolicy --all
+if [[ $UNINSTALL_ELK == 1 ]] ; then
+  echo "Uninstalling myelastic4hpcclogs:"
+  echo "PLEASE NOTE: Elastic Search declares PVC(s) which might require explicit manual removal if no longer needed."
+  helm uninstall myelastic4hpcclogs
+  kubectl get pvc
+fi
 if [[ $wait == 1 ]] ; then
   sleep 2
   while (kubectl get pods | grep -q ^NAME) ; do