HPCC-23515 Create Docker and Helm charts for Thor

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>

dockerfiles/README.md

@@ -70,4 +70,16 @@ When running under K8s, Roxie has 3 fundamental modes of operation:
      There will be numChannels*replicas slave pods and serverReplicas server pods in total
   
      This mode is somewhat experimental at present!
-  
+  
+Thor under K8s on WSL
+---------------------
+
+To expose host storage to containers running under WSL, it is necessary to do the following:
+1) Within WSL, bind an arbitary directory (e.g. "k8smountpoint") to a directory under /mnt/wsl:
+
+sudo mount --bind k8smountpoint /mnt/wsl/hostmount
+
+2) When starting helm, set global.hostMountPath to point to "/run/desktop/mnt/host/wsl/hostmount", e.g.:
+
+helm install mycluster hpcc/ --set global.image.version=<image-name>,global.hostMountPath=/run/desktop/mnt/host/wsl/hostmount
+

dockerfiles/buildall.sh

@@ -84,6 +84,8 @@ build_image esp
 build_image eclccserver
 build_image eclagent
 build_image toposerver
+build_image thormaster
+build_image thorslave
 
 if [[ -n ${INPUT_PASSWORD} ]] ; then
   echo "::set-output name=${BUILD_LABEL}"

dockerfiles/hpcc/templates/_util.tpl

@@ -90,3 +90,23 @@ volumeMounts:
 image: "{{ include "hpcc.utils.imageName" . }}"
 imagePullPolicy: {{ .root.Values.global.image.pullPolicy }}
 {{- end -}}
+
+{{- /* A kludge to ensure host mounted storage (e.g. for minikube or docker for desktop) has correct permissions for PV */ -}}
+{{- define "hpcc.utils.changeMountPerms" -}}
+initContainers:
+# This is a bit of a hack, to ensure that the persistent storage mounted
+# is writable. This is not something we would want to do if using anything other than
+# local storage (which is only sensible on single-node systems).
+# NB: uid=999 and gid=1000 are the uid/gid of the hpcc user, built into platform-core
+{{- $permCmd := printf "chown -R 999:1000 %s" .volumePath }}
+- name: volume-mount-hack
+  image: busybox
+  command: [
+             "sh",
+             "-c",
+             "{{ $permCmd }}"
+           ]
+  volumeMounts:
+    - name: {{ .volumeName | quote}}
+      mountPath: {{ .volumePath | quote }}
+{{- end }}

dockerfiles/hpcc/templates/dllserver.yaml

@@ -11,5 +11,5 @@ spec:
   accessModes:
     - ReadWriteMany
   hostPath:
-    path: "/var/lib/HPCCSystems/shared/queries"
+    path: {{ printf "%s/queries" $.Values.global.hostMountPath }}
     type: DirectoryOrCreate

dockerfiles/hpcc/templates/eclccserver.yaml

@@ -13,17 +13,8 @@ spec:
       labels:
         run: {{ .name | quote }}
     spec:
-      {{ if $.Values.global.singleNode | default false -}}
-      initContainers:
-      # This is a bit of a hack, to ensure that the persistent storage mounted for dllserver
-      # is writable. This is not something we would want to do if using anything other than
-      # local storage (which is only sensible on single-node systems).
-      - name: volume-mount-hack
-        image: busybox
-        command: ["sh", "-c", "chown -R 999:1000 /var/lib/HPCCSystems/queries"]
-        volumeMounts:
-        - mountPath: "/var/lib/HPCCSystems/queries"
-          name: dllserver-pv-storage
+      {{- if $.Values.global.singleNode | default false }}
+{{ include "hpcc.utils.changeMountPerms" (dict "volumeName" "dllserver-pv-storage" "volumePath" "/var/lib/HPCCSystems/queries") | indent 6 }}
       {{- end }}
       serviceAccountName: hpcc
       containers:

dockerfiles/hpcc/templates/thor.yaml

@@ -0,0 +1,126 @@
+{{ range $thor := $.Values.thor -}}
+{{- $masterPort := $thor.masterport | default 20000 -}}
+{{- $slavePort := $thor.slaveport | default 20100 -}}
+{{- $pvName  := printf "%s-pv" .name }}
+{{- $pvcName := printf "%s-pvc" .name }}
+{{- $slaveName := printf "%s-slave" .name }}
+{{- $serviceName := printf "%s-svc" .name }}
+
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ .name | quote }}
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      run: {{ .name | quote }}
+  template:
+    metadata:
+      labels:
+        run: {{ .name | quote }}
+    spec:
+      containers:
+      - name: {{ .name | quote }}
+        ports:
+          - containerPort: {{ $masterPort }}
+        args: [
+                {{ include "hpcc.utils.configArg" . }},
+                {{ include "hpcc.utils.daliArg" $ }},
+                --masterport={{ $masterPort }},
+                --numSlaves={{ $thor.numSlaves }}
+              ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "thormaster") | indent 8 }}
+{{ include "hpcc.utils.addVolumeMounts" . | indent 8 }}
+{{ include "hpcc.utils.addVolumes" . | indent 6 }}
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ $slaveName | quote }}
+spec:
+  replicas: {{ $thor.numSlaves }}
+  selector:
+    matchLabels:
+      run: {{ $slaveName | quote }}
+  template:
+    metadata:
+      labels:
+        run: {{ $slaveName | quote }}
+    spec:
+      {{- if $.Values.global.singleNode | default false }}
+{{ include "hpcc.utils.changeMountPerms" (dict "volumeName" "mythorstorage" "volumePath" "/var/lib/HPCCSystems/hpcc-data") | indent 6 }}
+      {{- end }}
+      containers:
+        - name: {{ $slaveName | quote }}
+          args: [
+                  {{ include "hpcc.utils.configArg" . }},
+                  {{ include "hpcc.utils.daliArg" $ }},
+                  --slaveport={{ $slavePort }},
+                  --master={{ printf "%s:%v" $serviceName $masterPort }}
+                ]
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "thorslave") | indent 10 }}
+          ports:
+            - containerPort: {{ $slavePort }}
+{{ include "hpcc.utils.addVolumeMounts" . | indent 10 }}
+          - name: "mythorstorage"
+            mountPath: "/var/lib/HPCCSystems/hpcc-data"
+        {{- if $thor.startDafilesrv | default false }}
+        - name: {{ printf "%s-dafilesrv" $slaveName | quote }}
+{{ include "hpcc.utils.addImageAttrs" (dict "root" $ "imagename" "thorslave") | indent 10 }}
+          command: ["dafilesrv"]
+          ports:
+            - containerPort: 7100
+          volumeMounts:
+            - name: "mythorstorage"
+              mountPath: "/var/lib/HPCCSystems/hpcc-data"
+        {{- end }}
+{{ include "hpcc.utils.addVolumes" . | indent 6 }}
+      - name: "mythorstorage"
+        persistentVolumeClaim:
+          claimName: {{ $pvcName | quote }}
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ $serviceName | quote }}
+spec:
+  ports:
+  - port: {{ $masterPort }}
+    protocol: TCP
+    targetPort: {{ $masterPort }}
+  selector:
+    run: {{ .name | quote }}
+  type: ClusterIP
+---
+apiVersion: v1
+kind: PersistentVolume
+metadata:
+  name: {{ $pvName | quote }}
+  labels:
+    type: local
+spec:
+  storageClassName: manual
+  capacity:
+    storage: {{ $thor.storageSize }}
+  accessModes:
+    - ReadWriteMany
+  persistentVolumeReclaimPolicy: Retain
+  hostPath:
+    path: {{ printf "%s/hpcc-data" $.Values.global.hostMountPath | quote }}
+    type: DirectoryOrCreate
+---
+apiVersion: v1
+kind: PersistentVolumeClaim
+metadata:
+  name: {{ $pvcName | quote }}
+spec:
+  storageClassName: manual
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: {{ $thor.storageSize }}
+---
+{{- include "hpcc.utils.generateConfigMap" (dict "root" $ "me" .) -}}
+{{- end }}

dockerfiles/hpcc/values.yaml

@@ -18,6 +18,10 @@ global:
   
   singleNode: true
 
+  # For singleMode only. Host mount point root for persistent storage.
+  hostMountPath: "/docker-host-mnt"
+
+
 dali:
 - name: mydali
 
@@ -50,5 +54,10 @@ roxie:
       Roxie:
         useAeron: true
 
-
-    
+thor:
+  - name: thor1
+    masterport: 20000
+    numSlaves: 2
+    slaveport: 20100
+    startDafilesrv: true # Temporary until non attached storage functionality added
+    storageSize: 100G

dockerfiles/incr.sh

@@ -58,6 +58,8 @@ docker image build -t hpccsystems/esp:${BUILD_LABEL} --build-arg BUILD_LABEL=${B
 docker image build -t hpccsystems/eclccserver:${BUILD_LABEL} --build-arg BUILD_LABEL=${BUILD_LABEL} eclccserver/  
 docker image build -t hpccsystems/eclagent:${BUILD_LABEL} --build-arg BUILD_LABEL=${BUILD_LABEL} eclagent/  
 docker image build -t hpccsystems/toposerver:${BUILD_LABEL} --build-arg BUILD_LABEL=${BUILD_LABEL} toposerver/  
+docker image build -t hpccsystems/thormaster:${BUILD_LABEL} --build-arg BUILD_LABEL=${BUILD_LABEL} thormaster/
+docker image build -t hpccsystems/thorslave:${BUILD_LABEL} --build-arg BUILD_LABEL=${BUILD_LABEL} thorslave/
 
 echo Built hpccsystems/*:${BUILD_LABEL}
 

dockerfiles/platform-core-debug/Dockerfile

@@ -24,7 +24,8 @@ RUN apt-get install -y \
     dnsutils \
     nano 
 
-RUN useradd -s /bin/bash -r -U -c "hpcc runtime User" hpcc
+RUN groupadd -g 1000 hpcc
+RUN useradd -s /bin/bash -r -N -c "hpcc runtime User" -u 999 -g hpcc hpcc
 RUN passwd -l hpcc 
 
 RUN mkdir /var/lib/HPCCSystems && chown hpcc:hpcc /var/lib/HPCCSystems

dockerfiles/platform-core/Dockerfile

@@ -57,7 +57,8 @@ RUN apt-get install -y \
 
 COPY --from=build /opt/HPCCSystems /
 
-RUN useradd -s /bin/bash -r -U -c "hpcc runtime User" hpcc
+RUN groupadd -g 1000 hpcc
+RUN useradd -s /bin/bash -r -N -c "hpcc runtime User" -u 999 -g hpcc hpcc
 RUN passwd -l hpcc 
 
 RUN mkdir /var/lib/HPCCSystems && chown hpcc:hpcc /var/lib/HPCCSystems

dockerfiles/thormaster/Dockerfile

@@ -0,0 +1,29 @@
+##############################################################################
+#
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+##############################################################################
+
+# Build container image for a single thorslave
+
+ARG BUILD_LABEL
+FROM hpccsystems/platform-core:${BUILD_LABEL}
+
+USER hpcc
+RUN mkdir -p /var/lib/HPCCSystems/thor
+WORKDIR /var/lib/HPCCSystems/thor
+COPY --chown=hpcc:hpcc thor.xml /var/lib/HPCCSystems/thor
+#RUN thormaster_lcr --init
+ENTRYPOINT ["thormaster_lcr"]
+

dockerfiles/thormaster/thor.xml

@@ -0,0 +1,3 @@
+<Thor name="mythor" daliServers="dali" numSlaves="1" pluginsPath="/opt/HPCCSystems/plugins/" replicateAsync="false"
+      watchdogEnabled="true" watchdogProgressEnabled="true" globalMemorySize="4096">
+</Thor>

dockerfiles/thorslave/Dockerfile

@@ -0,0 +1,27 @@
+##############################################################################
+#
+#    HPCC SYSTEMS software Copyright (C) 2020 HPCC Systems®.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+##############################################################################
+
+# Build container image for a single thormaster
+
+ARG BUILD_LABEL
+FROM hpccsystems/platform-core:${BUILD_LABEL}
+
+USER hpcc
+RUN mkdir -p /var/lib/HPCCSystems/thor
+WORKDIR /var/lib/HPCCSystems/thor
+ENTRYPOINT ["thorslave_lcr"]
+

initfiles/bin/init_thorslave.in

@@ -128,8 +128,8 @@ start_slaves()
             for (( slave=0; slave<${slavespernode}; slave++ )); do
                 slavenum=$(( ${clusternode} + (${slave} * ${clusternodes}) ))
                 if [ -z ${valgrindOptions} ]; then
-                    log "$slavename master=$master:$masterport slave=.:$slaveport slavenum=$slavenum logDir=$logpth/$hpcc_compname"
-                    ./$slavename master=$master:$masterport slave=.:$slaveport slavenum=$slavenum slaveprocessnum=$slave logDir=$logpth/$hpcc_compname 2>/dev/null 1>/dev/null &
+                    log "$slavename --master=$master:$masterport --slave=.:$slaveport --slavenum=$slavenum --logDir=$logpth/$hpcc_compname"
+                    ./$slavename --master=$master:$masterport --slave=.:$slaveport --slavenum=$slavenum --slaveprocessnum=$slave --logDir=$logpth/$hpcc_compname 2>/dev/null 1>/dev/null &
                 else
                     cmd="valgrind ${valgrindOptions} --log-file=$logpth/$hpcc_compname/valgrind.thorslave.${slavenum}.log ./$slavename master=$master:$masterport slave=.:$slaveport slavenum=$slavenum slaveprocessnum=$slave logDir=$logpth/$hpcc_compname 2>/dev/null 1>/dev/null &"
                     log "${cmd}"

system/jlib/jptree.cpp

@@ -7895,6 +7895,17 @@ IPropertyTree & queryGlobalConfig()
     return *globalConfiguration;
 }
 
+jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv)
+{
+    for (const char * * pArg = argv; *pArg; pArg++)
+    {
+        const char * cur = *pArg;
+        if (startsWith(cur, "--"))
+            applyCommandLineOption(config, cur + 2);
+    }
+    return config;
+}
+
 jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *))
 {
     if (componentConfiguration)
@@ -7988,12 +7999,7 @@ jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char
         applyEnvironmentConfig(*config, envPrefix, *cur);
     }
 
-    for (const char * * pArg = argv; *pArg; pArg++)
-    {
-        const char * cur = *pArg;
-        if (startsWith(cur, "--"))
-            applyCommandLineOption(config, cur + 2);
-    }
+    loadArgsIntoConfiguration(config, argv);
 
     if (outputConfig)
     {

system/jlib/jptree.hpp

@@ -288,6 +288,7 @@ inline static bool isValidXPathChr(char c)
     return ('\0' != c && (isalnum(c) || strchr(validChrs, c)));
 }
 
+jlib_decl IPropertyTree * loadArgsIntoConfiguration(IPropertyTree *config, const char * * argv);
 jlib_decl IPropertyTree * loadConfiguration(const char * defaultYaml, const char * * argv, const char * componentTag, const char * envPrefix, const char * legacyFilename, IPropertyTree * (mapper)(IPropertyTree *));
 jlib_decl StringBuffer & regenerateConfig(StringBuffer &jsonText, IPropertyTree * config, const char * componentTag);
 

thorlcr/master/thmastermain.cpp

@@ -607,15 +607,17 @@ int main( int argc, const char *argv[]  )
     else
         thorEp.setLocalHost(0);
 
-    setMasterPortBase(thorEp.port); // both same
-    thorEp.port = getMasterPortBase();
+    if (0 == thorEp.port)
+        thorEp.port = globals->getPropInt("@masterport", THOR_BASE_PORT);
+
+     // both same
+    setMasterPortBase(thorEp.port);
+    setMachinePortBase(thorEp.port);
 
     // Remove sentinel asap
     Owned<IFile> sentinelFile = createSentinelTarget();
     removeSentinelFile(sentinelFile);
 
-    setMachinePortBase(thorEp.port);
-
     EnableSEHtoExceptionMapping(); 
 #ifndef __64BIT__
     // Restrict stack sizes on 32-bit systems
@@ -775,7 +777,16 @@ int main( int argc, const char *argv[]  )
         if (getConfigurationDirectory(globals->queryPropTree("Directories"),"temp","thor",globals->queryProp("@name"), tempDirStr))
             globals->setProp("@thorTempDirectory", tempDirStr.str());
         else
+        {
             tempDirStr.append(globals->queryProp("@thorTempDirectory"));
+            if (0 == tempDirStr.length())
+            {
+                appendCurrentDirectory(tempDirStr, true);
+                if (tempDirStr.length())
+                    addPathSepChar(tempDirStr);
+                tempDirStr.append("temp");
+            }
+        }
         logDiskSpace(); // Log before temp space is cleared
         StringBuffer tempPrefix("thtmp");
         tempPrefix.append(getMasterPortBase()).append("_");

thorlcr/slave/thslavemain.cpp

@@ -74,12 +74,7 @@ static unsigned mySlaveNum;
 static const unsigned defaultStrandBlockSize = 512;
 static const unsigned defaultForceNumStrands = 0;
 
-static char **cmdArgs;
-void mergeCmdParams(IPropertyTree *props)
-{
-    while (*cmdArgs)
-        loadCmdProp(props, *cmdArgs++);
-}
+static const char **cmdArgs;
 
 static void replyError(unsigned errorCode, const char *errorMsg)
 {
@@ -123,7 +118,11 @@ static bool RegisterSelf(SocketEndpoint &masterEp)
             assertex(mySlaveNum == configSlaveNum);
 
         globals.setown(createPTree(msg));
-        mergeCmdParams(globals); // cmd line
+
+        /* NB: preserve command line option overrides
+         * Not sure if any cmdline options are actually needed by this stage..
+         */
+        loadArgsIntoConfiguration(globals, cmdArgs);
 
         unsigned channelsPerSlave = globals->getPropInt("@channelsPerSlave", 1);
         unsigned localThorPortInc = globals->getPropInt("@localThorPortInc", DEFAULT_SLAVEPORTINC);
@@ -136,6 +135,7 @@ static bool RegisterSelf(SocketEndpoint &masterEp)
             return false;
         }
 
+        ensurePTree(globals, "Debug");
         unsigned numStrands, blockSize;
         if (globals->hasProp("Debug/@forceNumStrands"))
             numStrands = globals->getPropInt("Debug/@forceNumStrands");
@@ -304,7 +304,7 @@ void setSlaveAffinity(unsigned processOnNode)
 }
 
 
-int main( int argc, char *argv[]  )
+int main( int argc, const char *argv[]  )
 {
     // If using systemd, we will be using daemon code, writing own pid
     for (unsigned i=0;i<(unsigned)argc;i++) {
@@ -349,8 +349,7 @@ int main( int argc, char *argv[]  )
             return 1;
         }
         cmdArgs = argv+1;
-        mergeCmdParams(globals);
-        cmdArgs = argv+1;
+        loadArgsIntoConfiguration(globals, cmdArgs);
 
         const char *master = globals->queryProp("@master");
         if (!master)

thorlcr/thorutil/thorport.cpp

@@ -63,12 +63,12 @@ unsigned short getExternalFixedPort(unsigned short masterBase, unsigned short ma
   
 void setMachinePortBase(unsigned short base)
 {
-    machineportbase = base?base:THOR_BASESLAVE_PORT;
+    machineportbase = base;
 }
 
 void setMasterPortBase(unsigned short base)
 {
-    masterportbase = base?base:THOR_BASE_PORT;
+    masterportbase = base;
 }
 
 unsigned short getMasterPortBase()