Merge branch 'candidate-8.2.x' into candidate-8.4.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

.gitignore

@@ -11,3 +11,4 @@ ln/
 !esp/src/.vscode
 eclcc.log
 *.pyc
+./helm/examples/azure/sa/env-sa

esp/bindings/http/platform/httptransport.cpp

@@ -2169,6 +2169,9 @@ int CHttpRequest::readContentToFiles(const char * netAddress, const char * path,
         if (writeError)
             break;
 
+        // if remote file is on Windows we must close it before renaming it
+        fileio->close();
+
         file->rename(fileNameWithPath);
 
         if (!foundAnotherFile)

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -2235,6 +2235,46 @@ void WsWuInfo::getWUProcessLogSpecs(const char* processName, const char* logSpec
 }
 #endif
 
+bool WsWuInfo::validateWUProcessLog(const char* file, bool eclAgent)
+{
+    Owned<IStringIterator> logs = cw->getLogs(eclAgent ? "EclAgent" : "Thor");
+    ForEach (*logs)
+    {
+        SCMStringBuffer logName;
+        logs->str(logName);
+        if (logName.length() < 1)
+            continue;
+
+        if (strieq(file, logName.str()))
+            return true;
+    }
+    return false;
+}
+
+bool WsWuInfo::validateWUAssociatedFile(const char* file, WUFileType type)
+{
+    Owned<IConstWUQuery> query = cw->getQuery();
+    if (!query)
+        return false;
+
+    Owned<IConstWUAssociatedFileIterator> iter = &query->getAssociatedFiles();
+    ForEach(*iter)
+    {
+        IConstWUAssociatedFile & cur = iter->query();
+        if (cur.getType() != type)
+            continue;
+
+        SCMStringBuffer name;
+        cur.getName(name);
+        if (name.length() < 1)
+            continue;
+
+        if (strieq(file, name.str()))
+            return true;
+    }
+    return false;
+}
+
 void WsWuInfo::getWorkunitResTxt(MemoryBuffer& buf)
 {
     Owned<IConstWUQuery> query = cw->getQuery();
@@ -4007,8 +4047,11 @@ IFileIOStream* CWsWuFileHelper::createWUFileIOStream(IEspContext& context, const
     return createIOStreamWithFileName(zipFileNameWithPath.str(), IFOread);
 }
 
-void CWsWuFileHelper::validateFilePath(const char *file, bool UNCFileName, const char *fileType, const char *compType, const char *compName)
+void CWsWuFileHelper::validateFilePath(const char* file, WsWuInfo& winfo, CWUFileType wuFileType, bool UNCFileName, const char* fileType, const char* compType, const char* compName)
 {
+    if (validateWUFile(file, winfo, wuFileType))
+        return;
+
     StringBuffer actualPath;
     if (UNCFileName)
         splitUNCFilename(file, nullptr, &actualPath, nullptr, nullptr);
@@ -4020,6 +4063,42 @@ void CWsWuFileHelper::validateFilePath(const char *file, bool UNCFileName, const
         throw makeStringExceptionV(ECLWATCH_INVALID_INPUT, "Invalid file path %s", actualPath.str());
 }
 
+bool CWsWuFileHelper::validateWUFile(const char* file, WsWuInfo& winfo, CWUFileType wuFileType)
+{
+    bool valid = false;
+    switch (wuFileType)
+    {
+    case CWUFileType_ThorLog:
+    {
+        valid = winfo.validateWUProcessLog(file, false);
+        break;
+    }
+    case CWUFileType_EclAgentLog:
+    {
+        valid = winfo.validateWUProcessLog(file, true);
+        break;
+    }
+    case CWUFileType_CPP:
+    {
+        valid = winfo.validateWUAssociatedFile(file, FileTypeCpp);
+        break;
+    }
+    case CWUFileType_LOG:
+    {
+        valid = winfo.validateWUAssociatedFile(file, FileTypeLog);
+        break;
+    }
+    case CWUFileType_XML:
+    {
+        valid = winfo.validateWUAssociatedFile(file, FileTypeXml);
+        break;
+    }
+    default:
+        throw MakeStringException(ECLWATCH_INVALID_INPUT, "Unsupported file type %d.", wuFileType);
+    }
+    return valid;
+}
+
 void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, WsWuInfo& winfo, IConstWUFileOption& item,
     StringBuffer& fileName, StringBuffer& fileMimeType)
 {
@@ -4039,9 +4118,9 @@ void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, Ws
     {
         const char *file=item.getName();
 #ifndef _CONTAINERIZED
-        validateFilePath(file, false, "run", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, false, "run", nullptr, nullptr);
 #else
-        validateFilePath(file, false, "query", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, false, "query", nullptr, nullptr);
 #endif
 
         const char *tail=pathTail(file);
@@ -4071,7 +4150,7 @@ void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, Ws
     case CWUFileType_ThorLog:
     {
         const char *file=item.getName();
-        validateFilePath(file, true, "log", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, true, "log", nullptr, nullptr);
 
         fileName.set("thormaster.log");
         fileMimeType.set(HTTP_TYPE_TEXT_PLAIN);
@@ -4092,7 +4171,7 @@ void CWsWuFileHelper::readWUFile(const char* wuid, const char* workingFolder, Ws
     case CWUFileType_EclAgentLog:
     {
         const char *file=item.getName();
-        validateFilePath(file, true, "log", nullptr, nullptr);
+        validateFilePath(file, winfo, fileType, true, "log", nullptr, nullptr);
 
         fileName.set("eclagent.log");
         fileMimeType.set(HTTP_TYPE_TEXT_PLAIN);

esp/services/ws_workunits/ws_workunitsHelpers.hpp

@@ -221,7 +221,8 @@ public:
         const char* instanceName, const char *ipAddress, const char* logDate, int slaveNum,
         MemoryBuffer& buf, const char* outFile, bool forDownload);
 #endif
-
+    bool validateWUProcessLog(const char* file, bool eclAgent);
+    bool validateWUAssociatedFile(const char* file, WUFileType type);
     void getWorkunitResTxt(MemoryBuffer& buf);
     void getWorkunitArchiveQuery(StringBuffer& str);
     void getWorkunitArchiveQuery(MemoryBuffer& mb);
@@ -654,7 +655,8 @@ public:
         CWUFileDownloadOption &downloadOptions, StringBuffer &contentType);
 
     IFileIOStream* createIOStreamWithFileName(const char *fileNameWithPath, IFOmode mode);
-    void validateFilePath(const char *file, bool UNCFileName, const char *fileType, const char *compType, const char *compName);
+    void validateFilePath(const char *file, WsWuInfo &winfo, CWUFileType wuFileType, bool UNCFileName, const char *fileType, const char *compType, const char *compName);
+    bool validateWUFile(const char *file, WsWuInfo &winfo, CWUFileType wuFileType);
 };
 
 class CWsWuEmailHelper

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -3042,9 +3042,9 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             {
                 const char* file = req.getName();
 #ifndef _CONTAINERIZED
-                helper.validateFilePath(file, false, "run", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, strieq(File_Cpp,req.getType())? CWUFileType_CPP : CWUFileType_LOG, false, "run", nullptr, nullptr);
 #else
-                helper.validateFilePath(file, false, "query", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, strieq(File_Cpp,req.getType())? CWUFileType_CPP : CWUFileType_LOG, false, "query", nullptr, nullptr);
 #endif
                 winfo.getWorkunitCpp(file, req.getDescription(), req.getIPAddress(),mb, opt > 0, nullptr);
                 openSaveFile(context, opt, req.getSizeLimit(), req.getName(), HTTP_TYPE_TEXT_PLAIN, mb, resp);
@@ -3066,7 +3066,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             else if (strncmp(req.getType(), File_ThorLog, 7) == 0)
             {
                 const char* file = req.getName();
-                helper.validateFilePath(file, true, "log", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, CWUFileType_ThorLog, true, "log", nullptr, nullptr);
                 winfo.getWorkunitThorMasterLog(nullptr, file, mb, nullptr);
                 openSaveFile(context, opt, req.getSizeLimit(), "thormaster.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
@@ -3079,7 +3079,7 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             else if (strieq(File_EclAgentLog,req.getType()))
             {
                 const char* file = req.getName();
-                helper.validateFilePath(file, true, "log", nullptr, nullptr);
+                helper.validateFilePath(file, winfo, CWUFileType_EclAgentLog, true, "log", nullptr, nullptr);
                 winfo.getWorkunitEclAgentLog(nullptr, file, req.getProcess(), mb, nullptr);
                 openSaveFile(context, opt, req.getSizeLimit(), "eclagent.log", HTTP_TYPE_TEXT_PLAIN, mb, resp);
             }
@@ -3088,9 +3088,9 @@ bool CWsWorkunitsEx::onWUFile(IEspContext &context,IEspWULogFileRequest &req, IE
             {
                 const char* name  = req.getName();
 #ifndef _CONTAINERIZED
-                helper.validateFilePath(name, false, "run", nullptr, nullptr);
+                helper.validateFilePath(name, winfo, CWUFileType_XML, false, "run", nullptr, nullptr);
 #else
-                helper.validateFilePath(name, false, "query", nullptr, nullptr);
+                helper.validateFilePath(name, winfo, CWUFileType_XML, false, "query", nullptr, nullptr);
 #endif
                 const char* ptr = strrchr(name, '/');
                 if (ptr)

esp/src/eclwatch/LFDetailsWidget.js

@@ -276,11 +276,11 @@ define([
             });
             this.logicalFile.refresh();
 
-            this.isProtected.on("change", function (evt) {
+            this.isProtected.on("click", function (evt) {
                 context._onSave();
             });
 
-            this.isRestricted.on("change", function (evt) {
+            this.isRestricted.on("click", function (evt) {
                 context._onSave();
             });
         },

esp/src/eclwatch/SFDetailsWidget.js

@@ -36,6 +36,7 @@ define([
     "dijit/form/Button",
     "dijit/form/DropDownButton",
     "dijit/form/ToggleButton",
+    "dijit/Fieldset",
     "dijit/TitlePane"
 ], function (exports, declare, nlsHPCCMod, arrayUtil, dom, domAttr, domClass, domForm, MemoryMod, Observable, all,
     registry,

helm/examples/azure/sa/create-sa.sh

@@ -31,7 +31,7 @@ fi
 rc=$(az group exists --name ${SA_RESOURCE_GROUP})
 if [ "$rc" != "true" ]
 then
-  az group create --name ${SA_RESOURCE_GROUP} --location ${SA_LOCATION}
+  az group create --name ${SA_RESOURCE_GROUP} --location ${SA_LOCATION} --tags ${TAGS}
 fi
 
 az storage account check-name -n $STORAGE_ACCOUNT_NAME | \
@@ -43,7 +43,8 @@ then
     -n $STORAGE_ACCOUNT_NAME \
     -g $SA_RESOURCE_GROUP \
     -l $SA_LOCATION \
-    --sku $SA_SKU
+    --sku $SA_SKU \
+    --tags ${TAGS}
 fi
 # Export the connection string as an environment variable,
 # this is used when creating the Azure file share
@@ -54,7 +55,7 @@ for shareName in $SHARE_NAMES
 do
   az storage share exists --connection-string "${AZURE_STORAGE_CONNECTION_STRING}" \
     --name  $shareName | grep -q  "\"exists\":[[:space:]]*false"
-  if [ $? -eq 0 ]
+  if [ $? -ne 0 ]
   then
     echo "create share $shareName"
     az storage share create \

helm/examples/azure/sa/env-sa

@@ -2,6 +2,7 @@
 SUBSCRIPTION=
 STORAGE_ACCOUNT_NAME=
 SA_RESOURCE_GROUP=
+TAGS=
 # Set the same location as Kubernetes cluster
 SA_LOCATION=eastus
 SA_KEY_DIR=~/.azure/storage
@@ -14,4 +15,4 @@ SA_SKU=Standard_LRS
 # Kubernetes secret.
 SECRET_NAME=
 SECRET_NAMESPACE="default"
-SHARE_NAMES=
+SHARE_NAMES="dalishare dllshare sashashare datashare lzshare"