HPCC-21959 Std.File functions to allow DFUServer queue selection

Update after review and retarget to master

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

Update after review

Signed-off-by: Attila Vamos <attila.vamos@gmail.com>

common/environment/environment.cpp

@@ -101,6 +101,25 @@ protected:
     unsigned maxIndex = 0;
 };
 
+class CConstDfuQueueInfoIterator : public CSimpleInterfaceOf<IConstDfuQueueInfoIterator>
+{
+public:
+    CConstDfuQueueInfoIterator();
+
+    virtual bool first() override;
+    virtual bool next() override;
+    virtual bool isValid() override;
+    virtual IConstDfuQueueInfo & query() override;
+    virtual unsigned count() const override;
+
+protected:
+    Owned<IConstDfuQueueInfo> curr;
+    Owned<CLocalEnvironment> constEnv;
+    unsigned index = 1;
+    unsigned maxIndex = 0;
+};
+
+
 class CConstSparkThorInfoIterator : public CSimpleInterfaceOf<IConstSparkThorInfoIterator>
 {
 public:
@@ -168,6 +187,12 @@ private:
     mutable unsigned numOfDropZones;
     mutable unsigned numOfSparkThors;
 
+    mutable bool isDropZoneRestrictionLoaded = false;
+    mutable bool dropZoneRestrictionEnabled = true;
+
+    void buildDfuQueueCache() const;
+    mutable unsigned numOfDfuQueues = 0;
+    mutable bool dfuQueueCacheBuilt = false;
 
     IConstEnvBase * getCache(const char *path) const;
     void setCache(const char *path, IConstEnvBase *value) const;
@@ -175,9 +200,6 @@ private:
     void buildDropZoneCache() const;
     void buildSparkThorCache() const;
     void init();
-    mutable bool isDropZoneRestrictionLoaded = false;
-    mutable bool dropZoneRestrictionEnabled = true;
-
 
     void ensureClusterGroupKeyMap() const // keyPairMap and keyGroupMap it alters is mutable
     {
@@ -316,6 +338,11 @@ public:
     IConstDropZoneInfo * getDropZoneByIndex(unsigned index) const;
     bool isDropZoneRestrictionEnabled() const;
 
+    unsigned getNumberOfDfuQueues() const { buildDfuQueueCache(); return numOfDfuQueues; }
+    IConstDfuQueueInfo * getDfuQueueByIndex(unsigned index) const;
+    virtual IConstDfuQueueInfoIterator * getDfuQueueIterator() const;
+    bool isValidDfuQueueName(const char * queueName) const;
+
     virtual const char *getClusterGroupKeyPairName(const char *group) const override
     {
         synchronized procedure(safeCache);
@@ -471,6 +498,12 @@ public:
             { return c->getSparkThor(name); }
     virtual IConstSparkThorInfoIterator *getSparkThorIterator() const
             { return c->getSparkThorIterator(); }
+
+    virtual IConstDfuQueueInfoIterator * getDfuQueueIterator() const
+            { return c->getDfuQueueIterator(); }
+    virtual bool isValidDfuQueueName(const char * queueName) const
+            { return c->isValidDfuQueueName(queueName); }
+
 };
 
 void CLockedEnvironment::commit()
@@ -1159,6 +1192,22 @@ private:
     StringBuffer posixPath;
 };
 
+class CConstDfuQueueInfo : public CConstEnvBase, implements IConstDfuQueueInfo
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    IMPLEMENT_ICONSTENVBASE;
+    CConstDfuQueueInfo(CLocalEnvironment *env, IPropertyTree *root) : CConstEnvBase(env, root)
+    {
+    }
+
+    virtual IStringVal& getDfuQueueName(IStringVal &str) const
+    {
+        str.set(root->queryProp("@queue"));
+        return str;
+    }
+};
+
 class CConstSparkThorInfo : public CConstEnvBase, implements IConstSparkThorInfo
 {
 public:
@@ -1318,9 +1367,11 @@ void CLocalEnvironment::init()
     machineCacheBuilt = false;
     dropZoneCacheBuilt = false;
     sparkThorCacheBuilt = false;
+    dfuQueueCacheBuilt = false;
     numOfMachines = 0;
     numOfDropZones = 0;
     numOfSparkThors = 0;
+    numOfDfuQueues = 0;
     isDropZoneRestrictionLoaded = false;
     clusterGroupKeyNameCache = false;
     ::getFileAccessUrl(fileAccessUrl);
@@ -1458,6 +1509,35 @@ void CLocalEnvironment::buildDropZoneCache() const
     }
 }
 
+
+void CLocalEnvironment::buildDfuQueueCache() const
+{
+    synchronized procedure(safeCache);
+    if (!dfuQueueCacheBuilt)
+    {
+        Owned<IPropertyTreeIterator> it = p->getElements("Software/DfuServerProcess");
+        ForEach(*it)
+        {
+            const char *qname = it->query().queryProp("@queue");
+
+            if (qname)
+            {
+                StringBuffer x("Software/DfuQueue[@qname=\"");
+                x.append(qname).append("\"]");
+                Owned<IConstEnvBase> cached = new CConstDfuQueueInfo((CLocalEnvironment *) this, &it->query());
+                cache.setValue(x.str(), cached);
+            }
+
+            numOfDfuQueues++;
+            StringBuffer x("Software/DfuQueue[@id=\"");
+            x.append(numOfDfuQueues).append("\"]");
+            Owned<IConstEnvBase> cached = new CConstDfuQueueInfo((CLocalEnvironment *) this, &it->query());
+            cache.setValue(x.str(), cached);
+        }
+        dfuQueueCacheBuilt = true;
+    }
+}
+
 IConstComputerTypeInfo * CLocalEnvironment::getComputerType(const char * name) const
 {
     if (!name)
@@ -1594,6 +1674,23 @@ IConstDropZoneInfo * CLocalEnvironment::getDropZoneByIndex(unsigned index) const
     return (CConstDropZoneInfo *) getCache(xpath.str());
 }
 
+IConstDfuQueueInfo * CLocalEnvironment::getDfuQueueByIndex(unsigned index) const
+{
+    if (!numOfDfuQueues || (index == 0))
+        return nullptr;
+
+    buildDfuQueueCache();
+    if (index > numOfDfuQueues)
+        return nullptr;
+
+    StringBuffer xpath("Software/DfuQueue[@id=\"");
+    xpath.append(index).append("\"]");
+    synchronized procedure(safeCache);
+
+    return (CConstDfuQueueInfo *) getCache(xpath.str());
+}
+
+
 
 IConstInstanceInfo * CLocalEnvironment::getInstance(const char *type, const char *version, const char *domain) const
 {
@@ -1920,6 +2017,29 @@ IConstDropZoneInfoIterator * CLocalEnvironment::getDropZoneIterator() const
     return new CConstDropZoneInfoIterator();
 }
 
+
+IConstDfuQueueInfoIterator * CLocalEnvironment::getDfuQueueIterator() const
+{
+    return new CConstDfuQueueInfoIterator();
+}
+
+bool CLocalEnvironment::isValidDfuQueueName(const char * queueName) const
+{
+    bool retVal = false;
+    if (!isEmptyString(queueName))
+    {
+        Owned<IConstDfuQueueInfoIterator> queueIt = getDfuQueueIterator();
+        ForEach(*queueIt)
+        {
+            SCMStringBuffer _queueName;
+            queueIt->query().getDfuQueueName(_queueName);
+            retVal = streq(queueName, _queueName.str());
+        }
+    }
+
+    return retVal;
+}
+
 IConstMachineInfoIterator * CLocalEnvironment::getMachineIterator() const
 {
     return new CConstMachineInfoIterator();
@@ -2181,6 +2301,51 @@ unsigned CConstDropZoneInfoIterator::count() const
 
 //--------------------------------------------------
 
+CConstDfuQueueInfoIterator::CConstDfuQueueInfoIterator()
+{
+    Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);
+    constEnv.setown((CLocalEnvironment *)factory->openEnvironment());
+    maxIndex = constEnv->getNumberOfDfuQueues();
+}
+
+bool CConstDfuQueueInfoIterator::first()
+{
+    index = 1;
+    curr.setown(constEnv->getDfuQueueByIndex(index));
+    return curr != nullptr;
+}
+
+bool CConstDfuQueueInfoIterator::next()
+{
+    if (index < maxIndex)
+    {
+        index++;
+        curr.setown(constEnv->getDfuQueueByIndex(index));
+    }
+    else
+        curr.clear();
+
+    return curr != nullptr;
+}
+
+bool CConstDfuQueueInfoIterator::isValid()
+{
+    return curr != nullptr;
+}
+
+IConstDfuQueueInfo & CConstDfuQueueInfoIterator::query()
+{
+    return *curr;
+}
+
+unsigned CConstDfuQueueInfoIterator::count() const
+{
+    return maxIndex;
+}
+
+
+//--------------------------------------------------
+
 CConstSparkThorInfoIterator::CConstSparkThorInfoIterator()
 {
     Owned<IEnvironmentFactory> factory = getEnvironmentFactory(true);

common/environment/environment.hpp

@@ -128,6 +128,16 @@ interface  IConstDropZoneInfoIterator : extends IIteratorOf<IConstDropZoneInfo>
     virtual unsigned count() const = 0;
 };
 
+interface IConstDfuQueueInfo : extends IConstEnvBase
+{
+    virtual IStringVal & getDfuQueueName(IStringVal & str) const = 0;
+};
+
+interface IConstDfuQueueInfoIterator : extends IIteratorOf<IConstDfuQueueInfo>
+{
+    virtual unsigned count() const = 0;
+};
+
 interface IConstDaFileSrvInfo : extends IConstEnvBase
 {
     virtual const char *getName() const = 0;
@@ -187,6 +197,9 @@ interface IConstEnvironment : extends IConstEnvBase
     virtual IConstDaFileSrvInfo *getDaFileSrvGroupInfo(const char *name) const = 0;
     virtual IConstSparkThorInfo *getSparkThor(const char *name) const = 0;
     virtual IConstSparkThorInfoIterator *getSparkThorIterator() const = 0;
+
+    virtual IConstDfuQueueInfoIterator * getDfuQueueIterator() const = 0;
+    virtual bool isValidDfuQueueName(const char * queueName) const = 0;
 };
 
 

dali/ft/fterror.hpp

@@ -156,7 +156,6 @@
 #define DFTERR_NoMatchingDropzonePath_Text      "No Drop Zone on '%s' configured at '%s'."
 #define DFTERR_LocalhostAddressUsed_Text        "Localhost address used in remote file name: '%s'"
 
-
 #define DFTERR_UnknownFormatType_Text           "INTERNAL: Save unknown format type"
 #define DFTERR_OutputOffsetMismatch_Text        "INTERNAL: Output offset does not match expected (%" I64F "d expected %" I64F "d) at %s of block %d"
 #define DFTERR_NoSolarisDir_Text                "Directory not yet supported for solaris"

esp/services/ws_fs/ws_fsService.cpp

@@ -1884,7 +1884,15 @@ bool CFileSprayEx::onSprayFixed(IEspContext &context, IEspSprayFixed &req, IEspS
         wu->setClusterName(gName.str());
 
         wu->setJobName(destTitle.str());
-        setDFUServerQueueReq(req.getDFUServerQueue(), wu);
+        const char * dfuQueue = req.getDFUServerQueue();
+        Owned<IEnvironmentFactory> envFactory = getEnvironmentFactory(true);
+        Owned<IConstEnvironment> constEnv = envFactory->openEnvironment();
+        if (!isEmptyString(dfuQueue))
+        {
+            if (!constEnv->isValidDfuQueueName(dfuQueue))
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid DFU server queue name:'%s'", dfuQueue);
+        }
+        setDFUServerQueueReq(dfuQueue, wu);
         setUserAuth(context, wu);
         wu->setCommand(DFUcmd_import);
 
@@ -2049,7 +2057,16 @@ bool CFileSprayEx::onSprayVariable(IEspContext &context, IEspSprayVariable &req,
 
         wu->setClusterName(gName.str());
         wu->setJobName(destTitle.str());
-        setDFUServerQueueReq(req.getDFUServerQueue(), wu);
+
+        const char * dfuQueue = req.getDFUServerQueue();
+        Owned<IEnvironmentFactory> envFactory = getEnvironmentFactory(true);
+        Owned<IConstEnvironment> constEnv = envFactory->openEnvironment();
+        if (!isEmptyString(dfuQueue))
+        {
+            if (!constEnv->isValidDfuQueueName(dfuQueue))
+                throw MakeStringException(ECLWATCH_INVALID_INPUT, "invalid DFU server queue name:'%s'", dfuQueue);
+        }
+        setDFUServerQueueReq(dfuQueue, wu);
         setUserAuth(context, wu);
         wu->setCommand(DFUcmd_import);
 

testing/regress/ecl/key/spray_queue_test.xml

@@ -0,0 +1,42 @@
+<Dataset name='setupCsv'>
+</Dataset>
+<Dataset name='setupXml'>
+</Dataset>
+<Dataset name='setupFix'>
+</Dataset>
+<Dataset name='desprayCsv'>
+ <Row><result>Pass</result></Row>
+</Dataset>
+<Dataset name='desprayOutXml'>
+ <Row><result>Pass</result></Row>
+</Dataset>
+<Dataset name='desprayOuFix'>
+ <Row><result>Pass</result></Row>
+</Dataset>
+<Dataset name='sprayVariableOut1'>
+ <Row><result>Spray variable without queue param: Pass</result></Row>
+</Dataset>
+<Dataset name='sprayVariableOut2'>
+ <Row><result>Spray variable with default queue: Pass</result></Row>
+</Dataset>
+<Dataset name='sprayVariableOut3'>
+ <Row><result>Spray variable with wrong queue name: Fail</result></Row>
+</Dataset>
+<Dataset name='sprayXmlOut1'>
+ <Row><result>Spray XML without queue param: Pass</result></Row>
+</Dataset>
+<Dataset name='sprayXmlOut2'>
+ <Row><result>Spray XML with default queue: Pass</result></Row>
+</Dataset>
+<Dataset name='sprayXmlOut3'>
+ <Row><result>Spray XML with wrong queue name: Fail</result></Row>
+</Dataset>
+<Dataset name='sprayFixedOut1'>
+ <Row><result>Spray fixed without queue param: Pass</result></Row>
+</Dataset>
+<Dataset name='sprayFixedOut2'>
+ <Row><result>Spray fixed with default queue: Pass</result></Row>
+</Dataset>
+<Dataset name='sprayFixedOut3'>
+ <Row><result>Spray fixed with wrong queue name: Fail</result></Row>
+</Dataset>

testing/regress/ecl/spray_queue_test.ecl

@@ -0,0 +1,391 @@
+/*##############################################################################
+
+    Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nothor
+//class=spray
+
+import std.system.thorlib;
+import Std.File AS FileServices;
+import $.setup;
+
+dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+espIpPort := 'http://127.0.0.1:8010/FileSpray' : STORED('espIpPort');
+prefix := setup.Files(false, false).QueryFilePrefix;
+sprayDestGroup := thorlib.group();
+
+unsigned VERBOSE := 0;
+
+Layout_Person := RECORD
+  STRING3  name;
+  UNSIGNED2 age;
+  BOOLEAN good;
+END;
+
+sprayPrepFileName := prefix + 'spray_prep';
+desprayOutFileName := dropzonePath + WORKUNIT + '-spray_input';
+sprayOutFileName := prefix + 'spray_test';
+
+allPeople := DATASET([ {'foo', 10, 1},
+                       {'bar', 12, 0},
+                       {'baz', 32, 1}]
+            ,Layout_Person);
+
+//  Create a small logical file
+setupCsv := output(allPeople, , sprayPrepFileName+'_CSV', CSV, OVERWRITE, NAMED('setupCsv'));
+setupXml := output(allPeople, , sprayPrepFileName+'_XML', XML('Rowtag'), OVERWRITE, NAMED('setupXml'));
+setupFix := output(allPeople, , sprayPrepFileName+'_FIX', OVERWRITE, NAMED('setupFix'));
+
+
+rec := RECORD
+  string result;
+  string msg;
+end;
+
+// Despray it to default drop zone
+rec despray(rec l) := TRANSFORM
+  SELF.msg := FileServices.fDespray(
+                       LOGICALNAME := sprayPrepFileName+'_CSV'
+                      ,DESTINATIONIP := '.'
+                      ,DESTINATIONPATH := desprayOutFileName+'_CSV'
+                      ,ALLOWOVERWRITE := True
+                      );
+  SELF.result := 'Pass';
+end;
+
+dst1 := NOFOLD(DATASET([{'', ''}], rec));
+p1 := NOTHOR(PROJECT(NOFOLD(dst1), despray(LEFT)));
+c1 := CATCH(NOFOLD(p1), ONFAIL(TRANSFORM(rec,
+                                 SELF.result := 'Despray Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    desprayOutCsv := output(c1,,NAMED('desprayCsv'));
+#else
+    desprayOutCsv := output(c1, {result},NAMED('desprayCsv'));
+#end
+
+// Despray it to default drop zone
+rec desprayXml(rec l) := TRANSFORM
+  SELF.msg := FileServices.fDespray(
+                       LOGICALNAME := sprayPrepFileName+'_XML'
+                      ,DESTINATIONIP := '.'
+                      ,DESTINATIONPATH := desprayOutFileName+'_XML'
+                      ,ALLOWOVERWRITE := True
+                      );
+  SELF.result := 'Pass';
+end;
+
+dst2 := NOFOLD(DATASET([{'', ''}], rec));
+p2 := NOTHOR(PROJECT(NOFOLD(dst2), desprayXml(LEFT)));
+c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
+                                 SELF.result := 'Despray Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    desprayOutXml := output(c2,,NAMED('desprayOutXml'));
+#else
+    desprayOutXml := output(c2, {result},NAMED('desprayOutXml'));
+#end
+
+// Despray it to default drop zone
+rec desprayFix(rec l) := TRANSFORM
+  SELF.msg := FileServices.fDespray(
+                       LOGICALNAME := sprayPrepFileName+'_FIX'
+                      ,DESTINATIONIP := '.'
+                      ,DESTINATIONPATH := desprayOutFileName+'_FIX'
+                      ,ALLOWOVERWRITE := True
+                      );
+  SELF.result := 'Pass';
+end;
+
+dst3 := NOFOLD(DATASET([{'', ''}], rec));
+p3 := NOTHOR(PROJECT(NOFOLD(dst3), desprayFix(LEFT)));
+c3 := CATCH(NOFOLD(p3), ONFAIL(TRANSFORM(rec,
+                                 SELF.result := 'Despray Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    desprayOutFix := output(c3,,NAMED('desprayOuFix'));
+#else
+    desprayOutFix := output(c3, {result},NAMED('desprayOuFix'));
+#end
+
+
+
+sprayRec := RECORD
+  string sourceFileName;
+  string destFileName;
+  string dfuQueue;
+  string result;
+  string msg;
+end;
+
+sprayRec sprayVariable(sprayRec l) := TRANSFORM
+    SELF.msg := FileServices.fSprayVariable(
+                        SOURCEIP := '.',
+                        SOURCEPATH := l.sourceFileName,
+                        DESTINATIONGROUP := sprayDestGroup,
+                        DESTINATIONLOGICALNAME := l.destFileName,
+                        TIMEOUT := -1,
+                        ESPSERVERIPPORT := espIpPort,
+                        ALLOWOVERWRITE := true,
+                        DFUSERVERQUEUE := l.dfuQueue
+                        );
+    SELF.result := l.result + ' Pass';
+    SELF.sourceFileName := l.sourceFileName;
+    SELF.destFileName := l.destFileName;
+    SELF.dfuQueue := l.dfuQueue;
+end;
+
+// Spray variable with default DFU queue
+sdst1 := NOFOLD(DATASET([{desprayOutFileName + '_CSV', sprayOutFileName + '_CSV', '', 'Spray variable without queue param:', ''}], sprayRec));
+sp1 := PROJECT(NOFOLD(sdst1), sprayVariable(LEFT));
+sc1 := CATCH(NOFOLD(sp1), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_CSV',
+                                 SELF.destFileName := sprayOutFileName + '_CSV',
+                                 SELF.dfuQueue := '',
+                                 SELF.result := 'Spray variable without queue param: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayVariableOut1 := output(sc1,,NAMED('sprayVariableOut1'));
+#else
+    sprayVariableOut1 := output(sc1, {result},NAMED('sprayVariableOut1'));
+#end
+
+
+// Spray variable with valid DFU queue
+sdst2 := NOFOLD(DATASET([{desprayOutFileName + '_CSV', sprayOutFileName + '_CSV', 'dfuserver_queue', 'Spray variable with default queue:', ''}], sprayRec));
+sp2 := PROJECT(NOFOLD(sdst2), sprayVariable(LEFT));
+sc2 := CATCH(NOFOLD(sp2), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_CSV',
+                                 SELF.destFileName := sprayOutFileName + '_CSV',
+                                 SELF.dfuQueue := 'dfuserver_queue',
+                                 SELF.result := 'Spray variable with default queue: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayVariableOut2 := output(sc2,,NAMED('sprayVariableOut2'));
+#else
+    sprayVariableOut2 := output(sc2, {result},NAMED('sprayVariableOut2'));
+#end
+
+// Spray variable with invalid DFU queue
+sdst3 := NOFOLD(DATASET([{desprayOutFileName + '_CSV', sprayOutFileName + '_CSV', 'bela_queue', 'Spray variable with wrong queue name:', ''}], sprayRec));
+sp3 := PROJECT(NOFOLD(sdst3), sprayVariable(LEFT));
+sc3 := CATCH(NOFOLD(sp3), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_CSV',
+                                 SELF.destFileName := sprayOutFileName + '_CSV',
+                                 SELF.dfuQueue := 'bela_queue',
+                                 SELF.result := 'Spray variable with wrong queue name: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayVariableOut3 := output(sc3,,NAMED('sprayVariableOut3'));
+#else
+    sprayVariableOut3 := output(sc3, {result},NAMED('sprayVariableOut3'));
+#end
+
+
+sprayRec sprayXml(sprayRec l) := TRANSFORM
+    SELF.msg := FileServices.fSprayXml(
+                            SOURCEIP := '.',
+                            SOURCEPATH := l.sourceFileName,
+                            SOURCEROWTAG := 'Rowtag',
+                            DESTINATIONGROUP := sprayDestGroup,
+                            DESTINATIONLOGICALNAME := l.destFileName,
+                            TIMEOUT := -1,
+                            ESPSERVERIPPORT := espIpPort,
+                            ALLOWOVERWRITE := true,
+                            DFUSERVERQUEUE := l.dfuQueue
+                            ); 
+    SELF.result := l.result + ' Pass';
+    SELF.sourceFileName := l.sourceFileName;
+    SELF.destFileName := l.destFileName;
+    SELF.dfuQueue := l.dfuQueue;
+end;
+
+// Spray XML with default DFU queue
+sxdst1 := NOFOLD(DATASET([{desprayOutFileName + '_XML', sprayOutFileName + '_XML', '', 'Spray XML without queue param:', ''}], sprayRec));
+sxp1 := PROJECT(NOFOLD(sxdst1), sprayXml(LEFT));
+sxc1 := CATCH(NOFOLD(sxp1), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_XML',
+                                 SELF.destFileName := sprayOutFileName + '_XML',
+                                 SELF.dfuQueue := '',
+                                 SELF.result := 'Spray XML without queue param: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayXmlOut1 := output(sxc1,,NAMED('sprayXmlOut1'));
+#else
+    sprayXmlOut1 := output(sxc1, {result},NAMED('sprayXmlOut1'));
+#end
+
+
+// Spray XML with valid DFU queue
+sxdst2 := NOFOLD(DATASET([{desprayOutFileName + '_XML', sprayOutFileName + '_XML', 'dfuserver_queue', 'Spray XML with default queue:', ''}], sprayRec));
+sxp2 := PROJECT(NOFOLD(sxdst2), sprayXml(LEFT));
+sxc2 := CATCH(NOFOLD(sxp2), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_XML',
+                                 SELF.destFileName := sprayOutFileName + '_XML',
+                                 SELF.dfuQueue := 'dfuserver_queue',
+                                 SELF.result := 'Spray XML with default queue: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayXmlOut2 := output(sxc2,,NAMED('sprayXmlOut2'));
+#else
+    sprayXmlOut2 := output(sxc2, {result},NAMED('sprayXmlOut2'));
+#end
+
+// Spray XML with invalid DFU queue
+sxdst3 := NOFOLD(DATASET([{desprayOutFileName + '_XML', sprayOutFileName + '_XML', 'bela_queue', 'Spray XML with wrong queue name:', ''}], sprayRec));
+sxp3 := PROJECT(NOFOLD(sxdst3), sprayXml(LEFT));
+sxc3 := CATCH(NOFOLD(sxp3), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_XML',
+                                 SELF.destFileName := sprayOutFileName + '_XML',
+                                 SELF.dfuQueue := 'bela_queue',
+                                 SELF.result := 'Spray XML with wrong queue name: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayXmlOut3 := output(sxc3,,NAMED('sprayXmlOut3'));
+#else
+    sprayXmlOut3 := output(sxc3, {result},NAMED('sprayXmlOut3'));
+#end
+
+
+sprayRec sprayFixed(sprayRec l) := TRANSFORM
+    SELF.msg := FileServices.fSprayFixed(
+                            SOURCEIP := '.',
+                            SOURCEPATH := l.sourceFileName,
+                            RECORDSIZE := 9,
+                            DESTINATIONGROUP := sprayDestGroup,
+                            DESTINATIONLOGICALNAME := l.destFileName,
+                            TIMEOUT := -1,
+                            ESPSERVERIPPORT := espIpPort,
+                            ALLOWOVERWRITE := true,
+                            DFUSERVERQUEUE := l.dfuQueue
+                            ); 
+    SELF.result := l.result + ' Pass';
+    SELF.sourceFileName := l.sourceFileName;
+    SELF.destFileName := l.destFileName;
+    SELF.dfuQueue := l.dfuQueue;
+end;
+
+// Spray fixed with default DFU queue
+sfdst1 := NOFOLD(DATASET([{desprayOutFileName + '_FIX', sprayOutFileName + '_FIX', '', 'Spray fixed without queue param:', ''}], sprayRec));
+sfp1 := PROJECT(NOFOLD(sfdst1), sprayFixed(LEFT));
+sfc1 := CATCH(NOFOLD(sfp1), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_FIX',
+                                 SELF.destFileName := sprayOutFileName + '_FIX',
+                                 SELF.dfuQueue := '',
+                                 SELF.result := 'Spray fixed without queue param: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayFixedOut1 := output(sfc1,,NAMED('sprayFixedOut1'));
+#else
+    sprayFixedOut1 := output(sfc1, {result},NAMED('sprayFixedOut1'));
+#end
+
+
+// Spray fixed with valid DFU queue
+sfdst2 := NOFOLD(DATASET([{desprayOutFileName + '_FIX', sprayOutFileName + '_FIX', 'dfuserver_queue', 'Spray fixed with default queue:', ''}], sprayRec));
+sfp2 := PROJECT(NOFOLD(sfdst2), sprayFixed(LEFT));
+sfc2 := CATCH(NOFOLD(sfp2), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_FIX',
+                                 SELF.destFileName := sprayOutFileName + '_FIX',
+                                 SELF.dfuQueue := 'dfuserver_queue',
+                                 SELF.result := 'Spray fixed with default queue: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayFixedOut2 := output(sfc2,,NAMED('sprayFixedOut2'));
+#else
+    sprayFixedOut2 := output(sfc2, {result},NAMED('sprayFixedOut2'));
+#end
+
+// Spray fixed with invalid DFU queue
+sfdst3 := NOFOLD(DATASET([{desprayOutFileName + '_FIX', sprayOutFileName + '_FIX', 'bela_queue', 'Spray fixed with wrong queue name:', ''}], sprayRec));
+sfp3 := PROJECT(NOFOLD(sfdst3), sprayFixed(LEFT));
+sfc3 := CATCH(NOFOLD(sfp3), ONFAIL(TRANSFORM(sprayRec,
+                                 SELF.sourceFileName := desprayOutFileName + '_FIX',
+                                 SELF.destFileName := sprayOutFileName + '_FIX',
+                                 SELF.dfuQueue := 'bela_queue',
+                                 SELF.result := 'Spray fixed with wrong queue name: Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+
+#if (VERBOSE = 1)
+    sprayFixedOut3 := output(sfc3,,NAMED('sprayFixedOut3'));
+#else
+    sprayFixedOut3 := output(sfc3, {result},NAMED('sprayFixedOut3'));
+#end
+
+
+
+sequential (
+    //output(dropzonePath, NAMED('dropzonePath')),
+    // Preparation
+    setupCsv,
+    setupXml,
+    setupFix,
+    
+    desprayOutCsv,
+    desprayOutXml,
+    desprayOutFix,
+
+    // Spray tests
+    sprayVariableOut1,
+    sprayVariableOut2,
+    sprayVariableOut3,
+    
+    sprayXmlOut1,
+    sprayXmlOut2,
+    sprayXmlOut3,
+    
+    sprayFixedOut1,
+    sprayFixedOut2,
+    sprayFixedOut3,
+
+    // Clean-up
+    FileServices.DeleteExternalFile('.', desprayOutFileName+'_CSV'),
+    FileServices.DeleteExternalFile('.', desprayOutFileName+'_XML'),
+    FileServices.DeleteExternalFile('.', desprayOutFileName+'_FIX'),
+    
+    FileServices.DeleteLogicalFile(sprayPrepFileName+'_CSV'),
+    FileServices.DeleteLogicalFile(sprayPrepFileName+'_XML'),
+    FileServices.DeleteLogicalFile(sprayPrepFileName+'_FIX'),
+    
+    FileServices.DeleteLogicalFile(sprayOutFileName+'_CSV'),
+    FileServices.DeleteLogicalFile(sprayOutFileName+'_XML'),
+    FileServices.DeleteLogicalFile(sprayOutFileName + '_FIX'),
+    
+);
+