HPCC-19259 Move all spray and despray tests from hthor to thor and roxie target

Change exclusion (//no<engine>) to //nohthor

Improve file name generation to avoid to use same file in different
test executed in parallel.

Exclude despray.ecl from Thor to avoid NOTHOR() complications

Signed-off-by: AttilaVamos <attila.vamos@gmail.com>

testing/regress/ecl/despray.ecl

@@ -15,16 +15,31 @@
     limitations under the License.
 ############################################################################# */
 
-import $.setup;
-prefix := setup.Files(false, false).IndexPrefix;
-
+// The aim of this code is to test the Dropzone Restriction feature implemented in DFU server and
+// in some cases (e.g. path points outside of the DZ) it forces the DFU server to throw exeption.
+//
+// NOTHOR() effectively forces something to be executed globally.  At the moment if a global operation
+// fails then the query fails - rather than continuing and only failing if the result of using that
+// operation causes a failure.
+// So, to avoid to abort this code is excluded from Thor target
 //nothor
 
+//nohthor
 //class=spray
 
+import std.system.thorlib;
+import $.setup;
 import Std.File AS FileServices;
 
+jlib:= SERVICE
+    unsigned8 rtlTick() : library='jlib',eclrtl,entrypoint='rtlNano';
+END;
+
+
 dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+engine := thorlib.platform() : stored('thor');
+prefix := setup.Files(false, false).FilePrefix + '-' + engine + '-';
+suffix := '-' + jlib.rtlTick() : stored('startTime');
 
 unsigned VERBOSE := 0;
 
@@ -38,12 +53,10 @@ allPeople := DATASET([ {1,'Fred','Smith'},
                        {2,'Joe','Blow'},
                        {3,'Jane','Smith'}],Layout_Person);
 
-import * from lib_fileservices;
-
 SrcAddrIp := '.';
 SrcAddrLocalhost := 'localhost';
 File := 'persons';
-SourceFile := prefix + File;
+SourceFile := prefix + File + suffix;
 
 //  Outputs  ---
 setupPeople := output(allPeople, , SourceFile, OVERWRITE);
@@ -78,7 +91,9 @@ end;
 // This should be fine based on valid target file path and SrcAddIp
 DestFile1 := dropzonePath + File;
 dst2 := NOFOLD(DATASET([{SourceFile, DestFile1, SrcAddrIp, True, '', ''}], rec));
+
 p2 := PROJECT(NOFOLD(dst2), t(LEFT));
+
 c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
                                  SELF.sourceFile := SourceFile,
                                  SELF.destFile := DestFile1,
@@ -96,7 +111,9 @@ c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
 
 // This should fail based on 'localhost' used as source address
 dst3 := NOFOLD(DATASET([{SourceFile, DestFile1, SrcAddrLocalhost, True, '', ''}], rec));
+
 p3 := PROJECT(NOFOLD(dst3), t(LEFT));
+
 c3 := CATCH(NOFOLD(p3), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile1,
@@ -115,7 +132,9 @@ c3 := CATCH(NOFOLD(p3), ONFAIL(TRANSFORM(rec,
 // This should fail based on '/./' used in target path
 DestFile4 := dropzonePath + './' + File;
 dst4 := NOFOLD(DATASET([{SourceFile, DestFile4, SrcAddrIp, True, '', ''}], rec));
+
 p4 := PROJECT(NOFOLD(dst4), t(LEFT));
+
 c4 := CATCH(NOFOLD(p4), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile4,
@@ -134,7 +153,9 @@ c4 := CATCH(NOFOLD(p4), ONFAIL(TRANSFORM(rec,
 // This should fail based on '/../' used in target path
 DestFile5 := dropzonePath + '../' + File;
 dst5 := NOFOLD(DATASET([{SourceFile, DestFile5, SrcAddrIp, True, '', ''}], rec));
+
 p5 := PROJECT(NOFOLD(dst5), t(LEFT));
+
 c5 := CATCH(NOFOLD(p5), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile5,
@@ -154,7 +175,9 @@ c5 := CATCH(NOFOLD(p5), ONFAIL(TRANSFORM(rec,
 // not an existing dropzone path used in target file path
 DestFile6 := '/var/lib/HPCCSystems/mydropzona/' + File;
 dst6 := NOFOLD(DATASET([{SourceFile, DestFile6, SrcAddrIp, True, '', ''}], rec));
+
 p6 := PROJECT(NOFOLD(dst6), t(LEFT));
+
 c6 := CATCH(NOFOLD(p6), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile6,
@@ -174,7 +197,9 @@ c6 := CATCH(NOFOLD(p6), ONFAIL(TRANSFORM(rec,
 // try to despray out of a drop zone
 DestFile7 := '/var/lib/HPCCSystems/' + File;
 dst7 := NOFOLD(DATASET([{SourceFile, DestFile7, SrcAddrIp, True, '', ''}], rec));
+
 p7 := PROJECT(NOFOLD(dst7), t(LEFT));
+
 c7 := CATCH(NOFOLD(p7), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile7,
@@ -194,7 +219,9 @@ c7 := CATCH(NOFOLD(p7), ONFAIL(TRANSFORM(rec,
 // not an existing dropzone path used in target file path
 DestFile8 := '/var/lib/HPCCSystems/mydropzone../' + File;
 dst8 := NOFOLD(DATASET([{SourceFile, DestFile8, SrcAddrIp, True, '', ''}], rec));
+
 p8 := PROJECT(NOFOLD(dst8), t(LEFT));
+
 c8 := CATCH(NOFOLD(p8), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile8,
@@ -211,9 +238,11 @@ c8 := CATCH(NOFOLD(p8), ONFAIL(TRANSFORM(rec,
 
 
 // This should pass based on valid target file path and valid source address used
-DestFile9 := dropzonePath + 'test/' + File;
+DestFile9 := dropzonePath + 'test/' + prefix + File + suffix;
 dst9 := NOFOLD(DATASET([{SourceFile, DestFile9, SrcAddrIp, True, '', ''}], rec));
+
 p9 := PROJECT(NOFOLD(dst9), t(LEFT));
+
 c9 := CATCH(NOFOLD(p9), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile9,
@@ -234,7 +263,9 @@ c9 := CATCH(NOFOLD(p9), ONFAIL(TRANSFORM(rec,
 // This shoud fail based on the previous despray already created a file on the target path
 // and overwrite not allowed.
 dst10 := NOFOLD(DATASET([{SourceFile, DestFile9, SrcAddrIp, False, '', ''}], rec));
+
 p10 := PROJECT(NOFOLD(dst10), t(LEFT));
+
 c10 := CATCH(NOFOLD(p10), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Fail',
                                  SELF.destFile := DestFile9,
@@ -260,6 +291,16 @@ SEQUENTIAL(
     o7,
     o8,
     o9,
-    o10
-  )
+  ),
+    // To ensure it running after o9
+    o10,
+    // Clean-up
+    FileServices.DeleteLogicalFile(SourceFile),
+    FileServices.DeleteExternalFile('.', DestFile1),
+    FileServices.DeleteExternalFile('.', DestFile4),
+    FileServices.DeleteExternalFile('.', DestFile5),
+    FileServices.DeleteExternalFile('.', DestFile6),
+    FileServices.DeleteExternalFile('.', DestFile7),
+    FileServices.DeleteExternalFile('.', DestFile8),
+    FileServices.DeleteExternalFile('.', DestFile9),
 );

testing/regress/ecl/spray_expire_test.ecl

@@ -15,19 +15,14 @@
     limitations under the License.
 ############################################################################## */
 
+//nohthor
+
+//class=spray
+
 import Std.File AS FileServices;
 import $.setup;
 prefix := setup.Files(false, false).FilePrefix;
 
-// This is not an engine test, but a DFU.
-// Doesn't matter much which engine does it, so we restrict to only one
-
-//noRoxie
-//noThorLCR
-//noThor
-
-//class=spray
-
 dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
 
 unsigned VERBOSE := 0;
@@ -69,7 +64,7 @@ rec despray(rec l) := TRANSFORM
 end;
 
 dst1 := NOFOLD(DATASET([{'', ''}], rec));
-p1 := PROJECT(NOFOLD(dst1), despray(LEFT));
+p1 := NOTHOR(PROJECT(NOFOLD(dst1), despray(LEFT)));
 c1 := CATCH(NOFOLD(p1), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Despray Fail',
                                  SELF.msg := FAILMESSAGE
@@ -93,7 +88,7 @@ rec desprayXml(rec l) := TRANSFORM
 end;
 
 dst2 := NOFOLD(DATASET([{'', ''}], rec));
-p2 := PROJECT(NOFOLD(dst1), desprayXml(LEFT));
+p2 := NOTHOR(PROJECT(NOFOLD(dst1), desprayXml(LEFT)));
 c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Despray Fail',
                                  SELF.msg := FAILMESSAGE
@@ -112,15 +107,15 @@ DestFile := prefix + 'spray_expire.txt';
 ESPportIP := 'http://127.0.0.1:8010/FileSpray';
 
 expireDaysOut1 := 9;
-expireDaysIn1 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
+expireDaysIn1 := NOTHOR(FileServices.GetLogicalFileAttribute(DestFile, 'expireDays'));
 res1:=if(expireDaysIn1 = intformat(expireDaysOut1,1,0), 'Pass', 'Fail ('+intformat(expireDaysOut1,1,0)+','+expireDaysIn1+')');
 
 expireDaysOut2 := 7;
-expireDaysIn2 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
+expireDaysIn2 := NOTHOR(FileServices.GetLogicalFileAttribute(DestFile, 'expireDays'));
 res2:=if(expireDaysIn2 = intformat(expireDaysOut2,1,0), 'Pass', 'Fail ('+intformat(expireDaysOut2,1,0)+','+expireDaysIn2+')');
 
 expireDaysOut3 := 11;
-expireDaysIn3 := FileServices.GetLogicalFileAttribute(DestFile, 'expireDays');
+expireDaysIn3 := NOTHOR(FileServices.GetLogicalFileAttribute(DestFile, 'expireDays'));
 res3:=if(expireDaysIn3 = intformat(expireDaysOut3,2,0), 'Pass', 'Fail ('+intformat(expireDaysOut3,2,0)+','+expireDaysIn3+')');
 
 expireDaysOut4 := 13;

testing/regress/ecl/spray_header_test.ecl

@@ -15,26 +15,27 @@
     limitations under the License.
 ############################################################################## */
 
-import Std.File AS FileServices;
-import $.setup;
-prefix := setup.Files(false, false).FilePrefix;
-
-// This is not an engine test, but a DFU.
-// Doesn't matter much which engine does it, so we restrict to only one
-
-//noRoxie
-//noThorLCR
+//nohthor
 
 //class=spray
 
 //version isTerminated=false
 //version isTerminated=true
 
+import std.system.thorlib;
+import Std.File AS FileServices;
+import $.setup;
 import ^ as root;
 
-isTerminated := #IFDEFINED(root.isTerminated, false);
+jlib:= SERVICE
+    unsigned8 rtlTick() : library='jlib',eclrtl,entrypoint='rtlNano';
+END;
 
+isTerminated := #IFDEFINED(root.isTerminated, false);
 dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+engine := thorlib.platform();
+prefix := setup.Files(false, false).FilePrefix + engine + '-';
+suffix := '-' + jlib.rtlTick() : stored('startTime');
 
 unsigned VERBOSE := 0;
 
@@ -49,19 +50,19 @@ END;
 header := DATASET([{'Id', 'Field1', 'Field2', 'Field3', 'Field4'}], Layout);
 
 #if (isTerminated)
-    sprayPrepFileName := prefix + 'spray_prep_terminated';
+    sprayPrepFileName := prefix + 'spray_prep_terminated' + suffix;
     // Create a one record CSV logical file with terminator a the end
     setupFile := output(header, , sprayPrepFileName, CSV, OVERWRITE);
 
-    desprayOutFileName := dropzonePath + 'spray_input_terminated';
-    sprayOutFileName := prefix + 'spray_test_terminated';
+    desprayOutFileName := dropzonePath + prefix + 'spray_input_terminated' + suffix;
+    sprayOutFileName := prefix + 'spray_test_terminated' + suffix;
 #else
-    sprayPrepFileName := prefix + 'spray_prep_not_terminated';
+    sprayPrepFileName := prefix + 'spray_prep_not_terminated' + suffix;
     // Create a one record CSV logical file without terminator a the end
     setupFile := output(header, , sprayPrepFileName, CSV(TERMINATOR('')), OVERWRITE);
 
-    desprayOutFileName := dropzonePath + 'spray_input_not_terminated';
-    sprayOutFileName := prefix + 'spray_test_not_terminated';
+    desprayOutFileName := dropzonePath + prefix + 'spray_input_not_terminated' +  suffix;
+    sprayOutFileName := prefix + 'spray_test_not_terminated' + suffix;
 #end
 
 rec := RECORD
@@ -82,7 +83,7 @@ rec despray(rec l) := TRANSFORM
 end;
 
 dst1 := NOFOLD(DATASET([{'', ''}], rec));
-p1 := PROJECT(NOFOLD(dst1), despray(LEFT));
+p1 := NOTHOR(PROJECT(NOFOLD(dst1), despray(LEFT)));
 c1 := CATCH(NOFOLD(p1), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Despray Fail',
                                  SELF.msg := FAILMESSAGE
@@ -111,7 +112,7 @@ rec spray(rec l) := TRANSFORM
 end;
 
 dst2 := NOFOLD(DATASET([{'', ''}], rec));
-p2 := PROJECT(NOFOLD(dst2), spray(LEFT));
+p2 := NOTHOR(PROJECT(NOFOLD(dst2), spray(LEFT)));
 c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Spray Fail',
                                  SELF.msg := FAILMESSAGE
@@ -140,4 +141,5 @@ SEQUENTIAL(
     FileServices.DeleteExternalFile('.', desprayOutFileName),
     FileServices.DeleteLogicalFile(sprayOutFileName),
     FileServices.DeleteLogicalFile(sprayPrepFileName),
+
 );

testing/regress/ecl/spray_test.ecl

@@ -15,16 +15,7 @@
     limitations under the License.
 ############################################################################## */
 
-import Std.File AS FileServices;
-import $.setup;
-prefix := setup.Files(false, false).FilePrefix;
-
-// This is not an engine test, but a DFU.
-// Doesn't matter much which engine does it, so we restrict to only one
-
-//noRoxie
-//noThorLCR
-//noThor
+//nohthor
 
 //class=spray
 
@@ -32,8 +23,21 @@ prefix := setup.Files(false, false).FilePrefix;
 //version sprayFixed=false,sprayEmpty=false
 //version sprayFixed=false,sprayEmpty=true
 
+
+import std.system.thorlib;
+import Std.File AS FileServices;
+import $.setup;
 import ^ as root;
 
+jlib:= SERVICE
+    unsigned8 rtlTick() : library='jlib',eclrtl,entrypoint='rtlNano';
+END;
+
+
+engine := thorlib.platform();
+prefix := setup.Files(false, false).FilePrefix + '-' + engine + '-';
+suffix := '-' + jlib.rtlTick() : stored('startTime');
+
 boolean sprayFixed := #IFDEFINED(root.sprayFixed, true);
 boolean sprayEmpty := #IFDEFINED(root.sprayEmpty, false);
 
@@ -55,20 +59,20 @@ allPeople := DATASET([ {'foo', 10, 1},
             ,Layout_Person);
 
 #if (sprayFixed)
-    sprayPrepFileName := prefix + 'spray_prep_fixed';
-    desprayOutFileName := dropzonePath + 'spray_input_fixed';
-    sprayOutFileName := prefix + 'spray_test_fixed';
+    sprayPrepFileName := prefix + 'spray_prep_fixed-' + suffix;
+    desprayOutFileName := dropzonePath + 'spray_input_fixed-' + suffix;
+    sprayOutFileName := prefix + 'spray_test_fixed-' + suffix;
     dsSetup := allPeople;
 #else
     #if (sprayEmpty)
-        sprayPrepFileName := prefix + 'spray_prep_empty';
-        desprayOutFileName := dropzonePath + 'spray_input_empty';
-        sprayOutFileName := prefix + 'spray_test_empty';
+        sprayPrepFileName := prefix + 'spray_prep_empty-' + suffix;
+        desprayOutFileName := dropzonePath + 'spray_input_empty-' + suffix;
+        sprayOutFileName := prefix + 'spray_test_empty-' + suffix;
         dsSetup := empty;
     #else
-        sprayPrepFileName := prefix + 'spray_prep';
-        desprayOutFileName := dropzonePath + 'spray_input';
-        sprayOutFileName := prefix + 'spray_test';
+        sprayPrepFileName := prefix + 'spray_prep-' + suffix;
+        desprayOutFileName := dropzonePath + 'spray_input-' + suffix;
+        sprayOutFileName := prefix + 'spray_test-' + suffix;
         dsSetup := allPeople;
     #end
 #end
@@ -94,7 +98,7 @@ rec despray(rec l) := TRANSFORM
 end;
 
 dst1 := NOFOLD(DATASET([{'', ''}], rec));
-p1 := PROJECT(NOFOLD(dst1), despray(LEFT));
+p1 := NOTHOR(PROJECT(NOFOLD(dst1), despray(LEFT)));
 c1 := CATCH(NOFOLD(p1), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Despray Fail',
                                  SELF.msg := FAILMESSAGE
@@ -138,7 +142,7 @@ c1 := CATCH(NOFOLD(p1), ONFAIL(TRANSFORM(rec,
 
 
 dst2 := NOFOLD(DATASET([{'', ''}], rec));
-p2 := PROJECT(NOFOLD(dst2), spray(LEFT));
+p2 := NOTHOR(PROJECT(NOFOLD(dst2), spray(LEFT)));
 c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
                                  SELF.result := 'Spray Fail',
                                  SELF.msg := FAILMESSAGE
@@ -160,8 +164,13 @@ END;
 
 
 SEQUENTIAL(
-  setupFile,
-  desprayOut,
-  sprayOut,
-  output(compareDatasets(dsSetup,ds))
+    setupFile,
+    desprayOut,
+    sprayOut,
+    output(compareDatasets(dsSetup,ds)),
+
+    // Clean-up
+    FileServices.DeleteExternalFile('.', desprayOutFileName),
+    FileServices.DeleteLogicalFile(sprayPrepFileName),
+    FileServices.DeleteLogicalFile(sprayOutFileName)
 );