Merge branch 'candidate-7.4.x' into candidate-7.6.x

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

dali/ft/filecopy.cpp

@@ -1757,7 +1757,21 @@ void FileSprayer::derivePartitionExtra()
 void FileSprayer::displayPartition()
 {
     ForEachItemIn(idx, partition)
+    {
         partition.item(idx).display();
+
+#ifdef _DEBUG
+        if ((partition.item(idx).whichInput >= 0) && (partition.item(idx).whichInput < sources.ordinality()) )
+            LOG(MCdebugInfoDetail, unknownJob,
+                     "   Header size: %" I64F "u, XML header size: %" I64F "u, XML footer size: %" I64F "u",
+                     sources.item(partition.item(idx).whichInput).headerSize,
+                     sources.item(partition.item(idx).whichInput).xmlHeaderLength,
+                     sources.item(partition.item(idx).whichInput).xmlFooterLength
+            );
+        else
+            LOG(MCdebugInfoDetail, unknownJob,"   No source file for this partition");
+#endif
+    }
 }
 
 
@@ -3001,27 +3015,19 @@ void FileSprayer::spray()
 bool FileSprayer::isSameSizeHeaderFooter()
 {
     bool retVal = true;
-    unsigned whichHeaderInput = 0;
-    bool isEmpty = true;
-    headerSize = 0;
-    footerSize = 0;
 
     if (sources.ordinality() == 0)
         return retVal;
 
+    unsigned whichHeaderInput = 0;
+    headerSize = sources.item(whichHeaderInput).xmlHeaderLength;
+    footerSize = sources.item(whichHeaderInput).xmlFooterLength;
+
     ForEachItemIn(idx, partition)
     {
         PartitionPoint & cur = partition.item(idx);
         if (cur.inputLength && (idx+1 == partition.ordinality() || partition.item(idx+1).whichOutput != cur.whichOutput))
         {
-            if (isEmpty)
-            {
-                headerSize = sources.item(whichHeaderInput).xmlHeaderLength;
-                footerSize = sources.item(cur.whichInput).xmlFooterLength;
-                isEmpty = false;
-                continue;
-            }
-
             if (headerSize != sources.item(whichHeaderInput).xmlHeaderLength)
             {
                 retVal = false;
@@ -3037,7 +3043,6 @@ bool FileSprayer::isSameSizeHeaderFooter()
             if ( idx+1 != partition.ordinality() )
                 whichHeaderInput = partition.item(idx+1).whichInput;
         }
-
     }
     return retVal;
 }

dali/ft/filecopy.ipp

@@ -117,7 +117,7 @@ public:
     offset_t                offset;
     offset_t                size;               // expanded size
     offset_t                psize;              // physical (compressed) size
-    unsigned                headerSize;
+    offset_t                headerSize;
     offset_t                xmlHeaderLength;
     offset_t                xmlFooterLength;
     unsigned                crc;

plugins/javaembed/java.ecllib

@@ -27,5 +27,5 @@ EXPORT syntaxCheck := Language.syntaxCheck;
 EXPORT checkImport := Language.checkImport;
 EXPORT boolean supportsImport := true;
 EXPORT boolean supportsScript := true;
-EXPORT boolean threadlocal := true;
+EXPORT boolean threadlocal := false;
 EXPORT boolean singletonEmbedContext := true;

plugins/javaembed/javaembed.cpp

@@ -635,6 +635,9 @@ public:
     inline void DeleteGlobalRef(jobject val)
     {
         JNIEnv::DeleteGlobalRef(val);
+#ifdef FORCE_GC
+        forceGC(this);
+#endif
     }
     inline jobject NewGlobalRef(jobject val, const char *)
     {
@@ -822,6 +825,16 @@ class PersistedObject : public MappingBase
 {
 public:
     PersistedObject(const char *_name) : name(_name) {}
+    ~PersistedObject()
+    {
+        if (instance)
+        {
+#ifdef TRACE_GLOBALREF
+            DBGLOG("DeleteGlobalRef(singleton): %p", instance);
+#endif
+            queryJNIEnv()->DeleteGlobalRef(instance);
+        }
+    }
     CriticalSection crit;
     jobject instance = nullptr;
     StringAttr name;
@@ -1015,14 +1028,6 @@ public:
     void doUnregister(const char *key)
     {
         CriticalBlock b(hashCrit);
-        PersistedObject *p = persistedObjects.find(key);
-        if (p && p->instance)
-        {
-            queryJNIEnv()->DeleteGlobalRef(p->instance);
-#ifdef FORCE_GC
-            forceGC(queryJNIEnv());
-#endif
-        }
         persistedObjects.remove(key);
     }
     static void unregister(const char *key);
@@ -2327,9 +2332,10 @@ public:
     }
     ~JavaThreadContext()
     {
-        // Make sure all thread-local function contexts are destroyed before we detach from
+        // Make sure all thread-local function contexts and saved objects are destroyed before we detach from
         // the Java thread
         contexts.kill();
+        persistedObjects.kill();
         // According to the Java VM 1.7 docs, "A native thread attached to
         // the VM must call DetachCurrentThread() to detach itself before
         // exiting."
@@ -2368,8 +2374,23 @@ public:
         // Note - this object is thread-local so no need for a critsec
         contexts.append(*ctx);
     }
+
+    PersistedObject *getLocalObject(CheckedJNIEnv *JNIenv, const char *name)
+    {
+        // Note - this object is thread-local so no need for a critsec
+        PersistedObject *p;
+        p = persistedObjects.find(name);
+        if (!p)
+        {
+            p = new PersistedObject(name);
+            persistedObjects.replaceOwn(*p);
+        }
+        p->crit.enter();  // needed to keep code common between local/global cases
+        return p;
+    }
 private:
     IArrayOf<IEmbedFunctionContext> contexts;
+    StringMapOf<PersistedObject> persistedObjects = { false };
 };
 
 class JavaXmlBuilder : implements IXmlWriterExt, public CInterface
@@ -3212,12 +3233,6 @@ public:
     }
     ~JavaEmbedImportContext()
     {
-        if (persistMode == persistThread)
-        {
-            StringBuffer scopeKey;
-            getScopeKey(scopeKey);
-            JavaGlobalState::unregister(scopeKey);
-        }
         if (javaClass)
             JNIenv->DeleteGlobalRef(javaClass);
         if (classLoader)
@@ -4110,7 +4125,7 @@ public:
                         StringBuffer scopeKey;
                         getScopeKey(scopeKey);
                         PersistedObjectCriticalBlock persistBlock;
-                        persistBlock.enter(globalState->getGlobalObject(JNIenv, scopeKey));
+                        persistBlock.enter(persistMode==persistThread ? sharedCtx->getLocalObject(JNIenv, scopeKey) : globalState->getGlobalObject(JNIenv, scopeKey));
                         instance = persistBlock.getInstance();
                         if (instance)
                             persistBlock.leave();
@@ -4125,7 +4140,7 @@ public:
                             if (persistMode==persistQuery || persistMode==persistWorkunit || persistMode==persistChannel)
                             {
                                 assertex(engine);
-                                engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode!=persistQuery);
+                                engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode==persistWorkunit);
                             }
                             persistBlock.leave(instance);
                         }
@@ -4377,7 +4392,7 @@ protected:
                     // If a persist scope is specified, we may want to use a pre-existing object. If we do we share its classloader, class, etc.
                     assertex(classname.length());  // MORE - what does this imply?
                     getScopeKey(scopeKey);
-                    persistBlock.enter(globalState->getGlobalObject(JNIenv, scopeKey));
+                    persistBlock.enter(persistMode==persistThread ? sharedCtx->getLocalObject(JNIenv, scopeKey) : globalState->getGlobalObject(JNIenv, scopeKey));
                     instance = persistBlock.getInstance();
                     if (instance)
                         persistBlock.leave();
@@ -4421,7 +4436,7 @@ protected:
                         if (persistMode==persistQuery || persistMode==persistWorkunit || persistMode==persistChannel)
                         {
                             assertex(engine);
-                            engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode!=persistQuery);
+                            engine->onTermination(JavaGlobalState::unregister, scopeKey.str(), persistMode==persistWorkunit);
                         }
                         persistBlock.leave(instance);
                     }
@@ -4510,12 +4525,12 @@ protected:
         case persistGlobal:
             ret.append("global");
             break;
-        case persistChannel:
-            ret.append(nodeNum).append('.');
             // Fall into
         case persistWorkunit:
             engine->getQueryId(ret, true);
             break;
+        case persistChannel:
+            ret.append(nodeNum).append('.');
         case persistQuery:
             engine->getQueryId(ret, false);
             break;

roxie/ccd/ccdcontext.cpp

@@ -3136,7 +3136,11 @@ public:
     virtual StringBuffer &getQueryId(StringBuffer &result, bool isShared) const
     {
         if (workUnit)
-            result.append(workUnit->queryWuid()); // In workunit mode, this works for both shared and non-shared variants
+        {
+            if (isShared)
+                result.append('Q');
+            result.append(workUnit->queryWuid());
+        }
         else if (isShared)
             result.append('Q').append(factory->queryHash());
         else

testing/regress/ecl/hthor/javascope.xml

@@ -0,0 +1,90 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>: parallel</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>: sequential</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>thread: parallel</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>channel: sequential</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>query: sequential</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>workunit: sequential</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><a>37</a></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><a>39</a></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><a>42</a></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><a>124</a></Row>
+</Dataset>

testing/regress/ecl/javascope.ecl

@@ -0,0 +1,110 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+IMPORT Java;
+
+//version forceNonThread=false
+//version forceNonThread=true
+
+
+import ^ as root;
+forceNonThread := #IFDEFINED(root.forceNonThread, false);
+
+// Check that implicitly-created objects have appropriate lifetime
+
+implicit(STRING p, STRING s) := MODULE
+  STRING st := ''
+#if (forceNonThread)
+   : STORED('st')
+#end
+  ;
+
+  EXPORT INTEGER accumulate(INTEGER b) := EMBED(Java : PERSIST(p), GLOBALSCOPE(s+st))
+    class x
+    {
+      public x()
+      {
+        synchronized (x.class)
+        { 
+          idx = nextIdx;
+          nextIdx = nextIdx+1;
+        }
+//        System.out.println("created  " + idx + x.class.getName());
+      }
+      public void finalize()
+      {
+//        System.out.println("finalize " + n + " " + idx);
+      }
+      public synchronized int accumulate(int b)
+      {
+        n = n + b;
+        return n;
+      }
+      int n = 0;
+      int idx = 0;
+      static int nextIdx = 0;
+    }
+  ENDEMBED;
+
+  SHARED r := RECORD
+    integer a; 
+  END;
+
+
+  // The parallel test runs all on separate threads (except the last two calls) to ensure that separate threads
+  // are independent when using PERSIST('thread') or PERSIST('none')
+
+  EXPORT ptest := PARALLEL (
+    output(p + ': parallel');
+    output(project(nofold(dataset([{1}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{2}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{3}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{10}], r)), transform(r, self.a := accumulate(LEFT.a)+accumulate(LEFT.a*2))));
+  );
+
+  // The sequential test runs sequentially for ones that are supposed to interact across threads (otherwise results are indeterminate)
+
+  EXPORT stest := ORDERED (
+    output(p + ': sequential');
+    output(project(nofold(dataset([{1}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{2}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{3}], r)), transform(r, self.a := accumulate(LEFT.a))));
+    output(project(nofold(dataset([{10}], r)), transform(r, self.a := accumulate(LEFT.a)+accumulate(LEFT.a*2))));
+  );
+
+END;
+
+gc() := EMBED(Java)
+  public static void gc()
+  {
+    System.gc();
+  }
+ENDEMBED;
+
+ORDERED (
+  implicit('','').ptest;
+  implicit('','').stest;
+  implicit('thread','').ptest;
+  implicit('channel','').stest;
+  implicit('query','').stest;
+  implicit('workunit','').stest;
+//  implicit('global','').stest;
+//  gc();
+);
+
+// Check that explicitly-created objects have appropriate lifetime (how?) but are not shared
+

testing/regress/ecl/key/javascope.xml

@@ -0,0 +1,90 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>: parallel</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>: sequential</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>thread: parallel</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><a>40</a></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>channel: sequential</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>query: sequential</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>workunit: sequential</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><a>52</a></Row>
+</Dataset>

testing/regress/ecl/key/spray_test_xml.xml

@@ -0,0 +1,23 @@
+<Dataset name='Result 1'>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><result>Despray Pass</result></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><result>Spray Pass</result></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><Result_4>Compare: Pass</Result_4></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><Result_5>Prep file header length: OK</Result_5></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>Prep file footer length: OK</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><Result_7>Spray file header length: OK</Result_7></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><Result_8>Spray file footer length: OK</Result_8></Row>
+</Dataset>

testing/regress/ecl/spray_test_xml.ecl

@@ -0,0 +1,185 @@
+/*##############################################################################
+
+    Copyright (C) 2019 HPCC Systems®.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//nohthor
+//noroxie
+
+//class=spray
+
+//version xml_header='none',xml_footer='none'
+//version xml_header='none',xml_footer='short'
+//version xml_header='none',xml_footer='long'
+//version xml_header='short',xml_footer='none'
+//version xml_header='short',xml_footer='short'
+//version xml_header='short',xml_footer='long'
+//version xml_header='long',xml_footer='none'
+//version xml_header='long',xml_footer='short'
+//version xml_header='long',xml_footer='long'
+
+import Std.File AS FileServices;
+import $.setup;
+import ^ as root;
+
+prefix := setup.Files(false, false).QueryFilePrefix;
+
+dropzonePath := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+
+espUrl := FileServices.GetEspURL() + '/FileSpray';
+
+unsigned VERBOSE := 0;
+unsigned CLEANUP := 1;
+
+string xml_header := #IFDEFINED(root.xml_header, 'short');
+#if (xml_header = 'none')
+    string header := '';
+#elseif (xml_header = 'short')
+    string header := '<Header>Head</Header>\n';
+#else
+    string header := '<Header>Head 123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890</Header>\n';
+#end
+unsigned expectedHeaderLength := LENGTH(header);
+
+string xml_footer := #IFDEFINED(root.xml_footer, 'none');
+
+#if (xml_footer = 'none')
+    string footer := '';
+#elseif (xml_footer = 'short')
+    string footer := '<Footer>Foot</Footer>';
+#else
+    string footer := '<Footer>Foot 123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890</Footer>';
+#end
+unsigned expectedFooterLength := LENGTH(footer);
+
+Layout_Person := RECORD
+  STRING3  name;
+  UNSIGNED2 age;
+  BOOLEAN good;
+END;
+
+
+allPeople := DATASET([ {'foo', 10, 1},
+                       {'bar', 12, 0},
+                       {'baz', 32, 1}]
+            ,Layout_Person);
+
+setupXmlPrepFileName := prefix + 'original_xml';
+desprayXmlOutFileName := dropzonePath + prefix + '-desprayed_xml';
+sprayXmlTargetFileName := prefix + 'sprayed_xml';
+dsSetup := DISTRIBUTE(allPeople);
+
+//  Create a small logical file
+setupXmlFile := output(dsSetup, , setupXmlPrepFileName, XML( 'Row', HEADING(header , footer), NOROOT), OVERWRITE);
+
+rec := RECORD
+  string result;
+  string msg;
+end;
+
+
+// Despray it to default drop zone
+rec despray(rec l) := TRANSFORM
+  SELF.msg := FileServices.fDespray(
+                       LOGICALNAME := setupXmlPrepFileName
+                      ,DESTINATIONIP := '.'
+                      ,DESTINATIONPATH := desprayXmlOutFileName
+                      ,ALLOWOVERWRITE := True
+                      );
+  SELF.result := 'Despray Pass';
+end;
+
+dst1 := NOFOLD(DATASET([{'', ''}], rec));
+p1 := NOTHOR(PROJECT(NOFOLD(dst1), despray(LEFT)));
+c1 := CATCH(NOFOLD(p1), ONFAIL(TRANSFORM(rec,
+                                 SELF.result := 'Despray Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+#if (VERBOSE = 1)
+    desprayOut := output(c1);
+#else
+    desprayOut := output(c1, {result});
+#end
+
+
+
+rec spray(rec l) := TRANSFORM
+    SELF.msg := FileServices.fSprayXml(
+                        SOURCEIP := '.',
+                        SOURCEPATH := desprayXmlOutFileName,
+                        SOURCEROWTAG := 'Row',
+                        DESTINATIONGROUP := 'mythor',
+                        DESTINATIONLOGICALNAME := sprayXmlTargetFileName,
+                        TIMEOUT := -1,
+                        ESPSERVERIPPORT := espUrl,
+                        ALLOWOVERWRITE := true
+                        );
+    self.result := 'Spray Pass';
+end;
+
+
+dst2 := NOFOLD(DATASET([{'', ''}], rec));
+p2 := NOTHOR(PROJECT(NOFOLD(dst2), spray(LEFT)));
+c2 := CATCH(NOFOLD(p2), ONFAIL(TRANSFORM(rec,
+                                 SELF.result := 'Spray Fail',
+                                 SELF.msg := FAILMESSAGE
+                                )));
+#if (VERBOSE = 1)
+    sprayOut := output(c2);
+#else
+    sprayOut := output(c2, {result});
+#end
+
+ds := DATASET(sprayXmlTargetFileName, Layout_Person, XML('Row', NOROOT));
+
+string compareDatasets(dataset(Layout_Person) ds1, dataset(Layout_Person) ds2) := FUNCTION
+   c := COUNT(JOIN(ds1, ds2, left.name=right.name, FULL ONLY));
+   boolean result := (0 = c);
+   #if (VERBOSE = 1)
+    retVal := 'Compare: ' + if(result, 'Pass', 'Fail') + ', Count = ' + intformat(c, 3, 0);
+   #else
+    retVal := 'Compare: ' + if(result, 'Pass', 'Fail');
+   #end
+   RETURN retVal;
+END;
+
+string checkFileHeaderFooterLen(string prefix, string fileName, unsigned expectedLen, boolean isHeader) := FUNCTION
+    len := (INTEGER) if( isHeader, fileservices.GetLogicalFileAttribute(setupXmlPrepFileName, 'headerLength'), fileservices.GetLogicalFileAttribute(setupXmlPrepFileName, 'footerLength'));
+    return ( prefix + ' file ' + if( isHeader, 'header', 'footer')  + ' length: ' +  if (len = expectedLen, 'OK', 'Bad ' + intformat(len, 3, 0) + '/' + intformat(expectedLen, 3, 0)));
+end;
+
+SEQUENTIAL(
+    setupXmlFile,
+    desprayOut,
+    sprayOut,
+    output(compareDatasets(dsSetup,ds)),
+    output(checkFileHeaderFooterLen('Prep', setupXmlPrepFileName, expectedHeaderLength, TRUE));
+    output(checkFileHeaderFooterLen('Prep', setupXmlPrepFileName, expectedFooterLength, FALSE));
+    output(checkFileHeaderFooterLen('Spray', sprayXmlTargetFileName, expectedHeaderLength, TRUE));
+    output(checkFileHeaderFooterLen('Spray', sprayXmlTargetFileName, expectedFooterLength, FALSE));
+
+#if (VERBOSE = 1)
+    output(dsSetup, NAMED('dsSetup')),
+    output(ds, NAMED('ds')),
+    output(JOIN(dsSetup, ds, left.name=right.name, FULL ONLY, LOCAL)),
+#end
+
+#if (CLEANUP = 1)
+    // Clean-up
+    FileServices.DeleteExternalFile('.', desprayXmlOutFileName),
+    FileServices.DeleteLogicalFile(setupXmlPrepFileName),
+    FileServices.DeleteLogicalFile(sprayXmlTargetFileName)
+#end
+);

testing/regress/ecl/thor/javascope.xml

@@ -0,0 +1,90 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>: parallel</Result_1></Row>
+</Dataset>
+<Dataset name='Result 2'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 6'>
+ <Row><Result_6>: sequential</Result_6></Row>
+</Dataset>
+<Dataset name='Result 7'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 8'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 9'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 10'>
+ <Row><a>30</a></Row>
+</Dataset>
+<Dataset name='Result 11'>
+ <Row><Result_11>thread: parallel</Result_11></Row>
+</Dataset>
+<Dataset name='Result 12'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 13'>
+ <Row><a>2</a></Row>
+</Dataset>
+<Dataset name='Result 14'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 15'>
+ <Row><a>40</a></Row>
+</Dataset>
+<Dataset name='Result 16'>
+ <Row><Result_16>channel: sequential</Result_16></Row>
+</Dataset>
+<Dataset name='Result 17'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 18'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 19'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 20'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 21'>
+ <Row><Result_21>query: sequential</Result_21></Row>
+</Dataset>
+<Dataset name='Result 22'>
+ <Row><a>1</a></Row>
+</Dataset>
+<Dataset name='Result 23'>
+ <Row><a>3</a></Row>
+</Dataset>
+<Dataset name='Result 24'>
+ <Row><a>6</a></Row>
+</Dataset>
+<Dataset name='Result 25'>
+ <Row><a>52</a></Row>
+</Dataset>
+<Dataset name='Result 26'>
+ <Row><Result_26>workunit: sequential</Result_26></Row>
+</Dataset>
+<Dataset name='Result 27'>
+ <Row><a>37</a></Row>
+</Dataset>
+<Dataset name='Result 28'>
+ <Row><a>39</a></Row>
+</Dataset>
+<Dataset name='Result 29'>
+ <Row><a>42</a></Row>
+</Dataset>
+<Dataset name='Result 30'>
+ <Row><a>124</a></Row>
+</Dataset>