Merge branch 'candidate-6.4.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

common/workunit/workunit.cpp

@@ -2696,13 +2696,23 @@ void CWorkUnitFactory::reportAbnormalTermination(const char *wuid, WUState &stat
 }
 
 static CriticalSection deleteDllLock;
-static Owned<IWorkQueueThread> deleteDllWorkQ;
-
+static IWorkQueueThread *deleteDllWorkQ = nullptr;
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    return true;
+}
+MODULE_EXIT()
+{
+    CriticalBlock b(deleteDllLock);
+    if (deleteDllWorkQ)
+        ::Release(deleteDllWorkQ);
+    deleteDllWorkQ = nullptr;
+}
 static void asyncRemoveDll(const char * name)
 {
     CriticalBlock b(deleteDllLock);
     if (!deleteDllWorkQ)
-        deleteDllWorkQ.setown(createWorkQueueThread());
+        deleteDllWorkQ = createWorkQueueThread();
     deleteDllWorkQ->post(new asyncRemoveDllWorkItem(name));
 }
 
@@ -2710,7 +2720,7 @@ static void asyncRemoveFile(const char * ip, const char * name)
 {
     CriticalBlock b(deleteDllLock);
     if (!deleteDllWorkQ)
-        deleteDllWorkQ.setown(createWorkQueueThread());
+        deleteDllWorkQ = createWorkQueueThread();
     deleteDllWorkQ->post(new asyncRemoveRemoteFileWorkItem(ip, name));
 }
 

ecl/hql/hqlir.cpp

@@ -858,6 +858,7 @@ public:
     virtual typeid_t addSimpleType(type_t tc, const SimpleTypeBuilderInfo & info) = 0;
     virtual typeid_t addExprType(type_t tc, exprid_t expr) = 0;
     virtual typeid_t addCompoundType(type_t tc, const CompoundTypeBuilderInfo & info) = 0;
+    virtual typeid_t addClassType(const char * name) = 0;
     virtual typeid_t addUnknownType(type_t tc) = 0;
     virtual typeid_t addTypeAnnotation(typemod_t kind, const TypeAnnotationBuilderInfo & info) = 0;
 
@@ -1180,6 +1181,17 @@ public:
         return def.id;
     }
 
+    virtual typeid_t addClassType(const char * name)
+    {
+        Definition def("t", nextId(), false);
+
+        startDefinition(def, "type");
+        line.append("class:").append(name);
+        finishDefinition(def);
+
+        return def.id;
+    }
+
 
     virtual typeid_t addUnknownType(type_t tc)
     {
@@ -1698,6 +1710,10 @@ public:
     {
         return 0;
     }
+    virtual typeid_t addClassType(const char * name)
+    {
+        return 0;
+    }
     virtual typeid_t addUnknownType(type_t tc)
     {
         return saveItem(makeNullType());
@@ -1896,10 +1912,11 @@ id_t ExpressionIRPlayer::doProcessType(ITypeInfo * type)
         case type_ifblock:
         case type_alias:
         case type_blob:
-        case type_class:
         case type_array:
             throwUnexpected();
             break;
+        case type_class:
+            return target->addClassType(type->queryTypeName());
         default:
             UNIMPLEMENTED;
         }

esp/scm/ws_esdlconfig.ecm

@@ -26,6 +26,7 @@ ESPStruct MethodConfig
     string Name;
     ESParray<ESPstruct NamedValue, Attribute> Attributes;
     string Elements;
+    string XML;
 };
 
 ESPStruct ESDLConfiguration

esp/services/ws_esdlconfig/ws_esdlconfigservice.cpp

@@ -1393,12 +1393,17 @@ bool CWsESDLConfigEx::onGetESDLBinding(IEspContext &context, IEspGetESDLBindingR
                         }
                     }
 
+                    StringBuffer methodconfigxml;
                     Owned<IPropertyTreeIterator> iter = esdlbindingtree->getElements("Definition[1]/Methods/Method");
                     ForEach(*iter)
                     {
                         Owned<IEspMethodConfig> methodconfig = createMethodConfig("","");
 
                         IPropertyTree & cur = iter->query();
+                        toXML(&cur, methodconfigxml.clear());
+                        if (methodconfigxml.length())
+                            methodconfig->setXML(methodconfigxml);
+
                         IArrayOf<IEspNamedValue> iespattributes;
                         Owned<IAttributeIterator> attributes = cur.getAttributes();
                         ForEach(*attributes)

plugins/cassandra/cassandrawu.cpp

@@ -2697,6 +2697,10 @@ public:
         CriticalBlock b(crit);
         for (const ChildTableInfo * const * table = childTables; *table != NULL; table++)
             checkChildLoaded(**table);
+        // And a hack for the fact that Dali stores state in both @state and <state>
+        const char *stateStr = p->queryProp("@state");
+        if (stateStr)
+            p->setProp("State", stateStr);
         return p;
     }
 

plugins/javaembed/javaembed.cpp

@@ -165,6 +165,7 @@ public:
         vm_args.options = options;
         vm_args.ignoreUnrecognized = true;
         vm_args.version = JNI_VERSION_1_6;
+
         /* load and initialize a Java VM, return a JNI interface pointer in env */
         JNIEnv *env;       /* receives pointer to native method interface */
         int createResult = JNI_CreateJavaVM(&javaVM, (void**)&env, &vm_args);
@@ -1554,6 +1555,22 @@ public:
         }
     }
 
+    bool checkException(StringBuffer &message)
+    {
+        if (JNIenv->ExceptionCheck())
+        {
+            jthrowable exception = JNIenv->ExceptionOccurred();
+            JNIenv->ExceptionClear();
+            jclass throwableClass = JNIenv->FindClass("java/lang/Throwable");
+            jmethodID throwableToString = JNIenv->GetMethodID(throwableClass, "toString", "()Ljava/lang/String;");
+            jstring cause = (jstring) JNIenv->CallObjectMethod(exception, throwableToString);
+            const char *text = JNIenv->GetStringUTFChars(cause, 0);
+            message.append(text);
+            JNIenv->ReleaseStringUTFChars(cause, text);
+            return true;
+        }
+        return false;
+    }
     jobject getSystemClassLoader()
     {
         JNIenv->ExceptionClear();
@@ -1698,16 +1715,18 @@ public:
             jobject classLoader = getThreadClassLoader();
             jmethodID loadClassMethod = JNIenv->GetMethodID(JNIenv->GetObjectClass(classLoader), "loadClass","(Ljava/lang/String;)Ljava/lang/Class;");
             jstring methodString = JNIenv->NewStringUTF(classname);
-            javaClass = (jclass) JNIenv->NewGlobalRef(JNIenv->CallObjectMethod(classLoader, loadClassMethod, methodString));
+            javaClass = (jclass) JNIenv->CallObjectMethod(classLoader, loadClassMethod, methodString);
+            StringBuffer message;
+            if (checkException(message) || !javaClass)
+                throw MakeStringException(MSGAUD_user, 0, "javaembed: Failed to resolve class name %s: %s", classname.str(), message.str());
+            javaClass = (jclass) JNIenv->NewGlobalRef(javaClass);
 
-            if (!javaClass)
-                throw MakeStringException(MSGAUD_user, 0, "javaembed: Failed to resolve class name %s", classname.str());
             if (instance)
                 javaMethodID = JNIenv->GetMethodID(javaClass, methodname, javaSignature);
             else
                 javaMethodID = JNIenv->GetStaticMethodID(javaClass, methodname, javaSignature);
-            if (!javaMethodID)
-                throw MakeStringException(MSGAUD_user, 0, "javaembed: Failed to resolve method name %s with signature %s", methodname.str(), signature);
+            if (checkException(message) || !javaMethodID)
+                throw MakeStringException(MSGAUD_user, 0, "javaembed: Failed to resolve method name %s with signature %s: %s", methodname.str(), signature, message.str());
             const char *returnSig = strrchr(signature, ')');
             assertex(returnSig);  // Otherwise how did Java accept it??
             returnSig++;

roxie/ccd/ccdserver.cpp

@@ -2699,7 +2699,7 @@ public:
 
     virtual void stopSink(unsigned outputIdx)
     {
-        if (!stopped[outputIdx])
+        if (outputIdx < numOutputs && !stopped[outputIdx])  // Implicit dependencies on DiskWrite activities do not count as outputs
         {
             stopped[outputIdx] = true;
             for (unsigned s = 0; s < numOutputs; s++)

thorlcr/activities/wuidread/thwuidreadslave.cpp

@@ -101,7 +101,7 @@ public:
     {
         initMetaInfo(info);
         info.isSource = true;
-        if (firstNode())
+        if (container.queryLocal() || firstNode())
             info.unknownRowsOutput = true;
         else
         {

thorlcr/graph/thgraphmaster.cpp

@@ -2828,7 +2828,7 @@ void CThorStats::processInfo()
     ForEachItemIn(n, counts)
     {
         unsigned __int64 thiscount = counts.item(n);
-        tallyValue(thiscount, n);
+        tallyValue(thiscount, n+1);
     }
     calculateSkew();
 }
@@ -2880,7 +2880,7 @@ void ProgressInfo::processInfo() // reimplement as counts have special flags (i.
         if (thiscount & THORDATALINK_STOPPED)
             stopcount++;
         thiscount = thiscount & THORDATALINK_COUNT_MASK;
-        tallyValue(thiscount, n);
+        tallyValue(thiscount, n+1);
     }
     calculateSkew();
 }

tools/wutool/wutool.cpp

@@ -478,6 +478,7 @@ class WuTool : public CppUnit::TestFixture
         CPPUNIT_TEST(testListByAppValueWild);
         CPPUNIT_TEST(testListByFilesRead);
         CPPUNIT_TEST(testListByFilesWritten);
+        CPPUNIT_TEST(testSortByThorTime);
         CPPUNIT_TEST(testSet);
         CPPUNIT_TEST(testResults);
         CPPUNIT_TEST(testWorkUnitServices);
@@ -487,7 +488,6 @@ class WuTool : public CppUnit::TestFixture
         CPPUNIT_TEST(testGraph);
         CPPUNIT_TEST(testGraphProgress);
         CPPUNIT_TEST(testGlobal); 
-        CPPUNIT_TEST(testSortByThorTime);
     CPPUNIT_TEST_SUITE_END();
 protected:
     static StringArray wuids;
@@ -697,6 +697,7 @@ protected:
                 "    AQAAAAAAAAA=   </Value>"
                 "  </Result>"
                 " </Results>"
+                " <State>completed</State>"
                 " <Statistics>"
                 "  <Statistic c='eclcc'"
                 "             count='1'"
@@ -786,7 +787,8 @@ protected:
         // This is complicated by the fact that the order is not preserved for statistics
         sortStatistics(xml2);
         sortStatistics(xml3);
-
+        if (wu->getState() != WUStateCompleted)
+            DBGLOG("Not completed");
         DBGLOG("Comparing xml2 and xml3");
         checkStringsMatch(xml2, xml3);
 
@@ -809,6 +811,11 @@ protected:
         p1->removeProp("Variables/Variable[@name='one']/rowCount");
         p1->removeProp("Variables/Variable[@name='one']/totalRowCount");
         p1->removeProp("Variables/Variable[@name='one']/Value");
+        // Checking that results were reset by the copy
+        p1->setProp("Results/Result[@name='Result 1']/@status", "undefined");
+        p1->removeProp("Results/Result[@name='Result 1']/rowCount");
+        p1->removeProp("Results/Result[@name='Result 1']/totalRowCount");
+        p1->removeProp("Results/Result[@name='Result 1']/Value");
         // Checking that workflow was reset by the copy
         p1->setProp("Workflow/Item[@wfid='1']/@state", "null");
         p1->setProp("Workflow/Item[@wfid='2']/@state", "null");
@@ -833,7 +840,7 @@ protected:
             query->setQueryName("qname");
             query->setQueryMainDefinition("fred");
             query->setQueryType(QueryTypeEcl);
-            query->addAssociatedFile(FileTypeCpp, "myfile", "1.2.3.4", "Description", 53, 3, 4);
+            query->addAssociatedFile(FileTypeCpp, "myfile", "nosuchurl.co.ggg", "Description", 53, 3, 4);
             createWu->setState(WUStateCompleted);
             createWu.clear();
         }
@@ -855,7 +862,7 @@ protected:
         ASSERT(file->getType()==FileTypeCpp);
         ASSERT(file->getMinActivityId()==3);
         ASSERT(file->getMaxActivityId()==4);
-        ASSERT(streq(file->getIp(s).str(), "1.2.3.4"));
+        ASSERT(streq(file->getIp(s).str(), "nosuchurl.co.ggg"));
         query.clear();
         wu.clear();
 
@@ -1665,7 +1672,7 @@ protected:
             numIterated++;
         }
         DBGLOG("%d workunits by totalThorTime in %d ms", numIterated, msTick()-start);
-        ASSERT(numIterated == (testSize+9)/10);
+        ASSERT(numIterated == testSize);
         numIterated++;
     }
     void testGlobal()
@@ -1804,6 +1811,8 @@ protected:
             virtual ISectionTimer * registerTimer(unsigned activityId, const char * name) { throwUnexpected(); }
         } ctx;
 
+ #if 0
+        // These tests are not valid at present, since the C++ structure does not accurately describe the variable layout fields
         size32_t lenResult;
         void * result;
         wsWorkunitList(&ctx, lenResult, result, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, true, false, NULL);
@@ -1857,6 +1866,7 @@ protected:
         ASSERT(numResults <= (testSize+49)/50);  // Not sure what the exact answer should be!
 
         rtlFree(result);
+#endif
     }
 };
 StringArray WuTool::wuids;