Merge remote-tracking branch 'origin/closedown-4.0.x' into candidate-4.0.0

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

ecl/hql/hql.hpp

@@ -47,9 +47,9 @@ Relevant changes include
 
 */
 
-#define LANGUAGE_VERSION_MAJOR      3
-#define LANGUAGE_VERSION_MINOR      6
-#define LANGUAGE_VERSION_SUB        1
+#define LANGUAGE_VERSION_MAJOR      4
+#define LANGUAGE_VERSION_MINOR      0
+#define LANGUAGE_VERSION_SUB        0
 
 #define LANGUAGE_VERSION   estringify(LANGUAGE_VERSION_MAJOR) "." estringify(LANGUAGE_VERSION_MINOR) "." estringify(LANGUAGE_VERSION_SUB)
 

esp/files/scripts/configmgr/configmgr.js

@@ -1060,9 +1060,10 @@ function handleConfigCellClickEvent(oArgs, caller, isComplex) {
           }
           var indexOfSubType = newParams.indexOf("::subType");
           var newParamsSubString = newParams.substr(indexOfSubType);
+          var indexOfName = newParamsSubString.indexof("@name")+7;
 
           top.document.selectRecordClick = true;
-          top.document.selectRecord = newParamsSubString.substr(newParamsSubString.indexOf("@name")+7).substring(0,newParamsSubString.substr(newParamsSubString.indexOf("@name")+7).indexOf("'"));
+          top.document.selectRecord = newParamsSubString.substr(indexOfName).substring(0,newParamsSubString.substr(indexOfName).indexOf("'"));
           top.document.navDT.clickCurrentSelOrName(top.document.navDT); // refresh
         } else {
           alert(r.replyText);
@@ -2912,7 +2913,39 @@ function onMenuItemClickGenericAddDelete(p_sType, p_aArgs, p_oValue) {
     return;
   }
 
-  if (dt.parentDT && dt.parentDT.getRecord(dt.parentDT.getSelectedRows()[0]) != null)
+  if (parentName == "Base Access Control Lists (Ordered List)")
+  {
+    var xmlArgs = "<XmlArgs ";
+    if ( this.cfg.getProperty("text") == "Add" )
+      xmlArgs += "XPath = \'Software/RoxieCluster[@name=\"" + top.document.navDT.getRecord(top.document.navDT.getSelectedRows()[0]).getData('Name') + "\"]/ACL[@name=\"" + dt.parentDT.getRecord(dt.parentDT.getSelectedRows()[0]).getData('name') + "\"]\'";
+    else
+    {
+      record = dt.getRecord(dt.getRecordIndex(dt.getSelectedRows()[0]));
+      record = record.getData('params').substring(record.getData('params').indexOf('subTypeKey=[@name=')+19);
+      record = record.substring(0,record.indexOf('\''));
+
+      xmlArgs += "XPath = \'Software/RoxieCluster[@name=\"" + top.document.navDT.getRecord(top.document.navDT.getSelectedRows()[0]).getData('Name') + "\"]/ACL[@name=\"" + dt.parentDT.getRecord(dt.parentDT.getSelectedRows()[0]).getData('name') + "\"]/BaseList[@name=\"" + record + "\"]\'";
+    }
+    xmlArgs += " />";
+
+    YAHOO.util.Connect.asyncRequest('POST', '/WsDeploy/HandleBaseAccessControlList', {
+      success: function(o)
+      {
+        top.document.selectRecord = dt.parentDT.getRecord((dt.parentDT.getSelectedRows()[0])).getData('name');
+        top.document.selectRecordClick = true;
+        top.document.navDT.clickCurrentSelOrName(top.document.navDT);
+      },
+      failure: function(o) {
+        top.document.stopWait(document);
+          alert(o.statusText);
+      },
+      scope: this
+    },
+    top.document.navDT.getFileName(true) + 'Operation=' + this.cfg.getProperty("text") + '&XmlArgs=' + xmlArgs);
+    return;
+  }
+
+ if (dt.parentDT && dt.parentDT.getRecord(dt.parentDT.getSelectedRows()[0]) != null)
     type = dt.parentDT.getRecord(dt.parentDT.getSelectedRows()[0]).getData('name');
 
   var subRecs = null;

esp/scm/WsDeploy.ecm

@@ -377,11 +377,24 @@ ESPrequest HandleAccessRulesRequest
     ESPstruct WsDeployReqInfo ReqInfo;
 };
 
+ESPrequest HandleBaseAccessControlListRequest
+{
+    string Operation;
+    string XmlArgs;
+    ESPstruct WsDeployReqInfo ReqInfo;
+};
+
+
 ESPresponse [exceptions_inline, encode(0)] HandleAccessRulesResponse
 {
     string Status;
 };
 
+ESPresponse [exceptions_inline, encode(0)] HandleBaseAccessControlListResponse
+{
+    string Status;
+};
+
 ESPrequest GetNavTreeDefnRequest
 {
     string XmlArgs;
@@ -608,6 +621,8 @@ ESPservice [exceptions_inline("xslt/exceptions.xslt")] WsDeploy
         HandleRows(HandleRowsRequest, HandleRowsResponse);
     ESPmethod[description("Handle Access Rules Requests suchs as add & delete e.t.c"), help("")]
         HandleAccessRules(HandleAccessRulesRequest, HandleAccessRulesResponse);
+    ESPmethod[description("Handle Base Access Control List Requests suchs as add & delete e.t.c"), help("")]
+        HandleBaseAccessControlList(HandleBaseAccessControlListRequest, HandleBaseAccessControlListResponse);
     ESPmethod[description("Return nav tree definition for display purposes"), help("")]
         GetNavTreeDefn(GetNavTreeDefnRequest, GetNavTreeDefnResponse);
     ESPmethod[description("Returns the value for the requested field"), help("")]       

esp/services/WsDeploy/WsDeployService.cpp

@@ -5155,6 +5155,42 @@ bool CWsDeployFileInfo::handleThorTopology(IEspContext &context, IEspHandleThorT
   return true;
 }
 
+bool CWsDeployFileInfo::handleBaseAccessControlList(IEspContext &context, IEspHandleBaseAccessControlListRequest &req, IEspHandleBaseAccessControlListResponse &resp)
+{
+  synchronized block(m_mutex);
+  checkForRefresh(context, &req.getReqInfo(), true);
+
+  StringBuffer buildSetPath;
+
+  IPropertyTree *pEnvRoot = getEnvTree(context,&req.getReqInfo());
+  const char* xmlArg = req.getXmlArgs();
+  const char* operation = req.getOperation();
+
+  Owned<IPropertyTreeIterator> buildSetIter = pEnvRoot->getElements("./Programs/Build/BuildSet[@name=\"roxie\"]");
+
+  buildSetIter->first();
+  IPropertyTree* pBuildSet = &buildSetIter->query();
+  Owned<IPropertyTree> pParams = createPTreeFromXMLString(xmlArg && *xmlArg ? xmlArg : "<XmlArgs/>");
+
+  const char* buildSetName = pBuildSet->queryProp(XML_ATTR_NAME);
+  const char* processName = pBuildSet->queryProp(XML_ATTR_PROCESS_NAME);
+  const char* xpath = pParams->queryProp("@XPath");
+
+  Owned<IPropertyTree> pSchema = loadSchema(pEnvRoot->queryPropTree("./Programs/Build[1]"), pBuildSet, buildSetPath, m_Environment);
+  Owned<IPropertyTree> pNewCompTree = generateTreeFromXsd(pEnvRoot, pSchema, processName, buildSetName, m_pService->getCfg(), m_pService->getName());
+
+  if (stricmp("Add",operation) == 0)
+  {
+    IPropertyTree* pTempTree = createPTreeFromIPT(pNewCompTree->queryPropTree("BaseList"));
+    (pEnvRoot->queryPropTree(xpath))->addPropTree("BaseList",pTempTree);
+  }
+  else if (stricmp("Delete", operation) == 0)
+    pEnvRoot->removeProp(xpath);
+
+  return true;
+}
+
+
 bool CWsDeployFileInfo::handleAccessRules(IEspContext &context, IEspHandleAccessRulesRequest &req, IEspHandleAccessRulesResponse &resp)
 {
   synchronized block(m_mutex);
@@ -6926,6 +6962,11 @@ bool CWsDeployEx::onHandleAccessRules(IEspContext &context, IEspHandleAccessRule
   return fi->handleAccessRules(context, req, resp);
 }
 
+bool CWsDeployEx::onHandleBaseAccessControlList(IEspContext &context, IEspHandleBaseAccessControlListRequest &req, IEspHandleBaseAccessControlListResponse &resp)
+{
+  CWsDeployFileInfo* fi = getFileInfo(req.getReqInfo().getFileName());
+  return fi->handleBaseAccessControlList(context, req, resp);
+}
 
 bool CWsDeployEx::onGraph(IEspContext &context, IEspEmptyRequest& req, IEspGraphResponse& resp)
 {
@@ -7182,6 +7223,11 @@ bool CWsDeployExCE::onHandleRows(IEspContext &context, IEspHandleRowsRequest &re
   return supportedInEEOnly();
 }
 
+bool CWsDeployExCE::onHandleBaseAccessControlList(IEspContext &context, IEspHandleBaseAccessControlListRequest &req, IEspHandleBaseAccessControlListResponse &resp)
+{
+  return supportedInEEOnly();
+}
+
 bool CWsDeployExCE::onHandleAccessRules(IEspContext &context, IEspHandleAccessRulesRequest &req, IEspHandleAccessRulesResponse &resp)
 {
   return supportedInEEOnly();

esp/services/WsDeploy/WsDeployService.hpp

@@ -483,6 +483,7 @@ public:
     virtual bool handleTopology(IEspContext &context, IEspHandleTopologyRequest &req, IEspHandleTopologyResponse &resp);
     virtual bool handleRows(IEspContext &context, IEspHandleRowsRequest &req, IEspHandleRowsResponse &resp);
     virtual bool handleAccessRules(IEspContext &context, IEspHandleAccessRulesRequest &req, IEspHandleAccessRulesResponse &resp);
+    virtual bool handleBaseAccessControlList(IEspContext &context, IEspHandleBaseAccessControlListRequest &req, IEspHandleBaseAccessControlListResponse &resp);
     virtual bool getNavTreeDefn(IEspContext &context, IEspGetNavTreeDefnRequest &req, IEspGetNavTreeDefnResponse &resp);
     virtual bool getValue(IEspContext &context, IEspGetValueRequest &req, IEspGetValueResponse &resp);
     virtual bool unlockUser(IEspContext &context, IEspUnlockUserRequest &req, IEspUnlockUserResponse &resp);
@@ -965,6 +966,7 @@ public:
     virtual bool onHandleTopology(IEspContext &context, IEspHandleTopologyRequest &req, IEspHandleTopologyResponse &resp);
     virtual bool onHandleRows(IEspContext &context, IEspHandleRowsRequest &req, IEspHandleRowsResponse &resp);
     virtual bool onHandleAccessRules(IEspContext &context, IEspHandleAccessRulesRequest &req, IEspHandleAccessRulesResponse &resp);
+    virtual bool onHandleBaseAccessControlList(IEspContext &context, IEspHandleBaseAccessControlListRequest &req, IEspHandleBaseAccessControlListResponse &resp);
     virtual bool onGetNavTreeDefn(IEspContext &context, IEspGetNavTreeDefnRequest &req, IEspGetNavTreeDefnResponse &resp);
     virtual bool onGetValue(IEspContext &context, IEspGetValueRequest &req, IEspGetValueResponse &resp);
     virtual bool onUnlockUser(IEspContext &context, IEspUnlockUserRequest &req, IEspUnlockUserResponse &resp);
@@ -1038,6 +1040,7 @@ public:
     virtual bool onHandleTopology(IEspContext &context, IEspHandleTopologyRequest &req, IEspHandleTopologyResponse &resp);
     virtual bool onHandleRows(IEspContext &context, IEspHandleRowsRequest &req, IEspHandleRowsResponse &resp);
     virtual bool onHandleAccessRules(IEspContext &context, IEspHandleAccessRulesRequest &req, IEspHandleAccessRulesResponse &resp);
+    virtual bool onHandleBaseAccessControlList(IEspContext &context, IEspHandleBaseAccessControlListRequest &req, IEspHandleBaseAccessControlListResponse &resp);
     virtual bool onGetNavTreeDefn(IEspContext &context, IEspGetNavTreeDefnRequest &req, IEspGetNavTreeDefnResponse &resp);
     virtual bool onGetValue(IEspContext &context, IEspGetValueRequest &req, IEspGetValueResponse &resp);
     virtual bool onLockEnvironmentForCloud(IEspContext &context, IEspLockEnvironmentForCloudRequest &req, IEspLockEnvironmentForCloudResponse &resp);

esp/services/ws_ecl/ws_ecl_service.cpp

@@ -1744,19 +1744,16 @@ void CWsEclBinding::getWsEcl2XmlRequest(StringBuffer& soapmsg, IEspContext &cont
     }
 }
 
-StringBuffer &appendJSONException(StringBuffer &s, IException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
+StringBuffer &appendJSONExceptionItem(StringBuffer &s, int code, const char *msg, const char *objname="Exceptions", const char *arrayName = "Exception")
 {
-    if (!e)
-        return s;
     if (objname && *objname)
         appendJSONName(s, objname).append('{');
     if (arrayName && *arrayName)
         appendJSONName(s, arrayName).append('[');
     delimitJSON(s);
     s.append('{');
-    appendJSONValue(s, "Code", e->errorCode());
-    StringBuffer temp;
-    appendJSONValue(s, "Message", e->errorMessage(temp).str());
+    appendJSONValue(s, "Code", code);
+    appendJSONValue(s, "Message", msg);
     s.append('}');
     if (arrayName && *arrayName)
         s.append(']');
@@ -1765,6 +1762,14 @@ StringBuffer &appendJSONException(StringBuffer &s, IException *e, const char *ob
     return s;
 }
 
+StringBuffer &appendJSONException(StringBuffer &s, IException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
+{
+    if (!e)
+        return s;
+    StringBuffer temp;
+    return appendJSONExceptionItem(s, e->errorCode(), e->errorMessage(temp).str(), objname, arrayName);
+}
+
 StringBuffer &appendJSONExceptions(StringBuffer &s, IMultiException *e, const char *objname="Exceptions", const char *arrayName = "Exception")
 {
     if (!e)
@@ -1830,29 +1835,29 @@ void CWsEclBinding::getWsEclJsonResponse(StringBuffer& jsonmsg, IEspContext &con
         element.append(wsinfo.queryname.sget());
         element.append("Response");
 
-        VStringBuffer xpath("Body/%s/Results/Result/Exception", element.str());
-        Owned<IPropertyTreeIterator> exceptions = parmtree->getElements(xpath.str());
+        IPropertyTree *node = parmtree;
+        if (node->hasProp("Body"))
+            node = node->queryPropTree("Body");
+        if (node->hasProp(element))
+            node = node->queryPropTree(element);
+        if (node->hasProp("Results"))
+            node = node->queryPropTree("Results");
+        if (node->hasProp("Result"))
+            node = node->queryPropTree("Result");
 
         jsonmsg.appendf("{\n  \"%s\": {\n    \"Results\": {\n", element.str());
 
+        Owned<IPropertyTreeIterator> exceptions = node->getElements("Exception");
         if (exceptions && exceptions->first())
         {
-            jsonmsg.append("      \"Exceptions\": {\n        \"Exception\": [\n");
-            bool first=true;
+            appendJSONName(jsonmsg.pad(3), "Exceptions").append("{\n");
+            appendJSONName(jsonmsg.pad(4), "Exception").append("[\n");
             ForEach(*exceptions)
-            {
-                if (first)
-                    first=false;
-                else
-                    jsonmsg.append(",\n");
-            jsonmsg.appendf("          {\n            \"Code\": %d,\n            \"Message\": \"%s\"\n          }", exceptions->query().getPropInt("Code"), exceptions->query().queryProp("Message"));
-            }
-            jsonmsg.append("\n        ]\n      }\n");
+                appendJSONExceptionItem(jsonmsg.pad(2), exceptions->query().getPropInt("Code"), exceptions->query().queryProp("Message"), NULL, NULL);
+            jsonmsg.append("\n   ]\n    }\n");
         }
 
-        xpath.clear().append("Body/*[1]/Results/Result/Dataset");
-        Owned<IPropertyTreeIterator> datasets = parmtree->getElements(xpath.str());
-
+        Owned<IPropertyTreeIterator> datasets = node->getElements("Dataset");
         ForEach(*datasets)
         {
             IPropertyTree &ds = datasets->query();
@@ -1878,7 +1883,6 @@ void CWsEclBinding::getWsEclJsonResponse(StringBuffer& jsonmsg, IEspContext &con
                 }
             }
         }
-
         jsonmsg.append("    }\n  }\n}");
     }
     catch (IException *e)
@@ -2382,8 +2386,10 @@ void CWsEclBinding::sendRoxieRequest(const char *target, StringBuffer &req, Stri
     }
 }
 
-int CWsEclBinding::onSubmitQueryOutputXML(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo, const char *format)
+int CWsEclBinding::onSubmitQueryOutput(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo, const char *format)
 {
+    bool outputJSON = !format ? false : strieq(format, "json");
+
     StringBuffer soapmsg;
 
     getSoapMessage(soapmsg, context, request, wsinfo, REQXML_TRIM|REQXML_ROOT);
@@ -2407,17 +2413,28 @@ int CWsEclBinding::onSubmitQueryOutputXML(IEspContext &context, CHttpRequest* re
         StringBuffer roxieresp;
         sendRoxieRequest(wsinfo.qsetname.get(), soapmsg, roxieresp, status, wsinfo.queryname);
 
-        Owned<IWuWebView> web = createWuWebView(*wsinfo.wu, wsinfo.queryname.get(), getCFD(), true);
-        if (web.get())
-            web->expandResults(roxieresp.str(), output, xmlflags);
+        if (outputJSON)
+            getWsEclJsonResponse(output, context, request, roxieresp.str(), wsinfo);
+        else
+        {
+            Owned<IWuWebView> web = createWuWebView(*wsinfo.wu, wsinfo.queryname.get(), getCFD(), true);
+            if (web.get())
+                web->expandResults(roxieresp.str(), output, xmlflags);
+        }
     }
     else
     {
         submitWsEclWorkunit(context, wsinfo, soapmsg.str(), output, xmlflags);
+        if (outputJSON)
+        {
+            StringBuffer jsonresp;
+            getWsEclJsonResponse(jsonresp, context, request, output.str(), wsinfo);
+            output.swapWith(jsonresp);
+        }
     }
 
     response->setContent(output.str());
-    response->setContentType(HTTP_TYPE_APPLICATION_XML);
+    response->setContentType(outputJSON ? "application/json" : "application/xml");
     response->setStatus("200 OK");
     response->send();
 
@@ -2725,7 +2742,7 @@ int CWsEclBinding::onGet(CHttpRequest* request, CHttpResponse* response)
             nextPathNode(thepath, format);
 
             WsEclWuInfo wsinfo(wuid.str(), qs.str(), qid.str(), context->queryUserId(), context->queryPassword());
-            return onSubmitQueryOutputXML(*context, request, response, wsinfo, format.str());
+            return onSubmitQueryOutput(*context, request, response, wsinfo, format.str());
         }
         else if (!stricmp(methodName.str(), "xslt"))
         {

esp/services/ws_ecl/ws_ecl_service.hpp

@@ -165,7 +165,7 @@ public:
     void buildSampleResponseXml(StringBuffer& msg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo);
     void getSoapMessage(StringBuffer& soapmsg, IEspContext &context, CHttpRequest* request, WsEclWuInfo &wsinfo, unsigned flags);
     int onGetSoapBuilder(IEspContext &context, CHttpRequest* request, CHttpResponse* response,  WsEclWuInfo &wsinfo);
-    int onSubmitQueryOutputXML(IEspContext &context, CHttpRequest* request, CHttpResponse* response,    WsEclWuInfo &wsinfo, const char *format);
+    int onSubmitQueryOutput(IEspContext &context, CHttpRequest* request, CHttpResponse* response,    WsEclWuInfo &wsinfo, const char *format);
     int onSubmitQueryOutputView(IEspContext &context, CHttpRequest* request, CHttpResponse* response, WsEclWuInfo &wsinfo);
 
     int submitWsEclWorkunit(IEspContext & context, WsEclWuInfo &wsinfo, const char *xml, StringBuffer &out, unsigned flags, const char *viewname=NULL, const char *xsltname=NULL);

esp/xslt/wsecl3_form.xsl

@@ -156,7 +156,9 @@ function setESPFormAction()  // reqType: 0: regular form, 1: soap, 2: form param
         else if (actval.value=="run_xslt")
             actionpath = "]]></xsl:text><xsl:value-of disable-output-escaping="yes"  select="concat('/WsEcl/xslt/query/', $queryPath, '/', $methodName)"/><xsl:text disable-output-escaping="yes"><![CDATA[";
         else if (actval.value=="xml")
-            actionpath = "]]></xsl:text><xsl:value-of disable-output-escaping="yes"  select="concat('/WsEcl/submit/query/', $queryPath, '/', $methodName, '?view=xml&amp;display')"/><xsl:text disable-output-escaping="yes"><![CDATA[";
+            actionpath = "]]></xsl:text><xsl:value-of disable-output-escaping="yes"  select="concat('/WsEcl/submit/query/', $queryPath, '/', $methodName, '/xml?view=xml&amp;display')"/><xsl:text disable-output-escaping="yes"><![CDATA[";
+        else if (actval.value=="json")
+            actionpath = "]]></xsl:text><xsl:value-of disable-output-escaping="yes"  select="concat('/WsEcl/submit/query/', $queryPath, '/', $methodName, '/json')"/><xsl:text disable-output-escaping="yes"><![CDATA[";
         else
             actionpath= "]]></xsl:text><xsl:value-of disable-output-escaping="yes"  select="concat('/WsEcl/xslt/query/', $queryPath, '/', $methodName, '?view=')"/><xsl:text disable-output-escaping="yes"><![CDATA["+actval.value;
     }
@@ -308,10 +310,11 @@ function switchInputForm()
                         <xsl:for-each select="/FormInfo/CustomViews/Result">
                             <option><xsl:attribute name="value"><xsl:value-of select="."/></xsl:attribute><xsl:value-of select="."/></option>
                         </xsl:for-each>
-                        <option value="run_xslt">OUTPUT TABLES</option>
-                        <option value="xml">OUTPUT XML</option>
-                        <option value="esp_soap">SOAP TEST</option>
-                        <option value="esp_json">JSON TEST</option>
+                        <option value="run_xslt">Output Tables</option>
+                        <option value="xml">Output XML</option>
+                        <option value="json">Output JSON</option>
+                        <option value="esp_soap">SOAP Test</option>
+                        <option value="esp_json">JSON Test</option>
                     </select>&nbsp;
                    <input type='submit' value='Submit' name='S1'/>
 

initfiles/componentfiles/configxml/roxie.xsd.in

@@ -432,13 +432,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="allowRoxieOnDemand" type="xs:boolean" use="optional" default="false">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Allow roxie to load, run, add queries meant to be run 1 time (no file data copies for these queries</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="callbackRetries" type="xs:nonNegativeInteger" use="optional" default="3">
       <xs:annotation>
         <xs:appinfo>
@@ -540,20 +533,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="deleteUnneededFiles" type="xs:boolean" use="optional" default="false">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>delete data files and query dlls when query is deleted from roxie</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
-    <xs:attribute name="enableForceKeyDiffCopy" type="xs:boolean" use="optional" default="false">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Enable / Disable always copying key diff files to roxie.</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="enableKeyDiff" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
@@ -568,13 +547,6 @@
             </xs:appinfo>
         </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="enableSNMP" type="xs:boolean" use="optional" default="true">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Enable SNMP monitoring/control interface</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="flushJHtreeCacheOnOOM" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
@@ -704,13 +676,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="resolveFilesInPackage" type="xs:boolean" use="optional" default="false">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Allow for resolving of file references in a package, but not known to roxie.</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="serverThreads" type="xs:nonNegativeInteger" use="optional" default="30">
       <xs:annotation>
         <xs:appinfo>
@@ -1065,13 +1030,6 @@
     </xs:attribute>
   </xs:attributeGroup>
   <xs:attributeGroup name="Debug">
-    <xs:attribute name="channel0onPrimary" type="xs:boolean" use="optional" default="true">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Will favor primary agents to handle requests multicasted on channel 0 </tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="checkCompleted" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
@@ -1079,13 +1037,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-      <xs:attribute name="checkingHeap" type="xs:nonNegativeInteger" use="optional" default="0">
-          <xs:annotation>
-              <xs:appinfo>
-                  <tooltip>Use additional checks in roxie heap to detect corruption, overwrites, use after release etc (slow). Level 2 is VERY slow</tooltip>
-              </xs:appinfo>
-          </xs:annotation>
-      </xs:attribute>
       <xs:attribute name="checkPrimaries" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
@@ -1093,13 +1044,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="checkVersion" type="xs:boolean" use="optional" default="true">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>check compiled version of dll before loading it</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="dafilesrvLookupTimeout" type="xs:nonNegativeInteger" use="optional" default="10000">
       <xs:annotation>
         <xs:appinfo>
@@ -1156,13 +1100,6 @@
               </xs:appinfo>
           </xs:annotation>
       </xs:attribute>
-      <xs:attribute name="diskReadStable" type="xs:boolean" use="optional" default="true">
-          <xs:annotation>
-              <xs:appinfo>
-                  <tooltip>Force diskreads to return in stable order</tooltip>
-              </xs:appinfo>
-          </xs:annotation>
-      </xs:attribute>
       <xs:attribute name="doIbytiDelay" type="xs:boolean" use="optional" default="true">
       <xs:annotation>
         <xs:appinfo>
@@ -1205,13 +1142,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="indexReadStable" type="xs:boolean" use="optional" default="true">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Force indexreads to return in stable order</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="initIbytiDelay" type="xs:nonNegativeInteger" use="optional" default="100">
       <xs:annotation>
         <xs:appinfo>
@@ -1226,20 +1156,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="keyedJoinFlowLimit" type="xs:nonNegativeInteger" use="optional" default="1000">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Number of keyed join groups permitted (per active activity) before blocking</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
-    <xs:attribute name="keyedJoinStable" type="xs:boolean" use="optional" default="true">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Force keyed joins to return results in stable order</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="linuxYield" type="xs:boolean" use="optional" default="false">
       <xs:annotation>
         <xs:appinfo>
@@ -1324,13 +1240,6 @@
         </xs:appinfo>
       </xs:annotation>
     </xs:attribute>
-    <xs:attribute name="smartSteppingChunkRows" type="xs:nonNegativeInteger" use="optional" default="100">
-      <xs:annotation>
-        <xs:appinfo>
-          <tooltip>Fetch up to N rows per request when global-smartstepping</tooltip>
-        </xs:appinfo>
-      </xs:annotation>
-    </xs:attribute>
     <xs:attribute name="socketCheckInterval" type="xs:nonNegativeInteger" use="optional" default="5000">
       <xs:annotation>
         <xs:appinfo>

initfiles/etc/DIR_NAME/environment.xml.in

@@ -754,7 +754,6 @@
              program="${EXEC_PATH}/ftslave"/>
   </FTSlaveProcess>
   <RoxieCluster allFilesDynamic="true"
-                allowRoxieOnDemand="false"
                 baseDataDir="${RUNTIME_PATH}/hpcc-data/roxie"
                 blindLogging="false"
                 blobCacheMem="0"
@@ -762,12 +761,9 @@
                 buildSet="roxie"
                 callbackRetries="3"
                 callbackTimeout="500"
-                channel0onPrimary="true"
                 checkCompleted="true"
                 checkFileDate="true"
-                checkingHeap="0"
                 checkPrimaries="true"
-                checkVersion="true"
                 clusterWidth="1"
                 copyResources="false"
                 crcResources="false"
@@ -788,16 +784,12 @@
                 defaultSLAPriorityTimeLimit="0"
                 defaultSLAPriorityTimeWarning="5000"
                 defaultStripLeadingWhitespace="1"
-                deleteUnneededFiles="false"
                 description="Roxie cluster"
                 directory="${RUNTIME_PATH}/myroxie"
                 diskReadBufferSize="65536"
-                diskReadStable="true"
                 doIbytiDelay="true"
-                enableForceKeyDiffCopy="false"
                 enableHeartBeat="true"
                 enableKeyDiff="true"
-                enableSNMP="true"
                 enableSysLog="true"
                 fastLaneQueue="true"
                 fieldTranslationEnabled="false"
@@ -806,11 +798,8 @@
                 highTimeout="2000"
                 ignoreMissingFiles="false"
                 indexReadChunkSize="60000"
-                indexReadStable="true"
                 initIbytiDelay="100"
                 jumboFrames="false"
-                keyedJoinFlowLimit="1000"
-                keyedJoinStable="true"
                 lazyOpen="smart"
                 leafCacheMem="50"
                 linuxYield="false"
@@ -850,7 +839,6 @@
                 preferredSubnetMask=""
                 preloadOnceData="true"
                 remoteFilesExpire="3600000"
-                resolveFilesInPackage="false"
                 roxieMulticastEnabled="true"
                 serverSideCacheSize="0"
                 serverThreads="30"
@@ -859,7 +847,6 @@
                 slaTimeout="2000"
                 slaveConfig="simple"
                 slaveThreads="30"
-                smartSteppingChunkRows="100"
                 soapTraceLevel="1"
                 socketCheckInterval="5000"
                 SSHidentityfile="$HOME/.ssh/id_rsa"

plugins/CMakeLists.txt

@@ -25,15 +25,7 @@ if ("${BUILD_LEVEL}" STREQUAL "COMMUNITY")
   add_subdirectory (proxies)
 endif ()
 
-if (USE_V8)
-  add_subdirectory (v8embed)
-endif ()
-if (USE_PYTHON)
-  add_subdirectory (pyembed)
-endif ()
-if (USE_JNI)
-  add_subdirectory (javaembed)
-endif()
-if (USE_RINSIDE)
-  add_subdirectory (Rembed)
-endif()
+add_subdirectory (v8embed)
+add_subdirectory (pyembed)
+add_subdirectory (javaembed)
+add_subdirectory (Rembed)

plugins/Rembed/CMakeLists.txt

@@ -25,9 +25,9 @@
 
 project( Rembed )
 
-ADD_PLUGIN(Rembed PACKAGES R OPTION MAKE_REMBED)
-
-if ( MAKE_REMBED )
+if (USE_RINSIDE)
+  ADD_PLUGIN(Rembed PACKAGES R OPTION MAKE_REMBED)
+  if ( MAKE_REMBED )
     set (    SRCS
              Rembed.cpp
         )
@@ -57,4 +57,8 @@ if ( MAKE_REMBED )
         eclrtl
         jlib
         )
+  endif()
 endif()
+
+# Even if not making the R plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/R.ecllib DESTINATION plugins COMPONENT Runtime)

plugins/Rembed/R.ecllib

@@ -0,0 +1,27 @@
+/*##############################################################################
+
+ HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ ############################################################################## */
+
+EXPORT Language := SERVICE : plugin('Rembed')
+  boolean getEmbedContext():cpp,pure,namespace='Rembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';
+  boolean syntaxCheck(const varstring src):cpp,pure,namespace='Rembed',entrypoint='syntaxCheck';
+  unload():cpp,pure,namespace='Rembed',entrypoint='unload';
+END;
+
+EXPORT getEmbedContext := Language.getEmbedContext;
+EXPORT syntaxCheck := Language.syntaxCheck;
+EXPORT boolean supportsImport := false;
+EXPORT boolean supportsScript := true;

plugins/Rembed/Rembed.cpp

@@ -19,7 +19,6 @@
 #include "RInside.h"
 #include "jexcept.hpp"
 #include "jthread.hpp"
-#include "hqlplugins.hpp"
 #include "deftype.hpp"
 #include "eclrtl.hpp"
 #include "eclrtl_imp.hpp"
@@ -30,40 +29,6 @@
 #define EXPORT
 #endif
 
-static const char * compatibleVersions[] =
-{ "R Embed Helper 1.0.0", NULL };
-
-static const char *version = "R Embed Helper 1.0.0";
-
-static const char * EclDefinition =
-    "EXPORT Language := SERVICE\n"
-    "  boolean getEmbedContext():cpp,pure,namespace='Rembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
-    "  boolean syntaxCheck(const varstring src):cpp,pure,namespace='Rembed',entrypoint='syntaxCheck';\n"
-    "  unload():cpp,pure,namespace='Rembed',entrypoint='unload';\n"
-    "END;"
-    "EXPORT getEmbedContext := Language.getEmbedContext;"
-    "EXPORT syntaxCheck := Language.syntaxCheck;"
-    "EXPORT boolean supportsImport := false;"
-    "EXPORT boolean supportsScript := true;";
-
-extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
-{
-    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
-    {
-        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
-        pbx->compatibleVersions = compatibleVersions;
-    }
-    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
-        return false;
-    pb->magicVersion = PLUGIN_VERSION;
-    pb->version = version;
-    pb->moduleName = "R";
-    pb->ECL = EclDefinition;
-    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
-    pb->description = "R Embed Helper";
-    return true;
-}
-
 namespace Rembed
 {
 

plugins/javaembed/CMakeLists.txt

@@ -25,9 +25,9 @@
 
 project( javaembed )
 
-ADD_PLUGIN(javaembed PACKAGES JNI OPTION MAKE_JAVAEMBED)
-
-if ( MAKE_JAVAEMBED )
+if (USE_JNI)
+  ADD_PLUGIN(javaembed PACKAGES JNI OPTION MAKE_JAVAEMBED)
+  if ( MAKE_JAVAEMBED )
     set (    SRCS
              javaembed.cpp
         )
@@ -71,4 +71,7 @@ if ( MAKE_JAVAEMBED )
         eclrtl
         jlib
         )
-endif()
+  endif()
+endif()
+# Even if not making the Java plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/java.ecllib DESTINATION plugins COMPONENT Runtime)

plugins/javaembed/java.ecllib

@@ -0,0 +1,23 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+EXPORT Language := SERVICE : plugin('javaembed')
+  boolean getEmbedContext():cpp,pure,namespace='javaembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';
+END;
+EXPORT getEmbedContext := Language.getEmbedContext;
+EXPORT boolean supportsImport := true;
+EXPORT boolean supportsScript := false;

plugins/javaembed/javaembed.cpp

@@ -19,7 +19,6 @@
 #include <jni.h>
 #include "jexcept.hpp"
 #include "jthread.hpp"
-#include "hqlplugins.hpp"
 #include "deftype.hpp"
 #include "eclrtl.hpp"
 #include "eclrtl_imp.hpp"
@@ -32,38 +31,6 @@
 #define EXPORT
 #endif
 
-static const char * compatibleVersions[] = {
-    "Java Embed Helper 1.0.0",
-    NULL };
-
-static const char *version = "Java Embed Helper 1.0.0";
-
-static const char * EclDefinition =
-    "EXPORT Language := SERVICE\n"
-    "  boolean getEmbedContext():cpp,pure,namespace='javaembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
-    "END;"
-    "EXPORT getEmbedContext := Language.getEmbedContext;"
-    "EXPORT boolean supportsImport := true;"
-    "EXPORT boolean supportsScript := false;";
-
-extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
-{
-    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
-    {
-        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
-        pbx->compatibleVersions = compatibleVersions;
-    }
-    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
-        return false;
-    pb->magicVersion = PLUGIN_VERSION;
-    pb->version = version;
-    pb->moduleName = "java";
-    pb->ECL = EclDefinition;
-    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
-    pb->description = "Java Embed Helper";
-    return true;
-}
-
 namespace javaembed {
 
 // Use a global object to ensure that the Java VM  is initialized once only.

plugins/pyembed/CMakeLists.txt

@@ -27,44 +27,46 @@ set ( debug_python Off )   # A lot slower but can assist in debugging...
 
 project( pyembed )
 
-ADD_PLUGIN(pyembed PACKAGES PythonLibs OPTION MAKE_PYEMBED MINVERSION 2.6 MAXVERSION 2.7)
+if (USE_PYTHON)
+  ADD_PLUGIN(pyembed PACKAGES PythonLibs OPTION MAKE_PYEMBED MINVERSION 2.6 MAXVERSION 2.7)
+  if ( MAKE_PYEMBED )
+    set ( SRCS
+          pyembed.cpp
+        )
 
-if ( MAKE_PYEMBED )
+    include_directories (
+         "${PYTHON_INCLUDE_DIR}"
+         ./../../system/include
+         ./../../rtl/eclrtl
+         ./../../rtl/include
+         ./../../common/deftype
+         ./../../system/jlib
+       )
 
-  set (    SRCS
-           pyembed.cpp
-      )
+    ADD_DEFINITIONS( -D_USRDLL -DPYEMBED_EXPORTS )
+    if (debug_python)
+      ADD_DEFINITIONS(-DPy_DEBUG)
+    endif()
 
-  include_directories (
-           "${PYTHON_INCLUDE_DIR}"
-           ./../../system/include
-           ./../../rtl/eclrtl
-           ./../../rtl/include
-           ./../../common/deftype
-           ./../../system/jlib
-      )
+    HPCC_ADD_LIBRARY( pyembed SHARED ${SRCS} )
+    if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
+      message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
+    else()
+      set_target_properties( pyembed PROPERTIES NO_SONAME 1 )
+    endif()
 
-  ADD_DEFINITIONS( -D_USRDLL -DPYEMBED_EXPORTS )
-  if (debug_python)
-    ADD_DEFINITIONS(-DPy_DEBUG)
-  endif()
-
-  HPCC_ADD_LIBRARY( pyembed SHARED ${SRCS} )
-  if (${CMAKE_VERSION} VERSION_LESS "2.8.9")
-    message("WARNING: Cannot set NO_SONAME. shlibdeps will give warnings when package is installed")
-  else()
-    set_target_properties( pyembed PROPERTIES NO_SONAME 1 )
-  endif()
+    install ( TARGETS pyembed DESTINATION plugins )
+    if (debug_python)
+      target_link_libraries ( pyembed ${PYTHON_DEBUG_LIBRARY} )
+    else()
+      target_link_libraries ( pyembed ${PYTHON_LIBRARY} )
+    endif()
 
-  install ( TARGETS pyembed DESTINATION plugins )
-  if (debug_python)
-    target_link_libraries ( pyembed ${PYTHON_DEBUG_LIBRARY} )
-  else()
-    target_link_libraries ( pyembed ${PYTHON_LIBRARY} )
+    target_link_libraries ( pyembed
+        eclrtl
+        jlib
+        )
   endif()
-
-  target_link_libraries ( pyembed
-      eclrtl
-      jlib
-      )
 endif()
+# Even if not making the Python plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/python.ecllib DESTINATION plugins COMPONENT Runtime)

plugins/pyembed/pyembed.cpp

@@ -19,7 +19,6 @@
 #include "Python.h"
 #include "jexcept.hpp"
 #include "jthread.hpp"
-#include "hqlplugins.hpp"
 #include "deftype.hpp"
 #include "eclrtl.hpp"
 #include "eclrtl_imp.hpp"
@@ -30,40 +29,6 @@
 #define EXPORT
 #endif
 
-static const char * compatibleVersions[] = {
-    "Python2.7 Embed Helper 1.0.0",
-    NULL };
-
-static const char *version = "Python2.7 Embed Helper 1.0.0";
-
-static const char * EclDefinition =
-    "EXPORT Language := SERVICE\n"
-    "  boolean getEmbedContext():cpp,pure,namespace='pyembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
-    "  boolean syntaxCheck(const varstring src):cpp,pure,namespace='pyembed',entrypoint='syntaxCheck';\n"
-    "END;"
-    "EXPORT getEmbedContext := Language.getEmbedContext;"
-    "EXPORT syntaxCheck := Language.syntaxCheck;"
-    "EXPORT boolean supportsImport := true;"
-    "EXPORT boolean supportsScript := true;";
-
-extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
-{
-    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
-    {
-        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
-        pbx->compatibleVersions = compatibleVersions;
-    }
-    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
-        return false;
-    pb->magicVersion = PLUGIN_VERSION;
-    pb->version = version;
-    pb->moduleName = "python";
-    pb->ECL = EclDefinition;
-    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
-    pb->description = "Python2.7 Embed Helper";
-    return true;
-}
-
 namespace pyembed {
 
 // Use class OwnedPyObject for any objects that are not 'borrowed references'

plugins/pyembed/python.ecllib

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+EXPORT Language := SERVICE : plugin('pyembed')
+  boolean getEmbedContext():cpp,pure,namespace='pyembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';
+  boolean syntaxCheck(const varstring src):cpp,pure,namespace='pyembed',entrypoint='syntaxCheck';
+END;
+EXPORT getEmbedContext := Language.getEmbedContext;
+EXPORT syntaxCheck := Language.syntaxCheck;
+EXPORT boolean supportsImport := true;
+EXPORT boolean supportsScript := true;

plugins/v8embed/CMakeLists.txt

@@ -25,9 +25,9 @@
 
 project( v8embed )
 
-ADD_PLUGIN(v8embed PACKAGES V8 OPTION MAKE_V8EMBED)
-
-if ( MAKE_V8EMBED )
+if (USE_V8)
+  ADD_PLUGIN(v8embed PACKAGES V8 OPTION MAKE_V8EMBED)
+  if ( MAKE_V8EMBED )
     set (    SRCS
              v8embed.cpp
         )
@@ -57,4 +57,7 @@ if ( MAKE_V8EMBED )
         eclrtl
         jlib
         )
-endif()
+  endif()
+endif()
+# Even if not making the V8 plugin, we want to install the header
+install ( FILES ${CMAKE_CURRENT_SOURCE_DIR}/javascript.ecllib DESTINATION plugins COMPONENT Runtime)

plugins/v8embed/javascript.ecllib

@@ -0,0 +1,25 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2013 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+EXPORT Language := SERVICE : plugin('v8embed')
+  boolean getEmbedContext():cpp,pure,namespace='javascriptLanguageHelper',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';
+  boolean syntaxCheck(const varstring src):cpp,pure,namespace='javascriptLanguageHelper',entrypoint='syntaxCheck';
+END;
+EXPORT getEmbedContext := Language.getEmbedContext;
+EXPORT syntaxCheck := Language.syntaxCheck;
+EXPORT boolean supportsImport := false;
+EXPORT boolean supportsScript := true;

plugins/v8embed/v8embed.cpp

@@ -19,7 +19,6 @@
 #include "v8.h"
 #include "jexcept.hpp"
 #include "jthread.hpp"
-#include "hqlplugins.hpp"
 #include "deftype.hpp"
 #include "eclrtl.hpp"
 #include "eclrtl_imp.hpp"
@@ -30,39 +29,6 @@
 #define EXPORT
 #endif
 
-static const char * compatibleVersions[] = {
-    "V8 JavaScript Embed Helper 1.0.0",
-    NULL };
-
-static const char *version = "V8 JavaScript Embed Helper 1.0.0";
-static const char * EclDefinition =
-    "EXPORT Language := SERVICE\n"
-    "  boolean getEmbedContext():cpp,pure,namespace='javascriptLanguageHelper',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()';\n"
-    "  boolean syntaxCheck(const varstring src):cpp,pure,namespace='javascriptLanguageHelper',entrypoint='syntaxCheck';\n"
-    "END;"
-    "export getEmbedContext := Language.getEmbedContext;"
-    "export syntaxCheck := Language.syntaxCheck;"
-    "EXPORT boolean supportsImport := false;"
-    "EXPORT boolean supportsScript := true;";
-
-extern "C" EXPORT bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
-{
-    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
-    {
-        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
-        pbx->compatibleVersions = compatibleVersions;
-    }
-    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
-        return false;
-    pb->magicVersion = PLUGIN_VERSION;
-    pb->version = version;
-    pb->moduleName = "javascript";
-    pb->ECL = EclDefinition;
-    pb->flags = PLUGIN_DLL_MODULE | PLUGIN_MULTIPLE_VERSIONS;
-    pb->description = "V8 JavaScript Embed Helper";
-    return true;
-}
-
 namespace javascriptLanguageHelper {
 
 class V8JavascriptEmbedFunctionContext : public CInterfaceOf<IEmbedFunctionContext>

roxie/ccd/ccd.hpp

@@ -339,14 +339,12 @@ extern unsigned initIbytiDelay;
 extern unsigned minIbytiDelay;
 extern bool copyResources;
 extern bool chunkingHeap;
-extern unsigned keyedJoinFlowLimit;
 extern unsigned perChannelFlowLimit;
 extern unsigned parallelLoopFlowLimit;
 extern unsigned numServerThreads;
 extern unsigned numRequestArrayThreads;
 extern unsigned readTimeout;
 extern unsigned indexReadChunkSize;
-extern unsigned smartSteppingChunkRows;
 extern SocketEndpoint ownEP;
 extern unsigned maxBlockSize;
 extern unsigned maxLockAttempts;
@@ -359,7 +357,6 @@ extern unsigned socketCheckInterval;
 extern memsize_t defaultMemoryLimit;
 extern unsigned defaultTimeLimit[3];
 extern unsigned defaultWarnTimeLimit[3];
-extern bool deleteUnneededFiles;
 extern bool checkPrimaries;
 extern bool pretendAllOpt;
 extern ClientCertificate clientCert;
@@ -374,7 +371,6 @@ extern unsigned preabortIndexReadsThreshold;
 extern bool traceStartStop;
 extern bool traceServerSideCache;
 extern bool timeActivities;
-extern int defaultCheckingHeap;
 extern unsigned watchActivityId;
 extern unsigned testSlaveFailure;
 extern unsigned dafilesrvLookupTimeout;
@@ -383,7 +379,6 @@ extern unsigned mtu_size;
 extern StringBuffer fileNameServiceDali;
 extern StringBuffer roxieName;
 extern bool trapTooManyActiveQueries;
-extern bool allowRoxieOnDemand;
 extern unsigned maxEmptyLoopIterations;
 extern unsigned maxGraphLoopIterations;
 extern HardwareInfo hdwInfo;
@@ -395,7 +390,6 @@ extern bool probeAllRows;
 extern bool steppingEnabled;
 extern bool simpleLocalKeyedJoins;
 extern bool enableKeyDiff;
-extern bool enableForceKeyDiffCopy;
 extern bool useTreeCopy;
 extern PTreeReaderOptions defaultXmlReadFlags;
 extern bool mergeSlaveStatistics;

roxie/ccd/ccdmain.cpp

@@ -59,7 +59,6 @@ unsigned numSlaveThreads = 30;
 unsigned numRequestArrayThreads = 5;
 unsigned headRegionSize;
 bool enableHeartBeat = true;
-unsigned keyedJoinFlowLimit = 1000;
 unsigned parallelLoopFlowLimit = 100;
 unsigned perChannelFlowLimit = 10;
 time_t startupTime;
@@ -67,11 +66,8 @@ unsigned statsExpiryTime = 3600;
 unsigned miscDebugTraceLevel = 0;  // separate trace settings purely for debugging specific items (i.e. all possible locations to look for files at startup)
 unsigned readTimeout = 300;
 unsigned indexReadChunkSize = 60000;
-unsigned smartSteppingChunkRows = 100;
 unsigned maxBlockSize = 10000000;
 unsigned maxLockAttempts = 5;
-bool checkVersion = true;
-bool deleteUnneededFiles = true;
 bool checkPrimaries = true;
 bool pretendAllOpt = false;
 bool traceStartStop = false;
@@ -107,7 +103,6 @@ unsigned initIbytiDelay; // In MillSec
 unsigned minIbytiDelay;  // In MillSec
 bool copyResources;
 bool enableKeyDiff = true;
-bool enableForceKeyDiffCopy = true;
 bool chunkingHeap = true;
 bool logFullQueries;
 bool blindLogging = false;
@@ -122,7 +117,6 @@ memsize_t defaultMemoryLimit;
 unsigned defaultTimeLimit[3] = {0, 0, 0};
 unsigned defaultWarnTimeLimit[3] = {0, 5000, 5000};
 
-int defaultCheckingHeap = 0;
 unsigned defaultParallelJoinPreload = 0;
 unsigned defaultPrefetchProjectPreload = 10;
 unsigned defaultConcatPreload = 0;
@@ -139,7 +133,6 @@ unsigned mtu_size = 1400; // upper limit on outbound buffer size - allow some he
 StringBuffer fileNameServiceDali;
 StringBuffer roxieName;
 bool trapTooManyActiveQueries;
-bool allowRoxieOnDemand;
 unsigned maxEmptyLoopIterations;
 unsigned maxGraphLoopIterations;
 bool probeAllRows;
@@ -628,7 +621,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         lowTimeout = topology->getPropInt("@lowTimeout", 10000);
         highTimeout = topology->getPropInt("@highTimeout", 2000);
         slaTimeout = topology->getPropInt("@slaTimeout", 2000);
-        keyedJoinFlowLimit = topology->getPropInt("@keyedJoinFlowLimit", 1000);
         parallelLoopFlowLimit = topology->getPropInt("@parallelLoopFlowLimit", 100);
         perChannelFlowLimit = topology->getPropInt("@perChannelFlowLimit", 10);
         copyResources = topology->getPropBool("@copyResources", true);
@@ -688,7 +680,6 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
             DBGLOG("WARNING: ignoring udpSnifferEnabled setting as multicast not enabled");
 
         indexReadChunkSize = topology->getPropInt("@indexReadChunkSize", 60000);
-        smartSteppingChunkRows = topology->getPropInt("@smartSteppingChunkRows", 100);
         numSlaveThreads = topology->getPropInt("@slaveThreads", 30);
         numServerThreads = topology->getPropInt("@serverThreads", 30);
         numRequestArrayThreads = topology->getPropInt("@requestArrayThreads", 5);
@@ -713,11 +704,9 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         defaultFullKeyedJoinPreload = topology->getPropInt("@defaultFullKeyedJoinPreload", 0);
         defaultKeyedJoinPreload = topology->getPropInt("@defaultKeyedJoinPreload", 0);
         defaultPrefetchProjectPreload = topology->getPropInt("@defaultPrefetchProjectPreload", 10);
+        diskReadBufferSize = topology->getPropInt("@diskReadBufferSize", 0x10000);
         fieldTranslationEnabled = topology->getPropBool("@fieldTranslationEnabled", false);
 
-        defaultCheckingHeap = topology->getPropInt("@checkingHeap", 0);
-        checkVersion = topology->getPropBool("@checkVersion", true);
-        deleteUnneededFiles = topology->getPropBool("@deleteUnneededFiles", true);
         checkPrimaries = topology->getPropBool("@checkPrimaries", true);
         pretendAllOpt = topology->getPropBool("@ignoreMissingFiles", false);
         memoryStatsInterval = topology->getPropInt("@memoryStatsInterval", 60);
@@ -746,14 +735,12 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
         dafilesrvLookupTimeout = topology->getPropInt("@dafilesrvLookupTimeout", 10000);
         topology->getProp("@daliServers", fileNameServiceDali);
         trapTooManyActiveQueries = topology->getPropBool("@trapTooManyActiveQueries", true);
-        allowRoxieOnDemand = topology->getPropBool("@allowRoxieOnDemand", false);
         maxEmptyLoopIterations = topology->getPropInt("@maxEmptyLoopIterations", 1000);
         maxGraphLoopIterations = topology->getPropInt("@maxGraphLoopIterations", 1000);
         useTreeCopy = topology->getPropBool("@useTreeCopy", true);
         mergeSlaveStatistics = topology->getPropBool("@mergeSlaveStatistics", true);
 
         enableKeyDiff = topology->getPropBool("@enableKeyDiff", true);
-        enableForceKeyDiffCopy = topology->getPropBool("@enableForceKeyDiffCopy", false);
 
         // MORE: Get parms from topology after it is populated from Hardware/computer types section in configenv
         //       Then if does not match and based on desired action in topolgy, either warn, or fatal exit or .... etc

roxie/ccd/ccdstate.cpp

@@ -1367,11 +1367,6 @@ private:
             {
                 reply.appendf("<Alive restarts='%d'/>", restarts);
             }
-            else if (stricmp(queryName, "control:allowRoxieOnDemand")==0)
-            {
-                allowRoxieOnDemand= control->getPropBool("@val", true);
-                topology->setPropBool("@allowRoxieOnDemand", allowRoxieOnDemand);
-            }
             else
                 unknown = true;
             break;
@@ -1425,11 +1420,6 @@ private:
                 dafilesrvLookupTimeout = control->getPropInt("@val", 10000);
                 topology->setPropInt("@dafilesrvLookupTimeout", dafilesrvLookupTimeout);
             }
-            else if (stricmp(queryName, "control:defaultCheckingHeap")==0)
-            {
-                defaultCheckingHeap = control->getPropInt("@val", false);
-                topology->setPropInt("@checkingHeap", defaultCheckingHeap);
-            }
             else if (stricmp(queryName, "control:defaultConcatPreload")==0)
             {
                 defaultConcatPreload = control->getPropInt("@val", 0);
@@ -1512,11 +1502,6 @@ private:
                 enableKeyDiff = control->getPropBool("@val", true);
                 topology->setPropBool("@enableKeyDiff", enableKeyDiff);
             }
-            else if (stricmp(queryName, "control:enableForceKeyDiffCopy")==0)
-            {
-                enableForceKeyDiffCopy = control->getPropBool("@val", false);
-                topology->setPropBool("@enableForceKeyDiffCopy", enableForceKeyDiffCopy);
-            }
             else
                 unknown = true;
             break;

testing/ecl/dict15c.ecl

@@ -54,4 +54,4 @@ withAnagrams := TABLE(NOFOLD(shortWords), { word, anagrams := findAnagrams(Word)
 
 moreThanOne := withAnagrams(count(anagrams)>1);
 
-OUTPUT(moreThanOne);
+OUTPUT(sort(moreThanOne, RECORD));

testing/ecl/key/assert.xml

@@ -1,19 +1,19 @@
-<Error><source>user</source><line>27</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
-<Error><source>user</source><line>28</line><code>100000</code><message>Abc2</message></Error>
-<Error><source>user</source><line>29</line><code>100000</code><message>Assert (1 = 2) failed [1 = val4]</message></Error>
-<Error><source>user</source><line>30</line><code>100000</code><message>Abc3</message></Error>
-<Error><source>user</source><line>37</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
-<Error><source>user</source><line>38</line><code>100000</code><message>Abc5</message></Error>
-<Error><source>user</source><line>39</line><code>100000</code><message>Assert (1 = 2) failed [val1 = val4]</message></Error>
-<Error><source>user</source><line>40</line><code>100000</code><message>Abc6</message></Error>
-<Error><source>user</source><line>35</line><code>100000</code><message>Assert (2 = 1) failed [val1 = 1]</message></Error>
-<Error><source>user</source><line>36</line><code>100000</code><message>Abc4</message></Error>
-<Error><source>user</source><line>47</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
-<Error><source>user</source><line>48</line><code>100000</code><message>Abc8</message></Error>
-<Error><source>user</source><line>49</line><code>100000</code><message>Assert (1 = 2) failed [val1 = val4]</message></Error>
-<Error><source>user</source><line>50</line><code>100000</code><message>Abc9</message></Error>
-<Error><source>user</source><line>45</line><code>100000</code><message>Assert (2 = 1) failed [val1 = 1]</message></Error>
-<Error><source>user</source><line>46</line><code>100000</code><message>Abc7</message></Error>
+<Error><source>user</source><line>26</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
+<Error><source>user</source><line>27</line><code>100000</code><message>Abc2</message></Error>
+<Error><source>user</source><line>28</line><code>100000</code><message>Assert (1 = 2) failed [1 = val4]</message></Error>
+<Error><source>user</source><line>29</line><code>100000</code><message>Abc3</message></Error>
+<Error><source>user</source><line>36</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
+<Error><source>user</source><line>37</line><code>100000</code><message>Abc5</message></Error>
+<Error><source>user</source><line>38</line><code>100000</code><message>Assert (1 = 2) failed [val1 = val4]</message></Error>
+<Error><source>user</source><line>39</line><code>100000</code><message>Abc6</message></Error>
+<Error><source>user</source><line>34</line><code>100000</code><message>Assert (2 = 1) failed [val1 = 1]</message></Error>
+<Error><source>user</source><line>35</line><code>100000</code><message>Abc4</message></Error>
+<Error><source>user</source><line>46</line><code>100000</code><message>Assert (1 = 2) failed [val1 = 2]</message></Error>
+<Error><source>user</source><line>47</line><code>100000</code><message>Abc8</message></Error>
+<Error><source>user</source><line>48</line><code>100000</code><message>Assert (1 = 2) failed [val1 = val4]</message></Error>
+<Error><source>user</source><line>49</line><code>100000</code><message>Abc9</message></Error>
+<Error><source>user</source><line>44</line><code>100000</code><message>Assert (2 = 1) failed [val1 = 1]</message></Error>
+<Error><source>user</source><line>45</line><code>100000</code><message>Abc7</message></Error>
 <Dataset name='Result 1'>
  <Row><val1>1</val1></Row>
  <Row><val1>2</val1></Row>

testing/ecl/key/dict_index.xml

@@ -1,3 +1,5 @@
+<Dataset name='Result 1'>
+</Dataset>
 <Dataset name='Result 2'>
  <Row><Result_2>4</Result_2></Row>
 </Dataset>
@@ -5,7 +7,7 @@
  <Row><a>1</a><b>2</b><c><Row><c1>f</c1><c2>F</c2></Row><Row><c1>e</c1><c2>E</c2></Row><Row><c1>d</c1><c2>D</c2></Row><Row><c1>3</c1><c2>4</c2></Row><Row><c1>b</c1><c2>B</c2></Row><Row><c1>a</c1><c2>A</c2></Row><Row><c1>c</c1><c2>C</c2></Row></c><d>5</d></Row>
 </Dataset>
 <Dataset name='Result 4'>
- <Row><a> </a><b> </b><c/><d> </d></Row>
+ <Row><a> </a><b> </b><c></c><d> </d></Row>
 </Dataset>
 <Dataset name='Result 5'>
  <Row><a>1</a><b>2</b><c><Row><c1>f</c1><c2>F</c2></Row><Row><c1>e</c1><c2>E</c2></Row><Row><c1>d</c1><c2>D</c2></Row><Row><c1>3</c1><c2>4</c2></Row><Row><c1>b</c1><c2>B</c2></Row><Row><c1>a</c1><c2>A</c2></Row><Row><c1>c</c1><c2>C</c2></Row></c><d>5</d></Row>

testing/ecl/roxie/key/dict_index.xml

@@ -0,0 +1,12 @@
+<Dataset name='Result 2'>
+ <Row><Result_2>4</Result_2></Row>
+</Dataset>
+<Dataset name='Result 3'>
+ <Row><a>1</a><b>2</b><c><Row><c1>f</c1><c2>F</c2></Row><Row><c1>e</c1><c2>E</c2></Row><Row><c1>d</c1><c2>D</c2></Row><Row><c1>3</c1><c2>4</c2></Row><Row><c1>b</c1><c2>B</c2></Row><Row><c1>a</c1><c2>A</c2></Row><Row><c1>c</c1><c2>C</c2></Row></c><d>5</d></Row>
+</Dataset>
+<Dataset name='Result 4'>
+ <Row><a> </a><b> </b><c/><d> </d></Row>
+</Dataset>
+<Dataset name='Result 5'>
+ <Row><a>1</a><b>2</b><c><Row><c1>f</c1><c2>F</c2></Row><Row><c1>e</c1><c2>E</c2></Row><Row><c1>d</c1><c2>D</c2></Row><Row><c1>3</c1><c2>4</c2></Row><Row><c1>b</c1><c2>B</c2></Row><Row><c1>a</c1><c2>A</c2></Row><Row><c1>c</c1><c2>C</c2></Row></c><d>5</d></Row>
+</Dataset>