Merge branch 'candidate-5.2.0'

Signed-off-by: Richard Chapman <rchapman@hpccsystems.com>

CMakeLists.txt

@@ -136,7 +136,7 @@ if ( PLATFORM )
     set(PACKAGE_FILE_NAME_PREFIX "hpccsystems-platform-${projname}")
 else()
     set(CPACK_PACKAGE_NAME "hpccsystems-clienttools-${majorver}.${minorver}")
-    set(PACKAGE_FILE_NAME_PREFIX  "hpccsystems-clienttools")
+    set(PACKAGE_FILE_NAME_PREFIX  "hpccsystems-clienttools-${projname}")
 endif()
 SET(CPACK_PACKAGE_VERSION_MAJOR ${majorver})
 SET(CPACK_PACKAGE_VERSION_MINOR ${minorver})
@@ -309,7 +309,7 @@ if ( PLATFORM )
     set ( CPACK_PACKAGE_DESCRIPTION_SUMMARY "${PACKAGE_FILE_NAME_PREFIX}")
 else ( PLATFORM )
     if ( APPLE OR WIN32 )
-        set ( CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${CPACK_RPM_PACKAGE_VERSION}-${version}-${stagever}${CPACK_SYSTEM_NAME}" )
+        set ( CPACK_PACKAGE_FILE_NAME "${PACKAGE_FILE_NAME_PREFIX}_${version}-${stagever}${CPACK_SYSTEM_NAME}" )
     endif()
     set ( CPACK_MONOLITHIC_INSTALL TRUE )
     file(WRITE "${PROJECT_BINARY_DIR}/welcome.txt"

ecl/ecl-bundle/ecl-bundle.cpp

@@ -103,49 +103,17 @@ static bool versionOk(const char *versionPresent, const char *minOk, const char
 
 unsigned doPipeCommand(StringBuffer &output, const char *cmd, const char *args, const char *input)
 {
-    try
+    VStringBuffer runcmd("%s %s", cmd, args);
+    if (optVerbose)
     {
-        Owned<IPipeProcess> pipe = createPipeProcess();
-        VStringBuffer runcmd("%s %s", cmd, args);
-        pipe->run(cmd, runcmd, ".", input != NULL, true, true, 1024*1024);
-        if (optVerbose)
-        {
-            printf("Running %s\n", runcmd.str());
-            if (input)
-                printf("with input %s\n", input);
-        }
+        printf("Running %s\n", runcmd.str());
         if (input)
-        {
-            pipe->write(strlen(input), input);
-            pipe->closeInput();
-        }
-        char buf[1024];
-        while (true)
-        {
-            size32_t read = pipe->read(sizeof(buf), buf);
-            if (!read)
-                break;
-            output.append(read, buf);
-        }
-        int ret = pipe->wait();
-        StringBuffer error;
-        while (true)
-        {
-            size32_t read = pipe->readError(sizeof(buf), buf);
-            if (!read)
-                break;
-            error.append(read, buf);
-        }
-        if (optVerbose && (ret > 0 || error.length()))
-            printf("%s return code was %d, output to stderr:\n%s", cmd, ret, error.str());
-        return ret;
-    }
-    catch (IException *E)
-    {
-        E->Release();
-        output.clear();
-        return 255;
+            printf("with input %s\n", input);
     }
+    unsigned ret = runExternalCommand(output, runcmd, input);
+    if (optVerbose && (ret > 0))
+        printf("%s return code was %d", cmd, ret);
+    return ret;
 }
 
 static bool platformVersionDone = false;

ecllibrary/std/Metaphone.ecl

@@ -0,0 +1,41 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.  All rights reserved.
+############################################################################## */
+
+
+EXPORT Metaphone := MODULE
+
+
+IMPORT lib_metaphone;
+
+/**
+ * Returns the primary metaphone value
+ *
+ * @param src           The string whose metphone is to be calculated.
+ * @see                 http://en.wikipedia.org/wiki/Metaphone#Double_Metaphone
+ */
+
+EXPORT String primary(STRING src) :=
+  lib_metaphone.MetaphoneLib.DMetaphone1(src);
+
+/**
+ * Returns the secondary metaphone value
+ *
+ * @param src           The string whose metphone is to be calculated.
+ * @see                 http://en.wikipedia.org/wiki/Metaphone#Double_Metaphone
+ */
+
+EXPORT String secondary(STRING src) :=
+  lib_metaphone.MetaphoneLib.DMetaphone2(src);
+
+/**
+ * Returns the double metaphone value (primary and secondary concatenated
+ *
+ * @param src           The string whose metphone is to be calculated.
+ * @see                 http://en.wikipedia.org/wiki/Metaphone#Double_Metaphone
+ */
+
+EXPORT String double(STRING src) :=
+  lib_metaphone.MetaphoneLib.DMetaphoneBoth(src);
+
+END;

ecllibrary/teststd/Metaphone/TestMetaphone.ecl

@@ -0,0 +1,15 @@
+/*##############################################################################
+## HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.  All rights reserved.
+############################################################################## */
+IMPORT Std.Metaphone;
+
+EXPORT TestMetaphone := MODULE
+
+  EXPORT TestConst := MODULE
+    EXPORT Test01 := ASSERT(Metaphone.primary('Algernon') = 'ALKRNN');
+    EXPORT Test02 := ASSERT(Metaphone.secondary('Algernon') = 'ALJRNN');
+    EXPORT Test03 := ASSERT(Metaphone.double('Algernon') = 'ALKRNNALJRNN');
+  END;
+
+  EXPORT Main := [EVALUATE(TestConst)];
+END;

esp/bindings/http/platform/httpservice.cpp

@@ -653,182 +653,6 @@ bool skipHeader(const char *name)
     return false;
 }
 
-typedef enum _cgi_resp_state
-{
-    cgi_resp_hname,
-    cgi_resp_hval,
-    cgi_resp_body
-} cgi_resp_state;
-
-
-int CEspHttpServer::onRunCGI(CHttpRequest* request, CHttpResponse* response, const char *path)
-{
-    char cwd[1024];
-    if (!GetCurrentDirectory(1024, cwd)) {
-        ERRLOG("onRunCGI: Current directory path too big, setting local path to null");
-        cwd[0] = 0;
-    }
-    StringBuffer docRoot(cwd);
-    docRoot.append("/files");
-    StringBuffer script(docRoot);
-    script.append("/").append(path);
-
-    StringArray env;
-    StringBuffer var;
-
-    make_env_var(env, var, "SERVER_SOFTWARE", "ESP/1.0");
-    make_env_var(env, var, "GATEWAY_INTERFACE", "CGI/1.1");
-    make_env_var(env, var, "SERVER_PROTOCOL", "HTTP/1.0");
-    make_env_var(env, var, "DOCUMENT_ROOT", docRoot);
-    make_env_var(env, var, "SCRIPT_FILENAME", script);
-    make_env_var(env, var, "REQUEST_METHOD", request->queryMethod());
-    make_env_var(env, var, "PATH", getenv("PATH"));
-    make_env_var(env, var, "QUERY_STRING", request->queryParamStr());
-    make_env_var(env, var, "REDIRECT_STATUS", 1);
-
-    ISocket *sock=request->getSocket();
-    if (sock)
-    {
-        char sname[512]={0};
-        int sport = sock->name(sname, 512);
-        SocketEndpoint ep;
-        sock->getPeerEndpoint(ep);
-        StringBuffer ipstr;
-        ep.getIpText(ipstr);
-        make_env_var(env, var, "REMOTE_ADDRESS", ipstr.str());
-        make_env_var(env, var, "REMOTE_PORT", ep.port);
-        make_env_var(env, var, "SERVER_PORT", sport);
-    }
-
-    IEspContext *ctx=request->queryContext();
-    if (ctx)
-    {
-        StringBuffer userstr(ctx->queryUserId());
-        if (userstr.length())
-            make_env_var(env, var, "REMOTE_USER", userstr);
-    }
-    make_env_var(env, var, "REQUEST_URI", request->queryPath());
-    make_env_var(env, var, "SCRIPT_NAME", request->queryPath());
-    
-
-    StringBuffer hostIpStr;
-    queryHostIP().getIpText(hostIpStr);
-    
-    make_env_var(env, var, "SERVER_ADDR", hostIpStr);
-    make_env_var(env, var, "SERVER_NAME", hostIpStr);
-    
-    if (!stricmp(request->queryMethod(), "POST"))
-    {
-        if (request->getContentLength())
-        {
-            StringBuffer type;
-            make_env_var(env, var, "CONTENT_TYPE", request->getContentType(type).str());
-            make_env_var(env, var, "CONTENT_LENGTH", request->getContentLength());
-        }
-    }
-
-    make_env_var(env, var, "SCRIPT_FILENAME", script);
-
-    StringArray &headers=request->queryHeaders();
-    ForEachItemIn(iheader, headers)
-    {
-        const char* header = headers.item(iheader);
-        if (header)
-        {
-            const char* colon = strchr(header, ':');
-            if(colon)
-            {
-                StringBuffer hname(colon-header, header);
-                hname.toUpperCase().replace('-','_');
-                if (!skipHeader(hname.str()))
-                {
-                    const char *finger=colon+1;
-                    while (*finger==' ') finger++;
-                    StringBuffer hstr;
-                    hstr.append(("HTTP_")).append(hname).append('=').append(finger);
-                    env.append(hstr.str());
-                    DBGLOG("%s", hstr.str());
-                }
-            }
-        }
-    }
-    
-    make_env_var(env, var, "XXX", "222 Running the script => ");
-
-    StringBuffer in;
-    StringBuffer out;
-    callExternalProgram("php-cgi.exe", in, out, &env);
-    const char *start=out.str();
-    cgi_resp_state rstate=cgi_resp_hname;
-
-    StringBuffer hname;
-    StringBuffer hval;
-    const char *finger;
-    for(finger=start; *finger!=0 && rstate!=cgi_resp_body; finger++)
-    {
-        switch (*finger)
-        {
-        case ':':
-            if (rstate==cgi_resp_hname)
-                rstate=cgi_resp_hval;
-            else if (rstate==cgi_resp_hval)
-                hval.append(':');
-            break;
-        case '\r':
-            if (!strncmp(finger, "\r\n\r\n", 4))
-            {
-                finger+=3;
-                rstate=cgi_resp_body;
-            }
-            else
-            {
-                if (hname.length() && hval.length())
-                {
-                    if (!stricmp(hname.str(), "content-type"))
-                        response->setContentType(strchr(hval.str(), '/') ? hval.str() : "text/html");
-                    else
-                        response->setHeader(hname.str(), hval.str());
-                }
-                hname.clear();
-                hval.clear();
-                rstate=cgi_resp_hname;
-            }
-            break;
-
-        case '\n':
-            if (finger[1]=='\n')
-            {
-                finger++;
-                rstate=cgi_resp_body;
-            }
-            else
-            {
-                if (hname.length() && hval.length())
-                {
-                    if (!stricmp(hname.str(), "content-type"))
-                        response->setContentType(strchr(hval.str(), '/') ? hval.str() : "text/html");
-                    else
-                        response->setHeader(hname.str(), hval.str());
-                }
-                hname.clear();
-                hval.clear();
-                rstate=cgi_resp_hname;
-            }
-            break;
-        default:
-            if (rstate==cgi_resp_hname)
-                hname.append(*finger);
-            else if (rstate==cgi_resp_hval)
-                hval.append(*finger);
-            break;
-        }
-    }
-
-    response->setContent(out.length()-(finger-start), finger);
-    response->send();
-    return 0;
-}
-
 static void httpGetFile(CHttpRequest* request, CHttpResponse* response, const char *urlpath, const char *filepath)
 {
     StringBuffer mimetype, etag, lastModified;
@@ -913,9 +737,6 @@ int CEspHttpServer::onGetFile(CHttpRequest* request, CHttpResponse* response, co
         StringBuffer tail;
         splitFilename(urlpath, NULL, NULL, &tail, &ext);
 
-        if (strieq(ext, ".php"))
-            return onRunCGI(request, response, urlpath);
-
         bool top = !urlpath || !*urlpath;
         StringBuffer httpPath;
         request->getPath(httpPath).str();

esp/bindings/http/platform/httpservice.hpp

@@ -78,7 +78,6 @@ public:
     virtual int onGetDynNavData(CHttpRequest* request, CHttpResponse* response);
     virtual int onGetNavEvent(CHttpRequest* request, CHttpResponse* response);
     virtual int onGetMainWindow(CHttpRequest* request, CHttpResponse* response);
-    virtual int onRunCGI(CHttpRequest* request, CHttpResponse* response, const char *path);
 #ifdef _USE_OPENLDAP
     virtual int onUpdatePasswordInput(CHttpRequest* request, CHttpResponse* response);
     virtual int onUpdatePassword(CHttpRequest* request, CHttpResponse* response);

esp/services/esdl_svc_engine/CMakeLists.txt

@@ -52,6 +52,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/ecl/hql
          ${HPCC_SOURCE_DIR}/rtl/include
          ${HPCC_SOURCE_DIR}/esp/esdllib
+         ${HPCC_SOURCE_DIR}/esp/logging
     )
 
 ADD_DEFINITIONS( -D_USRDLL )

esp/services/esdl_svc_engine/esdl_binding.cpp

@@ -164,6 +164,37 @@ bool loadDefinitions(const char * espServiceName, IEsdlDefinition * esdl, IPrope
     return true;
 }
 
+
+bool EsdlServiceImpl::loadLogggingManager()
+{
+    if (!loggingManager)
+    {
+        StringBuffer realName;
+        realName.append(SharedObjectPrefix).append(LOGGINGMANAGERLIB).append(SharedObjectExtension);
+
+        HINSTANCE loggingManagerLib = LoadSharedObject(realName.str(), true, false);
+
+        if(loggingManagerLib == NULL)
+        {
+            ESPLOG(LogNormal,"ESP service %s: cannot load logging manager library(%s)", m_espServiceName.str(), realName.str());
+            return false;
+        }
+
+        newLoggingManager_t_ xproc = NULL;
+        xproc = (newLoggingManager_t_)GetSharedProcedure(loggingManagerLib, "newLoggingManager");
+
+        if (!xproc)
+        {
+            ESPLOG(LogNormal,"ESP service %s: procedure newLogggingManager of %s can't be loaded\n", m_espServiceName.str(), realName.str());
+            return false;
+        }
+
+        loggingManager.setown((ILoggingManager*) xproc());
+    }
+
+    return true;
+}
+
 void EsdlServiceImpl::init(const IPropertyTree *cfg,
                            const char *process,
                            const char *service)
@@ -180,6 +211,19 @@ void EsdlServiceImpl::init(const IPropertyTree *cfg,
         m_espServiceType.set(srvcfg->queryProp("@type"));
         if (m_espServiceType.length() <= 0)
             throw MakeStringException(-1, "Could not determine ESDL service configuration type: esp process '%s' service name '%s'", process, service);
+
+        //Rodrigo: this will depend on how Kevin/Gleb structure the configuration
+        IPropertyTree* loggingConfig = srvcfg->queryPropTree("LoggingManager");
+        if (loggingConfig)
+        {
+            ESPLOG(LogNormal, "ESP Service %s attempting to load configured logging manager.", service);
+            if (loadLogggingManager())
+                loggingManager->init(loggingConfig, service);
+            else
+                throw MakeStringException(-1, "ESDL Service %s could not load logging manager", service);
+        }
+        else
+            ESPLOG(LogNormal, "ESP Service %s is not attached to any logging manager.", service);
     }
     else
         throw MakeStringException(-1, "Could not access ESDL service configuration: esp process '%s' service name '%s'", process, service);
@@ -307,8 +351,21 @@ void EsdlServiceImpl::handleServiceRequest(IEspContext &context,
     else
         m_pEsdlTransformer->process(context, EsdlResponseMode, srvdef.queryName(), mthdef.queryName(), out, soapresp.str(), ESDL_TRANS_OUTPUT_ROOT, ns, schema_location);
 
+    handleResultLogging(context, tgtcfg.get(), req,  soapresp.str(), out.str());
     ESPLOG(LogMax,"Customer Response: %s", out.str());
-    ESPLOG(LogMax,"Generic log data: %s", logdata.str());
+}
+
+bool EsdlServiceImpl::handleResultLogging(IEspContext &espcontext, IPropertyTree * reqcontext, IPropertyTree * request,  const char * rawresp, const char * finalresp)
+{
+    bool success = true;
+    if (loggingManager)
+    {
+        StringBuffer logresp;
+        success = loggingManager->updateLog(LOGGINGDBSINGLEINSERT, espcontext, reqcontext, request, rawresp, finalresp, logresp);
+        ESPLOG(LogMin,"ESDLService: Attempted to log ESP transaction: %s", logresp.str());
+    }
+
+    return success;
 }
 
 void EsdlServiceImpl::getSoapBody(StringBuffer& out,StringBuffer& soapresp)

esp/services/esdl_svc_engine/esdl_binding.hpp

@@ -24,6 +24,7 @@
 #include "dasds.hpp"
 #include "jptree.hpp"
 #include "xsdparser.hpp"
+#include "loggingmanager.h"
 
 static const char* ESDL_DEFS_ROOT_PATH="/ESDL/Definitions/";
 static const char* ESDL_DEF_PATH="/ESDL/Definitions/Definition";
@@ -64,6 +65,7 @@ class EsdlServiceImpl : public CInterface, implements IEspService
 private:
     IEspContainer *container;
     MapStringToMyClass<ISmartSocketFactory> connMap;
+    Owned<ILoggingManager> loggingManager;
 
 public:
     StringBuffer                m_espServiceType;
@@ -108,6 +110,7 @@ public:
             m_pServiceMethodTargets.clear();
     }
 
+    virtual bool loadLogggingManager();
     virtual void init(const IPropertyTree *cfg, const char *process, const char *service);
     virtual void configureTargets(IPropertyTree *cfg, const char *service);
     virtual void handleServiceRequest(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, Owned<IPropertyTree> &tgtcfg, Owned<IPropertyTree> &tgtctx, const char *ns, const char *schema_location, IPropertyTree *req, StringBuffer &out, StringBuffer &logdata, unsigned int flags);
@@ -122,7 +125,7 @@ public:
     virtual void processRequest(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, const char *ns, StringBuffer &req, StringBuffer &hashedReq) {};
     virtual void processResponse(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, const char *ns, StringBuffer &resp,StringBuffer &hashedReq) {};
     virtual void createServersList(IEspContext &context, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, StringBuffer &servers) {};
-
+    virtual bool handleResultLogging(IEspContext &espcontext, IPropertyTree * reqcontext, IPropertyTree * request,  const char * rawresp, const char * finalresp);
     void handleEchoTest(const char *mthName, IPropertyTree *req, StringBuffer &soapResp, unsigned flags=0);
     virtual void handleFinalRequest(IEspContext &context, Owned<IPropertyTree> &tgtcfg, Owned<IPropertyTree> &tgtctx, IEsdlDefService &srvdef, IEsdlDefMethod &mthdef, const char *ns, StringBuffer& req, StringBuffer &out, bool isroxie, bool isproxy);
     void getSoapBody(StringBuffer& out,StringBuffer& soapresp);

esp/services/ws_esdlconfig/CMakeLists.txt

@@ -24,8 +24,6 @@
 
 project( ws_esdlconfig )
 
-
-#include(${HPCC_LN_SOURCE_DIR}/esp/scm/lnespscm.cmake)
 include(${HPCC_SOURCE_DIR}/esp/scm/espscm.cmake)
 
 set (    SRCS
@@ -52,6 +50,7 @@ include_directories (
          ${HPCC_SOURCE_DIR}/esp/smc/SMCLib
          ${HPCC_SOURCE_DIR}/esp/services/esdl_svc_engine
          ${HPCC_SOURCE_DIR}/esp/esdllib
+         ${HPCC_SOURCE_DIR}/esp/logging
     )
 
 ADD_DEFINITIONS( -D_USRDLL )

initfiles/componentfiles/configxml/@temp/esp_service_DynamicESDL.xsl

@@ -53,7 +53,87 @@
                 <xsl:with-param name="plugin" select="Properties/@plugin"/>
             </xsl:call-template>
         </xsl:variable>
-        <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}"/>
+        <EspService name="{$serviceName}" type="{$serviceType}" plugin="{$servicePlugin}">
+            <xsl:if test="string(@LoggingManager) != ''">
+                <xsl:variable name="managerName" select="@LoggingManager"/>
+                <xsl:variable name="managerNode" select="/Environment/Software/LoggingManager[@name=$managerName]"/>
+
+                <xsl:if test="not($managerNode)">
+                    <xsl:message terminate="yes">Logging Manager is undefined!</xsl:message>
+                </xsl:if>
+
+                <xsl:if test="not($managerNode/ESPLoggingAgent/@ESPLoggingAgent[1])">
+                     <xsl:message terminate="yes">ESP Logging Agent <xsl:value-of select="$managerNode/@ESPLoggingAgent"/> is undefined for <xsl:value-of select="$managerName"/> !</xsl:message>
+                </xsl:if>
+
+                <LoggingManager name="{$managerNode/@name}">
+
+                <xsl:for-each select="$managerNode/ESPLoggingAgent">
+                    <xsl:variable name="agentName" select="@ESPLoggingAgent"/>
+                    <xsl:variable name="agentNode" select="/Environment/Software/ESPLoggingAgent[@name=$agentName]"/>
+
+                    <xsl:if test="not($agentNode)">
+                        <xsl:message terminate="yes">An ESP Logging Agent <xsl:value-of select="$agentName"/>  for <xsl:value-of select="$managerNode/@name"/> is undefined!</xsl:message>
+                    </xsl:if>
+
+                    <xsl:if test="string($agentNode/@ESPServer) = ''">
+                        <xsl:message terminate="yes">ESP server is undefined for <xsl:value-of select="$agentName"/> </xsl:message>
+                    </xsl:if>
+
+                    <xsl:variable name="espServer" select="$agentNode/@ESPServer"/>
+                    <xsl:variable name="espNode" select="/Environment/Software/EspProcess[@name=$espServer]"/>
+
+                    <xsl:if test="not($espNode)">
+                        <xsl:message terminate="yes">ESP process for <xsl:value-of select="$agentName"/> is undefined!</xsl:message>
+                    </xsl:if>
+
+                    <xsl:variable name="espPort" select="$espNode/EspBinding[@service='wslogging']/@port"/>
+
+                    <xsl:if test="string($espPort) = ''">
+                        <xsl:message terminate="yes">ESP server port is undefined for <xsl:value-of select="$espServer"/>!</xsl:message>
+                    </xsl:if>
+
+                    <xsl:variable name="espNetAddress" select="$espNode/Instance/@netAddress"/>
+
+                    <xsl:if test="string($espNetAddress) = ''">
+                        <xsl:message terminate="yes">ESP NetAddress is undefined!</xsl:message>
+                    </xsl:if>
+                    <xsl:variable name="wsloggingUrl"><xsl:text>http://</xsl:text><xsl:value-of select="$espNetAddress"/><xsl:text>:</xsl:text><xsl:value-of select="$espPort"/></xsl:variable>
+                        <LogAgent name="{$agentName}" type="LogAgent" services="GetTransactionSeed,UpdateLog" plugin="espserverloggingagent">
+                            <ESPServer url="{$wsloggingUrl}" user="{$agentNode/@User}" password="{$agentNode/@Password}"/>
+                            <xsl:if test="string($agentNode/@MaxServerWaitingSeconds) != ''">
+                                <MaxServerWaitingSeconds><xsl:value-of select="$agentNode/@MaxServerWaitingSeconds"/></MaxServerWaitingSeconds>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@FailSafe) != ''">
+                                <FailSafe><xsl:value-of select="$agentNode/@FailSafe"/></FailSafe>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@FailSafeLogsDir) != ''">
+                                <FailSafeLogsDir><xsl:value-of select="$agentNode/@FailSafeLogsDir"/></FailSafeLogsDir>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@MaxLogQueueLength) != ''">
+                                <MaxLogQueueLength><xsl:value-of select="$agentNode/@MaxLogQueueLength"/></MaxLogQueueLength>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@MaxTriesGTS) != ''">
+                                <MaxTriesGTS><xsl:value-of select="$agentNode/@MaxTriesGTS"/></MaxTriesGTS>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@MaxTriesRS) != ''">
+                                <MaxTriesRS><xsl:value-of select="$agentNode/@MaxTriesRS"/></MaxTriesRS>
+                            </xsl:if>
+                            <xsl:if test="string($agentNode/@QueueSizeSignal) != ''">
+                                <QueueSizeSignal><xsl:value-of select="$agentNode/@QueueSizeSignal"/></QueueSizeSignal>
+                            </xsl:if>
+                            <Filters>
+                                <xsl:for-each select="$agentNode/Filter">
+                                    <Filter value="{current()/@filter}" type="{current()/@type}"/>
+                                </xsl:for-each>
+                            </Filters>
+                        </LogAgent>
+                </xsl:for-each>
+
+                </LoggingManager>
+
+            </xsl:if>
+        </EspService>
         <EspBinding name="{$bindName}" service="{$serviceName}" protocol="{$bindingNode/@protocol}" type="{$bindType}" plugin="{$servicePlugin}" netAddress="0.0.0.0" port="{$bindingNode/@port}" defaultBinding="true">
             <xsl:call-template name="bindAuthentication">
                 <xsl:with-param name="bindingNode" select="$bindingNode"/>

initfiles/componentfiles/configxml/esdlsvcengine.xsd

@@ -35,6 +35,13 @@
                     </xs:appinfo>
                 </xs:annotation>
             </xs:attribute>
+            <xs:attribute name="LoggingManager" type="loggingmanagerType" use="optional">
+                <xs:annotation>
+                    <xs:appinfo>
+                        <tooltip>Specifies the Logging Manager.</tooltip>
+                    </xs:appinfo>
+                </xs:annotation>
+            </xs:attribute>
         </xs:complexType>
     </xs:element>
 </xs:schema>

initfiles/sbin/configmgr.in

@@ -21,6 +21,8 @@
 
 ###<REPLACE>###
 
+DEBUG=${DEBUG:-NO_DEBUG}
+
 createConf ()
 {
     awk -f ${reg_path}/regex.awk -v NEW_ENVFILE=$1 -v NEW_PORT=$2 -v NEW_CONFFILE=$3< ${path}${configs}/configmgr/esp.xml >${runtime}/${compName}/esp.xml
@@ -29,7 +31,7 @@ createConf ()
 removePidfiles ()
 {
     rm -rf ${runtime}/${compName}.pid
-    rm -rf ${lock}/${compName}.lock
+    rm -rf ${lock}/${compName}/${compName}.lock
     rm -rf ${pid}/${compName}_init.pid
     rm -rf ${pid}/${compName}.pid
     rm -rf ${pid}/init_${compName}.pid
@@ -120,7 +122,7 @@ logFile=${log}/${compName}/${compName}.log
 
 initPidFile=${pid}/${compName}_init.pid
 compPidFile=${pid}/${compName}.pid
-lockFile=${lock}/${compName}.lock
+lockFile=${lock}/${compName}/${compName}.lock
 defaultEnv=0
 
 # Checking input arguments
@@ -196,6 +198,14 @@ startcmd="${START_STOP_DAEMON} -S -p ${initPidFile} -c ${user}:${group} -d ${run
 eval ${startcmd}
 started=$?
 
+# Creating a Lock
+lockPath=${lock}/${compName}
+if [ ! -d $lockPath ]; then
+  mkdir -p $lockPath >> $logFile 2>&1
+fi
+chown -c $user:$group $lockPath >> /dev/null 2>&1
+lock $lockFile
+
 if [ ${DEBUG:-NO_DEBUG} != "NO_DEBUG" ]; then
     echo $startcmd
     echo "configesp status = $started"

initfiles/sbin/install-cluster.sh.in

@@ -61,7 +61,9 @@ getUserAndPasswd(){
        read -s PASS
        echo ""
 
-       echo "You entered user $ADMIN_USER and a password ($(echo $PASS | sed 's/./\./g'))."
+       password_string="and a password ($(echo $PASS | sed 's/./\./g'))."
+       [ -z "$password" ] && password_string="with an empty password for passwordless login."
+       echo "You entered user $ADMIN_USER $password_string"
        read -p  "Are these correct? [Y|n] " answer
        if [ "$answer" = "Y" ] || [ "$answer" = "y" ]
        then

initfiles/sbin/install-hpcc.exp

@@ -34,6 +34,21 @@ proc print_usage {} {
     exit 1
 }
 
+proc sshcmd { cmd params_string } {
+  global password user
+  if {[string length $password] == 0 } {
+     set cmd2 "$cmd -o BatchMode=yes -o StrictHostKeyChecking=no"
+     if {[string compare $::env(USER) $user] == 0} {
+        return "$cmd2 $params_string"
+     } else {
+        return "su $user -c \"$cmd2  [regsub -all "\"" $params_string \\\"]\""
+     }
+  } else {
+     return "$cmd $params_string"
+  }
+}
+
+
 
 proc testWithPing {} {
    global ip
@@ -54,7 +69,7 @@ proc checkSSHConnection {} {
 
    set timeout 60
    # JIRA 12585 in chance root user cannot ssh with key pairs
-   spawn su ${user} -c "ssh ${user}@${ip}"
+   eval "spawn [sshcmd ssh "${user}@${ip}"]"
    expect {
       *?assword:* {
          send "${password}\r";
@@ -83,7 +98,7 @@ proc copyPayload {} {
    global ip user password prompt
 
    set timeout 300
-   spawn su ${user} -c "scp /tmp/remote_install.tgz ${user}@${ip}:~;"
+   eval "spawn [sshcmd scp "/tmp/remote_install.tgz ${user}@${ip}:~;"]"
    expect {
       *?assword:* {
          send "${password}\r";
@@ -110,7 +125,7 @@ proc expandPayload {} {
    global ip user password prompt
 
    set timeout 180
-   spawn su ${user} -c "ssh ${user}@${ip} \"cd /; tar -zxf ~/remote_install.tgz\""
+   eval "spawn [sshcmd ssh "${user}@${ip} \"cd /; tar -zxf ~/remote_install.tgz\""]"
    expect {
       *?assword:* {
          send "${password}\r"
@@ -140,7 +155,7 @@ proc runPayload {} {
    set timeout 60
 
    expect -re $
-   spawn su ${user} -c "ssh ${user}@${ip}"
+   eval "spawn [sshcmd ssh ${user}@${ip}]"
    expect {
       *?assword:* {
          send "${password}\r"

plugins/CMakeLists.txt

@@ -15,6 +15,7 @@
 ################################################################################
 add_subdirectory (auditlib)
 add_subdirectory (debugservices)
+add_subdirectory (dmetaphone)
 add_subdirectory (fileservices)
 add_subdirectory (logging)
 add_subdirectory (parselib)

plugins/cassandra/cpp-driver

@@ -1 +1 @@
-Subproject commit 9bb19cf77f497768bef8364e68256ca06632b3f1
+Subproject commit 189e46e3386cc2be0fd7428c1072339b9c611d6e

plugins/common/cstring.h

@@ -0,0 +1,215 @@
+//  cstrings.h  support for C++ cString class
+//
+#ifndef _CSTRING_H
+#define _CSTRING_H
+
+#include <string.h>
+#include <ctype.h>
+
+#define CSINITIALSIZE 8         // size of statically allocated string buffer
+
+class cString
+{
+private:
+    int     iBufLen;        // current buffer length
+    char    buffer[CSINITIALSIZE];
+
+protected:
+    void Set(const char *Str) {
+        if (!Str) {
+            // treat null address as a null string
+            Len = 0;
+            *Ptr = '\0';
+            return;
+        }
+        Len = strlen( Str );
+        if (Len >= iBufLen && Len + 1 > CSINITIALSIZE) {
+            if (iBufLen != CSINITIALSIZE) {
+                delete [] Ptr;
+            }
+            iBufLen = Len + 1;
+            Ptr = new char[iBufLen];
+        }
+        memcpy( Ptr, Str, Len+1 );
+    }
+
+public:
+    int     Len;            // current string length
+    char    *Ptr;           // current string buffer
+    cString() {
+        iBufLen = CSINITIALSIZE;
+        Len = 0;
+        Ptr = buffer;
+        *Ptr = '\0';
+    }
+
+    cString(const char *Str) {
+        Len = strlen( Str );
+        iBufLen = Len + 1;
+        if (iBufLen <= CSINITIALSIZE) {
+            iBufLen = CSINITIALSIZE;
+            Ptr = buffer;
+        } else {
+            Ptr = new char[ iBufLen ];
+        }
+        memcpy( Ptr, Str, Len+1 );
+    }
+
+    cString(const cString &cStr) {
+        Len = cStr.Len;
+        iBufLen = Len + 1;
+        if (iBufLen <= CSINITIALSIZE) {
+            iBufLen = CSINITIALSIZE;
+            Ptr = buffer;
+        } else {
+            Ptr = new char[ iBufLen ];
+        }
+        memcpy( Ptr, cStr.Ptr, Len+1 );
+    }
+
+    inline ~cString() {
+        if (iBufLen != CSINITIALSIZE) {
+            delete [] Ptr;
+        }
+    }
+
+    void Set(const char *Str, int len) {
+        Len = len;
+        if (Len >= iBufLen && Len + 1 > CSINITIALSIZE) {
+            if (iBufLen != CSINITIALSIZE) {
+                delete [] Ptr;
+            }
+            iBufLen = Len + 1;
+            Ptr = new char[iBufLen];
+        }
+        memcpy( Ptr, Str, Len );
+        Ptr[Len] = 0;
+    }
+
+    void SetLength(int len) {
+        Len = len;
+        if (Len >= iBufLen && Len + 1 > CSINITIALSIZE) {
+            if (iBufLen != CSINITIALSIZE) {
+                delete [] Ptr;
+            }
+            iBufLen = Len + 1;
+            Ptr = new char[iBufLen];
+        }
+    }
+
+
+    void Trim() {
+        if (Len) {
+            char    *sp;
+            sp = Ptr + Len - 1;
+            while (Len) {
+                if (*sp == ' ') {
+                    Len--;
+                    sp--;
+                } else {
+                    break;
+                }
+            }
+            *(sp+1) = 0;
+        }
+    }
+
+    void Upper() {
+        char *sp = Ptr;
+        char *ep = Ptr + Len;
+
+        while (sp < ep) {
+            *sp = toupper(*sp);
+            sp++;
+        }
+    }
+
+    void Lower() {
+        char *sp = Ptr;
+        char *ep = Ptr + Len;
+
+        while (sp < ep) {
+            *sp = tolower(*sp);
+            sp++;
+        }
+    }
+
+    // Concatenate a string to the existing string
+    void Cat(const char *string, int tlen) {
+        if (Len + tlen >= iBufLen && Len + tlen + 1 >= CSINITIALSIZE) {
+            char *tPtr = new char[Len + tlen + 1];
+            memcpy( tPtr, Ptr, Len );
+            if(iBufLen != CSINITIALSIZE)
+                delete [] Ptr;
+            iBufLen = Len + tlen + 1;
+            Ptr = tPtr;
+            Ptr[Len] = 0;
+        }
+        memcpy( Ptr + Len, string, tlen );
+        Len += tlen;
+        Ptr[Len] = 0;
+    }
+
+    inline void Cat(cString &string) {
+        Cat(string.Ptr, string.Len);
+    }
+
+    inline void Cat( const char *string ) {
+        Cat( string, strlen(string));
+    }
+
+    inline cString& operator =(const char *Str) {
+        Set(Str);
+        return(*this);
+    }
+
+    inline cString& operator=(cString &cStr) {
+        Set(cStr.Ptr,cStr.Len);
+        return *this;
+    }
+
+    cString& operator+=(cString &cStr) {
+        if (Len + cStr.Len >= iBufLen && Len + cStr.Len + 1 > CSINITIALSIZE) {
+            char *tPtr = new char[iBufLen + cStr.Len + 1];
+            memcpy( tPtr, Ptr, Len );
+            if(iBufLen != CSINITIALSIZE)
+                delete [] Ptr;
+            iBufLen = Len + cStr.Len + 1;
+            Ptr = tPtr;
+        }
+        memcpy(Ptr+Len,cStr.Ptr,cStr.Len+1);
+        Len += cStr.Len;
+        return *this;
+    }
+
+    char *operator+=(const char *Str) {
+        int slen = strlen(Str);
+        if (Len + slen >= iBufLen && Len + slen + 1 > CSINITIALSIZE) {
+            char *tPtr = new char[iBufLen + slen + 1];
+            memcpy( tPtr, Ptr, Len );
+            if(iBufLen != CSINITIALSIZE)
+                delete [] Ptr;
+            iBufLen = Len + slen + 1;
+            Ptr = tPtr;
+        }
+        memcpy(Ptr+Len, Str, slen+1);
+        Len += slen;;
+        return Ptr;
+    }
+
+    cString& operator+=(const char ch)  {
+        if (Len + 1 < iBufLen) {
+            Ptr[Len++] = ch;
+            Ptr[Len] = '\0';
+        } else {
+            Cat(&ch, 1);
+        }
+        return(*this);
+    }
+
+    inline operator char*() {
+        return(Ptr);
+    }
+};
+
+#endif  //_CSTRING_H

plugins/dmetaphone/CMakeLists.txt

@@ -0,0 +1,30 @@
+# Component: dmetaphone
+
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for dmetaphone
+#####################################################
+
+set ( toolsdir "${HPCC_SOURCE_DIR}/tools" )
+
+
+project( dmetaphone )
+
+set (    SRCS
+         dmetaphone.cpp
+         metaphone.cpp
+    )
+
+include_directories (
+         ${HPCC_SOURCE_DIR}/plugins/common
+         ${HPCC_SOURCE_DIR}/system/include
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DDMETAPHONE_EXPORTS )
+
+HPCC_ADD_LIBRARY( dmetaphone SHARED ${SRCS} )
+install ( TARGETS dmetaphone DESTINATION plugins )
+target_link_libraries ( dmetaphone
+         jlib
+    )

plugins/dmetaphone/dmetaphone.cpp

@@ -0,0 +1,118 @@
+#include "platform.h"
+#include <time.h>
+#include <stdlib.h>
+#include <string.h>
+#include "dmetaphone.hpp"
+#include "metaphone.h"
+
+#define DMETAPHONE_VERSION "DMETAPHONE 1.1.05"
+
+static const char * compatibleVersions[] = {
+    "DMETAPHONE 1.1.05 [0e64c86ec1d5771d4ce0abe488a98a2a]",
+    "DMETAPHONE 1.1.05",
+    NULL };
+
+DMETAPHONE_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb)
+{
+    if (pb->size == sizeof(ECLPluginDefinitionBlockEx))
+    {
+        ECLPluginDefinitionBlockEx * pbx = (ECLPluginDefinitionBlockEx *) pb;
+        pbx->compatibleVersions = compatibleVersions;
+    }
+    else if (pb->size != sizeof(ECLPluginDefinitionBlock))
+        return false;
+
+    pb->magicVersion = PLUGIN_VERSION;
+    pb->version = DMETAPHONE_VERSION;
+    pb->moduleName = "lib_metaphone";
+    pb->ECL = NULL;  // Definition is in lib_metaphone.ecllib
+    pb->flags = PLUGIN_IMPLICIT_MODULE;
+    pb->description = "Metaphone library";
+    return true;
+}
+
+namespace nsDmetaphone {
+
+IPluginContext * parentCtx = NULL;
+
+}
+
+using namespace nsDmetaphone;
+
+DMETAPHONE_API void setPluginContext(IPluginContext * _ctx) { parentCtx = _ctx; }
+
+
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone1(size32_t & __ret_len,char * & __ret_str,unsigned _len_instr,const char * instr)
+{
+    cString metaph;
+    cString metaph2;
+    MString ms;
+    ms.Set(instr, _len_instr);
+    ms.DoubleMetaphone(metaph, metaph2);
+    __ret_len = strlen((char*) metaph);
+    __ret_str = (char *) CTXMALLOC(parentCtx, __ret_len+1);
+    strcpy(__ret_str, (char*) metaph);
+}
+
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone2(size32_t & __ret_len,char * & __ret_str,unsigned _len_instr,const char * instr)
+{
+    cString metaph;
+    cString metaph2;
+    MString ms;
+    ms.Set(instr, _len_instr);
+    ms.DoubleMetaphone(metaph, metaph2);
+    __ret_len = strlen((char*) metaph2);
+    __ret_str = (char *) CTXMALLOC(parentCtx, __ret_len+1);
+    strcpy(__ret_str, (char*) metaph2);
+}
+
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphoneBoth(size32_t & __ret_len,char * & __ret_str,unsigned _len_instr,const char * instr)
+{
+    cString metaph;
+    cString metaph2;
+    MString ms;
+    ms.Set(instr, _len_instr);
+    ms.DoubleMetaphone(metaph, metaph2);
+    __ret_len = strlen((char*) metaph) + strlen((char*) metaph2);
+    __ret_str = (char *) CTXMALLOC(parentCtx, __ret_len+1);
+    strcpy(__ret_str, (char*) metaph);
+    strcat(__ret_str, (char*) metaph2);
+}
+
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone1_20(char * __ret_str,unsigned _len_instr,const char * instr)
+{
+    cString metaph;
+    cString metaph2;
+    MString ms;
+    ms.Set(instr, _len_instr);
+    ms.DoubleMetaphone(metaph, metaph2);
+    memset(__ret_str, ' ', 20);
+    size32_t metaph_len = strlen((char*) metaph);
+    strncpy(__ret_str, (char*) metaph, (metaph_len > 20)?20:metaph_len);
+}
+
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone2_20(char * __ret_str,unsigned _len_instr,const char * instr)
+{
+    cString metaph;
+    cString metaph2;
+    MString ms;
+    ms.Set(instr, _len_instr);
+    ms.DoubleMetaphone(metaph, metaph2);
+    memset(__ret_str, ' ', 20);
+    size32_t metaph2_len = strlen((char*) metaph2);
+    strncpy(__ret_str, (char*) metaph2, (metaph2_len > 20)?20:metaph2_len);
+}
+
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphoneBoth_40(char * __ret_str,unsigned _len_instr,const char * instr)
+{
+    cString metaph;
+    cString metaph2;
+    MString ms;
+    ms.Set(instr, _len_instr);
+    ms.DoubleMetaphone(metaph, metaph2);
+    memset(__ret_str, ' ', 40);
+    size32_t metaph_len = strlen((char*) metaph);
+    strncpy(__ret_str, (char*) metaph, (metaph_len > 20)?20:metaph_len);
+    size32_t metaph2_len = strlen((char*) metaph2);
+    strncpy(__ret_str+metaph_len, (char*) metaph2, (metaph2_len > 20)?20:metaph2_len);
+}

plugins/dmetaphone/dmetaphone.hpp

@@ -0,0 +1,33 @@
+#ifndef DMETAPHONE_INCL
+#define DMETAPHONE_INCL
+
+#ifdef _WIN32
+#define DMETAPHONE_CALL _cdecl
+#ifdef DMETAPHONE_EXPORTS
+#define DMETAPHONE_API __declspec(dllexport)
+#else
+#define DMETAPHONE_API __declspec(dllimport)
+#endif
+#else
+#define DMETAPHONE_CALL
+#define DMETAPHONE_API
+#endif
+
+#include "hqlplugins.hpp"
+
+extern "C" {
+
+#ifdef DMETAPHONE_EXPORTS
+DMETAPHONE_API bool getECLPluginDefinition(ECLPluginDefinitionBlock *pb);
+DMETAPHONE_API void setPluginContext(IPluginContext * _ctx);
+#endif
+
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone1(size32_t & __ret_len,char * & __ret_str,unsigned _len_instr,const char * instr);
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone2(size32_t & __ret_len,char * & __ret_str,unsigned _len_instr,const char * instr);
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphoneBoth(size32_t & __ret_len,char * & __ret_str,unsigned _len_instr,const char * instr);
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone1_20(char * __ret_str,unsigned _len_instr,const char * instr);
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphone2_20(char * __ret_str,unsigned _len_instr,const char * instr);
+DMETAPHONE_API void DMETAPHONE_CALL mpDMetaphoneBoth_40(char * __ret_str,unsigned _len_instr,const char * instr);
+}
+
+#endif

plugins/dmetaphone/metaphone.cpp

@@ -0,0 +1,895 @@
+#include "platform.h"
+#include <time.h>
+#include <stdlib.h>
+#include <string.h>
+#include "metaphone.h"
+
+////////////////////////////////////////////////////////////////////////////////
+// Double Metaphone (c) 1998, 1999 by Lawrence Philips
+//
+// Slightly modified by Kevin Atkinson to fix several bugs and
+// to allow it to give back more than 4 characters.
+//
+//  13-Dec-00   mtw Modified to return a number (e.g. 77th returns 77)
+//
+// Placed in the public domain by Lawrence Philips
+//
+////////////////////////////////////////////////////////////////////////////////
+#include "metaphone.h"
+#include <ctype.h>
+
+#define AND &&
+#define OR ||
+
+namespace nsDmetaphone {
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+MString::MString()
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+MString::MString(const char* in) : cString(in)
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+MString::MString(const cString& in) : cString(in)
+{
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+bool MString::SlavoGermanic()
+{
+    return (Find('W') OR Find('K') OR Find("CZ") OR Find("WITZ"));
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+inline void MString::MetaphAdd(const char* main)
+{
+    primary.Cat(main);
+    secondary.Cat(main);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+inline void MString::MetaphAdd(const char main)
+{
+    primary += main;
+    secondary += main;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+inline void MString::MetaphAdd(const char* main, const char* alt)
+{
+    if(*main)
+        primary.Cat(main);
+    if(*alt)
+    {
+        alternate = true;
+        if(alt[0] != ' ')
+            secondary.Cat(alt);
+    }else
+        if(*main AND (main[0] != ' '))
+            secondary.Cat(main);
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+bool MString::IsVowel(int at)
+{
+
+    if((at < 0) OR (at >= length))
+        return false;
+
+    char it = GetAt(at);
+
+    if((it == 'A') OR (it == 'E') OR (it == 'I') OR (it == 'O') OR (it == 'U') OR (it == 'Y') )
+        return true;
+
+    return false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+bool MString::StringAt(int start, int len, ... )
+{
+
+    if (start < 0) return false;
+
+    char    target[64];
+    char*   test;
+
+    if (Len - start < len)
+    {
+        return false;
+    }
+    memcpy( target, Ptr + start, len );
+    target[len] = 0;
+
+    va_list sstrings;
+    va_start(sstrings, len);
+
+    do
+    {
+        test = va_arg(sstrings, char*);
+        if(*test AND (strcmp(target, test) == 0))
+            return true;
+
+    }while(strcmp(test, ""));
+
+    va_end(sstrings);
+
+    return false;
+}
+
+////////////////////////////////////////////////////////////////////////////////
+// main deal
+////////////////////////////////////////////////////////////////////////////////
+void MString::DoubleMetaphone(cString &metaph, cString &metaph2)
+{
+
+    int current = 0;
+
+    length = Len;
+    if(length < 1)
+        return;
+    last = length - 1;//zero based index
+
+    alternate = false;
+    primary = "";
+    secondary = "";
+
+    Upper();
+
+    //pad the original string so that we can index beyond the edge of the world
+    Cat("     ");
+
+    //skip these when at start of word
+    if(StringAt(0, 2, "GN", "KN", "PN", "WR", "PS", ""))
+        current += 1;
+
+    //Initial 'X' is pronounced 'Z' e.g. 'Xavier'
+    if(GetAt(0) == 'X')
+    {
+        MetaphAdd('S'); //'Z' maps to 'S'
+        current += 1;
+    }
+
+    if (isdigit(GetAt(0)))
+    {
+        while (isdigit(GetAt(current)) && current < length)
+        {
+            MetaphAdd(GetAt(current));
+            current++;
+        }
+    }
+    else while(true OR (primary.Len < 4) OR (secondary.Len < 4))
+        ///////////main loop//////////////////////////
+    {
+        if(current >= length)
+            break;
+
+        switch(GetAt(current))
+        {
+        case 'A':
+        case 'E':
+        case 'I':
+        case 'O':
+        case 'U':
+        case 'Y':
+            if(current == 0)
+                //all init vowels now map to 'A'
+                MetaphAdd('A');
+            current +=1;
+            break;
+
+        case 'B':
+
+            //"-mb", e.g", "dumb", already skipped over...
+            MetaphAdd('P');
+
+            if(GetAt(current + 1) == 'B')
+                current +=2;
+            else
+                current +=1;
+            break;
+
+        case '\307': // ascii 0xc7 = C with cedilla
+            MetaphAdd('S');
+            current += 1;
+            break;
+
+        case 'C':
+            //various germanic
+            if((current > 1)
+                AND !IsVowel(current - 2)
+                AND StringAt((current - 1), 3, "ACH", "")
+                AND ((GetAt(current + 2) != 'I') AND ((GetAt(current + 2) != 'E')
+                OR StringAt((current - 2), 6, "BACHER", "MACHER", "")) ))
+            {
+                MetaphAdd('K');
+                current +=2;
+                break;
+            }
+
+            //special case 'caesar'
+            if((current == 0) AND StringAt(current, 6, "CAESAR", ""))
+            {
+                MetaphAdd('S');
+                current +=2;
+                break;
+            }
+
+            //italian 'chianti'
+            if(StringAt(current, 4, "CHIA", ""))
+            {
+                MetaphAdd('K');
+                current +=2;
+                break;
+            }
+
+            if(StringAt(current, 2, "CH", ""))
+            {
+                //find 'michael'
+                if((current > 0) AND StringAt(current, 4, "CHAE", ""))
+                {
+                    MetaphAdd("K", "X");
+                    current +=2;
+                    break;
+                }
+
+                //greek roots e.g. 'chemistry', 'chorus'
+                if((current == 0)
+                    AND (StringAt((current + 1), 5, "HARAC", "HARIS", "")
+                    OR StringAt((current + 1), 3, "HOR", "HYM", "HIA", "HEM", ""))
+                    AND !StringAt(0, 5, "CHORE", ""))
+                {
+                    MetaphAdd('K');
+                    current +=2;
+                    break;
+                }
+
+                //germanic, greek, or otherwise 'ch' for 'kh' sound
+                if((StringAt(0, 4, "VAN ", "VON ", "") OR StringAt(0, 3, "SCH", ""))
+                    // 'architect but not 'arch', 'orchestra', 'orchid'
+                    OR StringAt((current - 2), 6, "ORCHES", "ARCHIT", "ORCHID", "")
+                    OR StringAt((current + 2), 1, "T", "S", "")
+                    OR ((StringAt((current - 1), 1, "A", "O", "U", "E", "") OR (current == 0))
+                    //e.g., 'wachtler', 'wechsler', but not 'tichner'
+                    AND StringAt((current + 2), 1, "L", "R", "N", "M", "B", "H", "F", "V", "W", " ", "")))
+                {
+                    MetaphAdd('K');
+                }else{
+                    if(current > 0)
+                    {
+                        if(StringAt(0, 2, "MC", ""))
+                            //e.g., "McHugh"
+                            MetaphAdd('K');
+                        else
+                            MetaphAdd("X", "K");
+                    }else
+                        MetaphAdd('X');
+                }
+                current +=2;
+                break;
+            }
+            //e.g, 'czerny'
+            if(StringAt(current, 2, "CZ", "") AND !StringAt((current - 2), 4, "WICZ", ""))
+            {
+                MetaphAdd("S", "X");
+                current += 2;
+                break;
+            }
+
+            //e.g., 'focaccia'
+            if(StringAt((current + 1), 3, "CIA", ""))
+            {
+                MetaphAdd('X');
+                current += 3;
+                break;
+            }
+
+            //double 'C', but not if e.g. 'McClellan'
+            if(StringAt(current, 2, "CC", "") AND !((current == 1) AND (GetAt(0) == 'M')))
+            {
+                //'bellocchio' but not 'bacchus'
+                if(StringAt((current + 2), 1, "I", "E", "H", "") AND !StringAt((current + 2), 2, "HU", ""))
+                {
+                    //'accident', 'accede' 'succeed'
+                    if(((current == 1) AND (GetAt(current - 1) == 'A'))
+                        OR StringAt((current - 1), 5, "UCCEE", "UCCES", ""))
+                        MetaphAdd("KS");
+                    //'bacci', 'bertucci', other italian
+                    else
+                        MetaphAdd('X');
+                    current += 3;
+                    break;
+                }else{//Pierce's rule
+                    MetaphAdd('K');
+                    current += 2;
+                    break;
+                }
+            }
+
+            if(StringAt(current, 2, "CK", "CG", "CQ", ""))
+            {
+                MetaphAdd('K');
+                current += 2;
+                break;
+            }
+
+            if(StringAt(current, 2, "CI", "CE", "CY", ""))
+            {
+                //italian vs. english
+                if(StringAt(current, 3, "CIO", "CIE", "CIA", ""))
+                    MetaphAdd("S", "X");
+                else
+                    MetaphAdd('S');
+                current += 2;
+                break;
+            }
+
+            //else
+            MetaphAdd('K');
+
+            //name sent in 'mac caffrey', 'mac gregor
+            if(StringAt((current + 1), 2, " C", " Q", " G", ""))
+                current += 3;
+            else
+                if(StringAt((current + 1), 1, "C", "K", "Q", "")
+                    AND !StringAt((current + 1), 2, "CE", "CI", ""))
+                    current += 2;
+                else
+                    current += 1;
+            break;
+
+        case 'D':
+            if(StringAt(current, 2, "DG", ""))
+            {
+                if(StringAt((current + 2), 1, "I", "E", "Y", ""))
+                {
+                    //e.g. 'edge'
+                    MetaphAdd('J');
+                    current += 3;
+                    break;
+                }else{
+                    //e.g. 'edgar'
+                    MetaphAdd("TK");
+                    current += 2;
+                    break;
+                }
+            }
+
+            if(StringAt(current, 2, "DT", "DD", ""))
+            {
+                MetaphAdd('T');
+                current += 2;
+                break;
+            }
+
+            //else
+            MetaphAdd('T');
+            current += 1;
+            break;
+
+       case 'F':
+            if(GetAt(current + 1) == 'F')
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('F');
+            break;
+
+        case 'G':
+            if(GetAt(current + 1) == 'H')
+            {
+                if((current > 0) AND !IsVowel(current - 1))
+                {
+                    MetaphAdd('K');
+                    current += 2;
+                    break;
+                }
+
+                if(current < 3)
+                {
+                    //'ghislane', ghiradelli
+                    if(current == 0)
+                    {
+                        if(GetAt(current + 2) == 'I')
+                            MetaphAdd('J');
+                        else
+                            MetaphAdd('K');
+                        current += 2;
+                        break;
+                    }
+                }
+                //Parker's rule (with some further refinements) - e.g., 'hugh'
+                if(((current > 1) AND StringAt((current - 2), 1, "B", "H", "D", "") )
+                    //e.g., 'bough'
+                    OR ((current > 2) AND StringAt((current - 3), 1, "B", "H", "D", "") )
+                    //e.g., 'broughton'
+                    OR ((current > 3) AND StringAt((current - 4), 1, "B", "H", "") ) )
+                {
+                    current += 2;
+                    break;
+                }else{
+                    //e.g., 'laugh', 'McLaughlin', 'cough', 'gough', 'rough', 'tough'
+                    if((current > 2)
+                        AND (GetAt(current - 1) == 'U')
+                        AND StringAt((current - 3), 1, "C", "G", "L", "R", "T", "") )
+                    {
+                        MetaphAdd('F');
+                    }else
+                        if((current > 0) AND GetAt(current - 1) != 'I')
+                            MetaphAdd('K');
+
+                        current += 2;
+                        break;
+                }
+            }
+
+            if(GetAt(current + 1) == 'N')
+            {
+                if((current == 1) AND IsVowel(0) AND !SlavoGermanic())
+                {
+                    MetaphAdd("KN", "N");
+                }else
+                    //not e.g. 'cagney'
+                    if(!StringAt((current + 2), 2, "EY", "")
+                        AND (GetAt(current + 1) != 'Y') AND !SlavoGermanic())
+                    {
+                        MetaphAdd("N", "KN");
+                    }else
+                        MetaphAdd("KN");
+                    current += 2;
+                    break;
+            }
+
+            //'tagliaro'
+            if(StringAt((current + 1), 2, "LI", "") AND !SlavoGermanic())
+            {
+                MetaphAdd("KL", "L");
+                current += 2;
+                break;
+            }
+
+            //-ges-,-gep-,-gel-, -gie- at beginning
+            if((current == 0)
+                AND ((GetAt(current + 1) == 'Y')
+                OR StringAt((current + 1), 2, "ES", "EP", "EB", "EL", "EY", "IB", "IL", "IN", "IE", "EI", "ER", "")) )
+            {
+                MetaphAdd("K", "J");
+                current += 2;
+                break;
+            }
+
+            // -ger-,  -gy-
+            if((StringAt((current + 1), 2, "ER", "") OR (GetAt(current + 1) == 'Y'))
+                AND !StringAt(0, 6, "DANGER", "RANGER", "MANGER", "")
+                AND !StringAt((current - 1), 1, "E", "I", "")
+                AND !StringAt((current - 1), 3, "RGY", "OGY", "") )
+            {
+                MetaphAdd("K", "J");
+                current += 2;
+                break;
+            }
+
+            // italian e.g, 'biaggi'
+            if(StringAt((current + 1), 1, "E", "I", "Y", "") OR StringAt((current - 1), 4, "AGGI", "OGGI", ""))
+            {
+                //obvious germanic
+                if((StringAt(0, 4, "VAN ", "VON ", "") OR StringAt(0, 3, "SCH", ""))
+                    OR StringAt((current + 1), 2, "ET", ""))
+                    MetaphAdd('K');
+                else
+                    //always soft if french ending
+                    if(StringAt((current + 1), 4, "IER ", ""))
+                        MetaphAdd('J');
+                    else
+                        MetaphAdd("J", "K");
+                    current += 2;
+                    break;
+            }
+
+            if(GetAt(current + 1) == 'G')
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('K');
+            break;
+
+        case 'H':
+            //only keep if first & before vowel or btw. 2 vowels
+            if(((current == 0) OR IsVowel(current - 1))
+                AND IsVowel(current + 1))
+            {
+                MetaphAdd('H');
+                current += 2;
+            }else//also takes care of 'HH'
+                current += 1;
+            break;
+
+        case 'J':
+            //obvious spanish, 'jose', 'san jacinto'
+            if(StringAt(current, 4, "JOSE", "") OR StringAt(0, 4, "SAN ", "") )
+            {
+                if(((current == 0) AND (GetAt(current + 4) == ' ')) OR StringAt(0, 4, "SAN ", "") )
+                    MetaphAdd('H');
+                else
+                {
+                    MetaphAdd("J", "H");
+                }
+                current +=1;
+                break;
+            }
+
+            if((current == 0) AND !StringAt(current, 4, "JOSE", ""))
+                MetaphAdd("J", "A");//Yankelovich/Jankelowicz
+            else
+                //spanish pron. of e.g. 'bajador'
+                if(IsVowel(current - 1)
+                    AND !SlavoGermanic()
+                    AND ((GetAt(current + 1) == 'A') OR (GetAt(current + 1) == 'O')))
+                    MetaphAdd("J", "H");
+                else
+                    if(current == last)
+                        MetaphAdd("J", " ");
+                    else
+                        if(!StringAt((current + 1), 1, "L", "T", "K", "S", "N", "M", "B", "Z", "")
+                            AND !StringAt((current - 1), 1, "S", "K", "L", ""))
+                            MetaphAdd('J');
+
+                        if(GetAt(current + 1) == 'J')//it could happen!
+                            current += 2;
+                        else
+                            current += 1;
+                        break;
+
+        case 'K':
+            if(GetAt(current + 1) == 'K')
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('K');
+            break;
+
+        case 'L':
+            if(GetAt(current + 1) == 'L')
+            {
+                //spanish e.g. 'cabrillo', 'gallegos'
+                if(((current == (length - 3))
+                    AND StringAt((current - 1), 4, "ILLO", "ILLA", "ALLE", ""))
+                    OR ((StringAt((last - 1), 2, "AS", "OS", "") OR StringAt(last, 1, "A", "O", ""))
+                    AND StringAt((current - 1), 4, "ALLE", "")) )
+                {
+                    MetaphAdd("L", " ");
+                    current += 2;
+                    break;
+                }
+                current += 2;
+            }else
+                current += 1;
+            MetaphAdd('L');
+            break;
+
+        case 'M':
+            if((StringAt((current - 1), 3, "UMB", "")
+                AND (((current + 1) == last) OR StringAt((current + 2), 2, "ER", "")))
+                //'dumb','thumb'
+                OR  (GetAt(current + 1) == 'M') )
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('M');
+            break;
+
+        case 'N':
+            if(GetAt(current + 1) == 'N')
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('N');
+            break;
+
+        case '\321': // Ascii 0xD1 = capital N with tilde
+            current += 1;
+            MetaphAdd('N');
+            break;
+
+        case 'P':
+            if(GetAt(current + 1) == 'H')
+            {
+                MetaphAdd('F');
+                current += 2;
+                break;
+            }
+
+            //also account for "campbell", "raspberry"
+            if(StringAt((current + 1), 1, "P", "B", ""))
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('P');
+            break;
+
+        case 'Q':
+            if(GetAt(current + 1) == 'Q')
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('K');
+            break;
+
+        case 'R':
+            //french e.g. 'rogier', but exclude 'hochmeier'
+            if((current == last)
+                AND !SlavoGermanic()
+                AND StringAt((current - 2), 2, "IE", "")
+                AND !StringAt((current - 4), 2, "ME", "MA", ""))
+                MetaphAdd("", "R");
+            else
+                MetaphAdd('R');
+
+            if(GetAt(current + 1) == 'R')
+                current += 2;
+            else
+                current += 1;
+            break;
+
+        case 'S':
+            //special cases 'island', 'isle', 'carlisle', 'carlysle'
+            if(StringAt((current - 1), 3, "ISL", "YSL", ""))
+            {
+                current += 1;
+                break;
+            }
+
+            //special case 'sugar-'
+            if((current == 0) AND StringAt(current, 5, "SUGAR", ""))
+            {
+                MetaphAdd("X", "S");
+                current += 1;
+                break;
+            }
+
+            if(StringAt(current, 2, "SH", ""))
+            {
+                //germanic
+                if(StringAt((current + 1), 4, "HEIM", "HOEK", "HOLM", "HOLZ", ""))
+                    MetaphAdd('S');
+                else
+                    MetaphAdd('X');
+                current += 2;
+                break;
+            }
+
+            //italian & armenian
+            if(StringAt(current, 3, "SIO", "SIA", "") OR StringAt(current, 4, "SIAN", ""))
+            {
+                if(!SlavoGermanic())
+                    MetaphAdd("S", "X");
+                else
+                    MetaphAdd('S');
+                current += 3;
+                break;
+            }
+
+            //german & anglicisations, e.g. 'smith' match 'schmidt', 'snider' match 'schneider'
+            //also, -sz- in slavic language altho in hungarian it is pronounced 's'
+            if(((current == 0)
+                AND StringAt((current + 1), 1, "M", "N", "L", "W", ""))
+                OR StringAt((current + 1), 1, "Z", ""))
+            {
+                MetaphAdd("S", "X");
+                if(StringAt((current + 1), 1, "Z", ""))
+                    current += 2;
+                else
+                    current += 1;
+                break;
+            }
+
+            if(StringAt(current, 2, "SC", ""))
+            {
+                //Schlesinger's rule
+                if(GetAt(current + 2) == 'H')
+                {
+                    //dutch origin, e.g. 'school', 'schooner'
+                    if(StringAt((current + 3), 2, "OO", "ER", "EN", "UY", "ED", "EM", ""))
+                    {
+                        //'schermerhorn', 'schenker'
+                        if(StringAt((current + 3), 2, "ER", "EN", ""))
+                        {
+                            MetaphAdd("X", "SK");
+                        }else
+                            MetaphAdd("SK");
+                        current += 3;
+                        break;
+                    }else{
+                        if((current == 0) AND !IsVowel(3) AND (GetAt(3) != 'W'))
+                            MetaphAdd("X", "S");
+                        else
+                            MetaphAdd('X');
+                        current += 3;
+                        break;
+                    }
+                }
+
+                if(StringAt((current + 2), 1, "I", "E", "Y", ""))
+                {
+                    MetaphAdd('S');
+                    current += 3;
+                    break;
+                }
+                //else
+                MetaphAdd("SK");
+                current += 3;
+                break;
+            }
+
+            //french e.g. 'resnais', 'artois'
+            if((current == last) AND StringAt((current - 2), 2, "AI", "OI", ""))
+                MetaphAdd("", "S");
+            else
+                MetaphAdd('S');
+
+            if(StringAt((current + 1), 1, "S", "Z", ""))
+                current += 2;
+            else
+                current += 1;
+            break;
+
+        case 'T':
+            if(StringAt(current, 4, "TION", ""))
+            {
+                MetaphAdd('X');
+                current += 3;
+                break;
+            }
+
+            if(StringAt(current, 3, "TIA", "TCH", ""))
+            {
+                MetaphAdd('X');
+                current += 3;
+                break;
+            }
+
+            if(StringAt(current, 2, "TH", "")
+                OR StringAt(current, 3, "TTH", ""))
+            {
+                //special case 'thomas', 'thames' or germanic
+                if(StringAt((current + 2), 2, "OM", "AM", "")
+                    OR StringAt(0, 4, "VAN ", "VON ", "")
+                    OR StringAt(0, 3, "SCH", ""))
+                {
+                    MetaphAdd('T');
+                }else{
+                    MetaphAdd("0", "T");
+                }
+                current += 2;
+                break;
+            }
+
+            if(StringAt((current + 1), 1, "T", "D", ""))
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('T');
+            break;
+
+        case 'V':
+            if(GetAt(current + 1) == 'V')
+                current += 2;
+            else
+                current += 1;
+            MetaphAdd('F');
+            break;
+
+        case 'W':
+            //can also be in middle of word
+            if(StringAt(current, 2, "WR", ""))
+            {
+                MetaphAdd('R');
+                current += 2;
+                break;
+            }
+
+            if((current == 0)
+                AND (IsVowel(current + 1) OR StringAt(current, 2, "WH", "")))
+            {
+                //Wasserman should match Vasserman
+                if(IsVowel(current + 1))
+                    MetaphAdd("A", "F");
+                else
+                    //need Uomo to match Womo
+                    MetaphAdd('A');
+            }
+
+            //Arnow should match Arnoff
+            if(((current == last) AND IsVowel(current - 1))
+                OR StringAt((current - 1), 5, "EWSKI", "EWSKY", "OWSKI", "OWSKY", "")
+                OR StringAt(0, 3, "SCH", ""))
+            {
+                MetaphAdd("", "F");
+                current +=1;
+                break;
+            }
+
+            //polish e.g. 'filipowicz'
+            if(StringAt(current, 4, "WICZ", "WITZ", ""))
+            {
+                MetaphAdd("TS", "FX");
+                current +=4;
+                break;
+            }
+
+            //else skip it
+            current +=1;
+            break;
+
+        case 'X':
+            //french e.g. breaux
+            if(!((current == last)
+                AND (StringAt((current - 3), 3, "IAU", "EAU", "")
+                OR StringAt((current - 2), 2, "AU", "OU", ""))) )
+                MetaphAdd("KS");
+
+            if(StringAt((current + 1), 1, "C", "X", ""))
+                current += 2;
+            else
+                current += 1;
+            break;
+
+        case 'Z':
+            //chinese pinyin e.g. 'zhao'
+            if(GetAt(current + 1) == 'H')
+            {
+                MetaphAdd('J');
+                current += 2;
+                break;
+            }else
+                if(StringAt((current + 1), 2, "ZO", "ZI", "ZA", "")
+                    OR (SlavoGermanic() AND ((current > 0) AND GetAt(current - 1) != 'T')))
+                {
+                    MetaphAdd("S", "TS");
+                }
+                else
+                    MetaphAdd('S');
+
+                if(GetAt(current + 1) == 'Z')
+                    current += 2;
+                else
+                    current += 1;
+                break;
+
+        default:
+            current += 1;
+        }
+    }
+
+    metaph = primary.Ptr;
+    //only give back 4 char metaph
+    //if(metaph.Len > 4)
+    //        metaph.SetAt(4,'\0');
+    metaph2 = secondary.Ptr;
+    //if(metaph2.Len > 4)
+    //        metaph2.SetAt(4,'\0');
+
+}
+
+}//namespace

plugins/dmetaphone/metaphone.h

@@ -0,0 +1,69 @@
+#ifndef _METAPHONE_H
+#define _METAPHONE_H
+
+////////////////////////////////////////////////////////////////////////////////
+//
+////////////////////////////////////////////////////////////////////////////////
+
+#include "platform.h"
+#include "dmetaphone.hpp"
+#include "cstring.h"
+#include <string.h>
+
+//#include <varargs.h>
+//#define false FALSE
+//#define true TRUE
+
+namespace nsDmetaphone {
+
+#ifdef _MSC_VER
+//Disable warnings about cString not matching the export specification of MString,
+//because all methods of cString are inline, so they will be compiled into each
+//plugin.
+#pragma warning(push)
+#pragma warning(disable: 4251 4275)
+#endif
+
+class DMETAPHONE_API MString : public cString
+{
+        int             length, last;
+        bool    alternate;
+        cString primary, secondary;
+
+public:
+        MString();
+        MString(const char*);
+        MString& operator =(const char *Str)
+        {
+            Set(Str);
+            return(*this);
+        }
+        MString(const cString&);
+        bool SlavoGermanic();
+        bool IsVowel(int at);
+        inline void MetaphAdd(const char* main);
+        inline void MetaphAdd(const char main);
+        inline void MetaphAdd(const char* main, const char* alt);
+        bool StringAt(int start, int length, ... );
+        void DoubleMetaphone(cString &metaph, cString &metaph2);
+        char GetAt( int x )
+        {
+            return Ptr[x];
+        }
+        bool Find(char c)
+        {
+            return (strchr(Ptr, c) != NULL);
+        }
+        bool Find(const char * str)
+        {
+            return (strstr(Ptr, str) != NULL);
+        }
+};
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
+}//namespace
+
+#endif

plugins/fileservices/fileservices.cpp

@@ -489,7 +489,7 @@ FILESERVICES_API char * FILESERVICES_CALL fsCmdProcess(const char *prog, const c
     StringBuffer in, out;
     in.append(src);
 
-    callExternalProgram(prog, in, out);
+    runExternalCommand(out, prog, in);
 
     return CTXSTRDUP(parentCtx, out.str());
 }
@@ -500,7 +500,7 @@ FILESERVICES_API void FILESERVICES_CALL fsCmdProcess2(unsigned & tgtLen, char *
     StringBuffer in, out;
     in.append(srcLen, src);
 
-    callExternalProgram(prog, in, out);
+    runExternalCommand(out, prog, in);
 
     tgtLen = out.length();
     tgt = (char *)CTXDUP(parentCtx, out.str(), out.length());

roxie/ccd/ccdcontext.cpp

@@ -1410,8 +1410,14 @@ public:
                     StringBuffer s;
                     CTXLOG("Exception thrown in query - cleaning up: %d: %s", e->errorCode(), e->errorMessage(s).str());
                 }
-                if (created)
+                if (created)  // Partially-created graphs are liable to crash if you call abort() on them...
                     endGraph(startCycles, true);
+                else
+                {
+                    // Bit of a hack... needed to avoid pure virtual calls if these are left to the CRoxieContextBase destructor
+                    graph.clear();
+                    childGraphs.kill();
+                }
                 CTXLOG("Done cleaning up");
                 throw;
             }
@@ -1420,6 +1426,12 @@ public:
                 CTXLOG("Exception thrown in query - cleaning up");
                 if (created)
                     endGraph(startCycles, true);
+                else
+                {
+                    // Bit of a hack... needed to avoid pure virtual calls if these are left to the CRoxieContextBase destructor
+                    graph.clear();
+                    childGraphs.kill();
+                }
                 CTXLOG("Done cleaning up");
                 throw;
             }

system/jlib/jutil.cpp

@@ -1549,181 +1549,6 @@ static int exec(const char* _command)
 }
 #endif 
 
-bool callExternalProgram(const char *progname, const StringBuffer &input, StringBuffer &output, StringArray *env_in)
-{
-#ifdef _WIN32 
-    StringBuffer envp;
-    if (env_in)
-    {
-        ForEachItemIn(index, *env_in)
-            envp.append(env_in->item(index)).append('\0');
-    }
-    win32::ProcessPipe p(progname, envp.length() ? envp.str() : NULL);
-    p.Write(input.str(), input.length());
-    p.CloseWrite();
-
-    char buf[4096];
-    for(;;)
-    {
-        // Read program output
-        DWORD bread = (DWORD)p.Read(buf, sizeof(buf));
-        if(!bread)
-        {
-            break;
-        }
-        output.append(bread, buf);
-    }
-#else
-    struct Pipe
-    {
-        Pipe()
-        { 
-            p[0]=p[1]=-1;
-            if(pipe(p))
-                throw MakeStringException(-1,"Pipe create failed: %d",errno);
-        }
-
-        ~Pipe()
-        {
-            if(p[0]>=0)
-                close(p[0]);
-            if(p[1]>=0)
-                close(p[1]);
-        }
-
-        int Read(void *buf, size32_t nbyte)
-        {
-            return read(p[0],buf,nbyte);
-        }
-
-        int Write(const void *buf, size32_t nbyte)
-        {
-            return write(p[1],buf,nbyte);
-        }
-
-        void SetStdout()
-        {
-            if(p[1]!=STDOUT_FILENO)
-            {
-                if(dup2(p[1],STDOUT_FILENO)<0)
-                    throw MakeStringException(-1,"stdout failed: %d",errno);
-                close(p[1]);
-            }
-        }
-
-        void SetStdin()
-        {
-            if(p[0]!=STDIN_FILENO)
-            {
-                if(dup2(p[0],STDIN_FILENO)<0)
-                    throw MakeStringException(-1,"stdin failed: %d",errno);
-                close(p[0]);
-            }
-        }
-
-        void CloseRead()
-        {
-            close(p[0]);
-            p[0]=-1;
-        }
-
-        void CloseWrite()
-        {
-            close(p[1]);
-            p[1]=-1;
-        }
-
-        int p[2];
-    } pipe1, pipe2;
-    
-    struct ChildProc
-    {
-        ChildProc()
-        {
-            if((pid=fork())<0)
-                throw MakeStringException(-1,"Fork failed: %d",errno);
-        }
-        ~ChildProc()
-        {
-            if(!inChild())
-            {
-                for(;;)
-                {
-                    if(waitpid(pid,0,0)>=0)
-                        break;
-                    else if (errno==EINTR)
-                    {
-                    }
-                }
-            }
-        }
-        bool inChild()
-        {
-            return pid==0;
-        }
-        int pid;
-    } fchild;
-
-    if(fchild.inChild())
-    {
-        pipe1.CloseWrite();
-        pipe1.SetStdin();
-
-        pipe2.CloseRead();
-        pipe2.SetStdout();
-
-        if (env_in)
-        {
-            const char **envp = (const char **) alloca((env_in->ordinality()+1) * sizeof(const char *));
-            ForEachItemIn(index, *env_in)
-                envp[index]=env_in->item(index);
-            envp[env_in->ordinality()] = NULL;
-            execle(progname, progname, (const char *) NULL, (char * const *)envp);  // will not return, on success
-        }
-        else
-            execlp(progname, progname, (const char *) NULL);  // will not return, on success
-        _exit(EXIT_FAILURE); // must be _exit!!
-    }
-    else
-    {
-        pipe1.CloseRead();
-        pipe2.CloseWrite();
-        const char* data=input.str();
-        size32_t count=input.length();
-        for(;count>0;)
-        {
-            ssize_t w=pipe1.Write(data,count);
-            if(w<0)
-            {
-                if (errno!=EINTR)
-                    throw MakeStringException(-1,"Pipe write failed: %d",errno);
-            }
-            else
-            {
-                data+=w;
-                count-=w;
-            }
-        }
-        pipe1.CloseWrite();
-
-        char buf[4096];
-        for(;;)
-        {
-            int r=pipe2.Read(buf, sizeof(buf));
-            if(r>0)
-            {
-                output.append(r, buf);
-            }
-            else if(r==0)
-                break;
-            else if(errno!=EINTR)
-                throw MakeStringException(-1,"Pipe read failed: %d",errno); 
-        }
-    }
-#endif
-    return true;
-}
-
 //Calculate the greatest common divisor using Euclid's method
 unsigned __int64 greatestCommonDivisor(unsigned __int64 left, unsigned __int64 right)
 {
@@ -1887,6 +1712,44 @@ static const char *findExtension(const char *fn)
     return ret;
 }
 
+unsigned runExternalCommand(StringBuffer &output, const char *cmd, const char *input)
+{
+    try
+    {
+        Owned<IPipeProcess> pipe = createPipeProcess();
+        pipe->run(cmd, cmd, ".", input != NULL, true, true, 1024*1024);
+        if (input)
+        {
+            pipe->write(strlen(input), input);
+            pipe->closeInput();
+        }
+        char buf[1024];
+        while (true)
+        {
+            size32_t read = pipe->read(sizeof(buf), buf);
+            if (!read)
+                break;
+            output.append(read, buf);
+        }
+        int ret = pipe->wait();
+        StringBuffer error;
+        while (true)
+        {
+            size32_t read = pipe->readError(sizeof(buf), buf);
+            if (!read)
+                break;
+            error.append(read, buf);
+        }
+        return ret;
+    }
+    catch (IException *E)
+    {
+        E->Release();
+        output.clear();
+        return 255;
+    }
+}
+
 bool matchesMask(const char *fn, const char *mask, unsigned p, unsigned n)
 {
     StringBuffer match;

system/jlib/jutil.hpp

@@ -207,7 +207,7 @@ extern jlib_decl void doStackProbe();
 #define arraysize(T) (sizeof(T)/sizeof(*T))
 #endif
 
-extern jlib_decl bool callExternalProgram(const char *progname, const StringBuffer &input, StringBuffer &output, StringArray *env=NULL);
+extern unsigned runExternalCommand(StringBuffer &output, const char *cmd, const char *input);
 
 extern jlib_decl unsigned __int64 greatestCommonDivisor(unsigned __int64 left, unsigned __int64 right);
 

testing/regress/ecl/key/metaphone.xml

@@ -0,0 +1,6 @@
+<Dataset name='Result 1'>
+ <Row><name>Algernon</name><d1>ALKRNN</d1><d2>ALJRNN</d2><db>ALKRNNALJRNN</db><d1_20>ALKRNN              </d1_20><d2_20>ALJRNN              </d2_20><db_40>ALKRNNALJRNN                            </db_40></Row>
+ <Row><name>Englebert</name><d1>ANKLPRT</d1><d2>ANKLPRT</d2><db>ANKLPRTANKLPRT</db><d1_20>ANKLPRT             </d1_20><d2_20>ANKLPRT             </d2_20><db_40>ANKLPRTANKLPRT                          </db_40></Row>
+ <Row><name>Cholmondley</name><d1>XLMNTL</d1><d2>XLMNTL</d2><db>XLMNTLXLMNTL</db><d1_20>XLMNTL              </d1_20><d2_20>XLMNTL              </d2_20><db_40>XLMNTLXLMNTL                            </db_40></Row>
+ <Row><name>Farquar</name><d1>FRKR</d1><d2>FRKR</d2><db>FRKRFRKR</db><d1_20>FRKR                </d1_20><d2_20>FRKR                </d2_20><db_40>FRKRFRKR                                </db_40></Row>
+</Dataset>

testing/regress/ecl/metaphone.ecl

@@ -0,0 +1,47 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2015 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+import Std.Metaphone, lib_metaphone;
+
+input := DATASET([
+  {'Algernon'},
+  {'Englebert'},
+  {'Cholmondley'},
+  {'Farquar'}
+], { string name});
+
+outrec := RECORD
+  STRING name;
+  STRING d1;
+  STRING d2;
+  STRING db;
+  STRING20 d1_20;
+  STRING20 d2_20;
+  STRING40 db_40;
+END;
+
+outrec t(string name) := TRANSFORM
+   SELF.name := name;
+   SELF.d1 := Metaphone.primary(name);
+   SELF.d2 := Metaphone.secondary(name);
+   SELF.db := Metaphone.double(name);
+   SELF.d1_20 := lib_metaphone.MetaphoneLib.DMetaphone1_20(name);
+   SELF.d2_20 := lib_metaphone.MetaphoneLib.DMetaphone2_20(name);
+   SELF.db_40  := lib_metaphone.MetaphoneLib.DMetaphoneBoth_40(name);
+END;
+
+output(PROJECT(input, t(LEFT.name)));