Initial commit corresponding to svn revision 66146

Signed-off-by: richard.chapman <richard.chapman@lexisnexis.com>

.gitattributes

@@ -0,0 +1,11 @@
+*               text=auto
+*.bat           eol=crlf
+*.jpg           -text
+*.gif           -text
+*.png           -text
+*.jpg           -text
+*.swf           -text
+*.ico           -text
+*.psd           -text
+*.doc           -text
+*.mk            -text

.gitignore

@@ -0,0 +1,3 @@
+externals/
+.svn/
+ln/

CMakeLists.txt

@@ -0,0 +1,437 @@
+###############################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################    
+
+#
+#########################################################
+# Description:
+# ------------
+#           This file is the top level handling for
+#           cmake based compilation and build process.
+#
+#   To build for Linux:
+#   1. Check out sources (for example, to directory ~/hpcc)
+#   2. Create a build directory - either as a child of hpcc or elsewhere
+#   3. cd to the build directory
+#   4a.To create makefiles to build native release version for local machine, run 
+#       cmake ~/hpcc
+#   4b.To create makefiles to build native debug version, run
+#       cmake -DCMAKE_BUILD_TYPE=Debug ~/hpcc
+#   4c.To create makefiles to build 32-bit version from 64-bit host, run
+#       cmake -DCMAKE_C_FLAGS:STRING="-m32 -march=i386" -DCMAKE_CXX_FLAGS:STRING="-m32 -march=i386" ~/hpcc
+#   5. To build the makefiles just created above, run
+#       make
+#   6. Executables will be created in ./<releasemode>/bin and ./<releasemode>/libs
+#
+#   To build for Windows:
+#   1. Check out sources (for example, to directory c:\hpcc)
+#   2. Create a build directory - either as a child of hpcc or elsewhere
+#   3. cd to the build directory
+#   4. To create a Visual Studio project, run 
+#       cmake c:\hpcc -G "Visual Studio 9 2008"
+#      The sln file hpccsystems-platform.sln will be created in the current directory, and will support debug and release targets
+#   5. To build the project, load the solution into the visual studio IDE and build in the usual way.
+#   6. Executables will be created in (for example) c:\hpcc\bin\<release_mode>
+#
+#########################################################
+
+project (hpccsystems-platform)
+cmake_minimum_required (VERSION 2.6)
+
+include(CTest)
+ENABLE_TESTING()
+
+set (HPCC_SOURCE_DIR ${CMAKE_CURRENT_SOURCE_DIR})
+set(CMAKE_MODULE_PATH "${HPCC_SOURCE_DIR}/cmake_modules/")
+
+###
+## Version Information
+###
+set ( HPCC_PROJECT "community" )
+set ( HPCC_MAJOR 3 )
+set ( HPCC_MINOR 0 )
+set ( HPCC_POINT 0 )
+set ( HPCC_MATURITY "trunk" )
+set ( HPCC_SEQUENCE 1 )
+###
+
+###
+## Build Level
+###
+if( NOT BUILD_LEVEL )
+    set ( BUILD_LEVEL "COMMUNITY" )
+endif()
+###
+
+
+###
+## Config Block
+###
+option(PREFIX "Set the install prefix")
+option(EXEC_PREFIX "Set the execution prefix")
+option(CONFIG_PREFIX "Set the configuration prefix")
+option(DIR_NAME "Set the install directory name")
+option(LIB_DIR "Set the library install dir")
+option(EXEC_DIR "Set the executable install dir")
+option(COMPONENTFILES_DIR "Set the componentfiles dir")
+option(ADMIN_DIR "Set the admin dir")
+option(PLUGINS_DIR "Set the plugins dir")
+option(CONFIG_SOURCE_DIR "Set the configuration source dir")
+option(RUNTIME_DIR "Set the runtime dir")
+option(HOME_DIR "Set the home dir")
+option(LOCK_DIR "Set the lock dir")
+option(PID_DIR "Set the pid dir")
+option(LOG_DIR "Set the log dir")
+option(RUNTIME_USER "Set the runtime username")
+option(RUNTIME_GROUP "Set the runtime group")
+option(ENV_XML_FILE "Set the environment xml file name.")
+option(ENV_CONF_FILE "Set the environment conf file name.")
+option(LICENSE_FILE "Set the liscense file to use.")
+
+if( NOT LICENSE_FILE )
+    set(LICENSE_FILE "LICENSE.txt")
+endif()
+
+include(${HPCC_SOURCE_DIR}/cmake_modules/optionDefaults.cmake)
+###
+
+###
+## The following sets the install directories and names.
+###
+set ( OSSDIR "${DIR_NAME}" )
+set ( CPACK_SET_DESTDIR "ON" )
+set ( CPACK_INSTALL_PREFIX "${PREFIX}" )
+
+SET(CMAKE_INSTALL_PREFIX "${INSTALL_DIR}")
+SET(CMAKE_SKIP_BUILD_RPATH  FALSE)
+SET(CMAKE_BUILD_WITH_INSTALL_RPATH FALSE) 
+SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+SET(CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
+include(${HPCC_SOURCE_DIR}/cmake_modules/commonSetup.cmake)
+
+add_subdirectory (initfiles)
+
+add_subdirectory (tools/esdl)
+add_subdirectory (tools/hidl)
+
+add_subdirectory (esp/scm)
+add_subdirectory (common/commonext)
+add_subdirectory (common/deftype)
+add_subdirectory (common/dllserver)
+add_subdirectory (common/environment)
+add_subdirectory (common/fileview2)
+add_subdirectory (common/monitoring/prosysinfo)
+add_subdirectory (common/remote)
+add_subdirectory (common/roxiecommlib)
+add_subdirectory (common/roxiehelper)
+add_subdirectory (common/roxiemanager)
+add_subdirectory (common/thorhelper)
+add_subdirectory (common/workunit)
+add_subdirectory (dali/base)
+add_subdirectory (dali/dafilesrv)
+add_subdirectory (dali/dalidiag)
+add_subdirectory (dali/dalistop)
+add_subdirectory (dali/datest)
+if (CPPUNIT)
+    add_subdirectory (dali/daunittest)
+endif()
+add_subdirectory (dali/dfu)
+add_subdirectory (dali/dfuplus)
+add_subdirectory (dali/dfuxref)
+add_subdirectory (dali/dfuXRefLib)
+add_subdirectory (dali/ft)
+add_subdirectory (dali/regress)
+add_subdirectory (dali/sdsfix)
+add_subdirectory (dali/daliadmin)
+add_subdirectory (dali/server)
+add_subdirectory (dali/treeview)
+add_subdirectory (dali/updtdalienv)
+add_subdirectory (deployment/configgen)
+add_subdirectory (deployment/deploy)
+add_subdirectory (deployment/deployutils)
+add_subdirectory (deployment/envgen)
+add_subdirectory (ecl/agentexec)
+add_subdirectory (ecl/eclagent)
+add_subdirectory (ecl/eclcc)
+add_subdirectory (ecl/eclccserver)
+add_subdirectory (ecl/eclscheduler)
+add_subdirectory (ecl/eclplus)
+add_subdirectory (ecl/hql)
+add_subdirectory (ecl/hqlcpp)
+add_subdirectory (ecl/hthor)
+add_subdirectory (ecl/scheduleadmin)
+add_subdirectory (ecl/schedulectrl)
+add_subdirectory (ecl/wutest)
+add_subdirectory (esp/bindings/SOAP/soaplib)
+add_subdirectory (esp/clients/LoggingClient)
+add_subdirectory (esp/clients/wsecl)
+add_subdirectory (esp/clients/WUManager)
+add_subdirectory (esp/platform)
+add_subdirectory (esp/protocols/http)
+add_subdirectory (esp/services/ecldirect)
+add_subdirectory (esp/services/ws_access)
+add_subdirectory (esp/services/ws_account)
+add_subdirectory (esp/services/ws_config)
+add_subdirectory (esp/services/ws_dfu)
+add_subdirectory (esp/services/ws_ecl)
+add_subdirectory (esp/services/ws_fileio)
+add_subdirectory (esp/services/ws_fs)
+add_subdirectory (esp/services/ws_machine)
+add_subdirectory (esp/services/ws_roxiequery)
+add_subdirectory (esp/services/ws_smc)
+add_subdirectory (esp/services/ws_topology)
+add_subdirectory (esp/services/ws_workunits)
+add_subdirectory (esp/services/WsDeploy)
+add_subdirectory (esp/smc/SMCLib)
+add_subdirectory (esp/test/httptest)
+add_subdirectory (esp/tools/soapplus)
+add_subdirectory (plugins/auditlib)
+add_subdirectory (plugins/debugservices)
+add_subdirectory (plugins/fileservices)
+add_subdirectory (plugins/logging)
+add_subdirectory (plugins/parselib)
+add_subdirectory (plugins/stringlib)
+add_subdirectory (plugins/unicodelib)
+add_subdirectory (plugins/workunitservices)
+add_subdirectory (roxie/ccd)
+add_subdirectory (roxie/roxie)
+add_subdirectory (roxie/roxieclient)
+add_subdirectory (roxie/roxieclienttest)
+add_subdirectory (roxie/roxiemem)
+add_subdirectory (roxie/roxiepipe)
+add_subdirectory (roxie/udplib)
+add_subdirectory (rtl/eclrtl)
+add_subdirectory (rtl/nbcd)
+add_subdirectory (dali/sasha)
+add_subdirectory (services/runagent)
+add_subdirectory (system/hrpc)
+add_subdirectory (system/jhtree)
+add_subdirectory (system/jlib)
+add_subdirectory (system/lzma)
+add_subdirectory (system/mp)
+add_subdirectory (system/mp/test)
+add_subdirectory (system/security/LdapSecurity)
+add_subdirectory (system/security/securesocket)
+add_subdirectory (system/security/test/ldapsecuritytest)
+add_subdirectory (system/security/test/myssl)
+if (USE_ZLIB)
+  add_subdirectory (system/security/zcrypt)
+endif()
+add_subdirectory (system/xmllib)
+
+if(LEGACYTHOR)
+    add_subdirectory (ln/thor/activities)
+    add_subdirectory (ln/thor/graph)
+    add_subdirectory (ln/thor/master)
+    add_subdirectory (ln/thor/mfilemanager)
+    add_subdirectory (ln/thor/msort)
+    add_subdirectory (ln/thor/slave)
+    add_subdirectory (ln/thor/thorcodectx)
+endif(LEGACYTHOR)
+
+add_subdirectory (thorlcr/activities)
+add_subdirectory (thorlcr/graph)
+add_subdirectory (thorlcr/master)
+add_subdirectory (thorlcr/mfilemanager)
+add_subdirectory (thorlcr/msort)
+add_subdirectory (thorlcr/slave)
+add_subdirectory (thorlcr/thorcodectx)
+add_subdirectory (tools/backupnode)
+add_subdirectory (tools/combine)
+add_subdirectory (tools/dumpkey)
+add_subdirectory (tools/keydiff)
+add_subdirectory (tools/pstart)
+add_subdirectory (tools/pskill)
+add_subdirectory (tools/testsocket)
+add_subdirectory (tools/swapnode)
+add_subdirectory (tools/vkey)
+add_subdirectory (tools/xmlsize)
+add_subdirectory (tools/wuget)
+add_subdirectory (tools/copyexp)
+add_subdirectory (tools/genht)
+
+
+# start-stop-daemon is linux only.
+if ( CMAKE_SYSTEM MATCHES Linux )
+    message ("-- Building start-stop-daemon")
+    add_subdirectory (tools/start-stop-daemon)
+endif()
+
+###
+## CPack install and packaging setup.
+###
+INCLUDE(InstallRequiredSystemLibraries)
+SET(CPACK_PACKAGE_VERSION_MAJOR ${majorver})
+SET(CPACK_PACKAGE_VERSION_MINOR ${minorver})
+SET(CPACK_PACKAGE_VERSION_PATCH ${point}${stagever})
+set ( CPACK_PACKAGE_CONTACT "ossdevelopment@lexisnexis.com" )
+set ( CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${CPACK_RPM_PACKAGE_VERSION}_${version}" )
+set( CPACK_SOURCE_GENERATOR TGZ )
+set ( CPACK_RPM_PACKAGE_VERSION "${projname}")
+SET(CPACK_RPM_PACKAGE_RELEASE "${version}")
+if ( ${ARCH64BIT} EQUAL 1 )
+    set ( CPACK_RPM_PACKAGE_ARCHITECTURE "x86_64")
+else( ${ARCH64BIT} EQUAL 1 )
+    set ( CPACK_RPM_PACKAGE_ARCHITECTURE "i386")
+endif ( ${ARCH64BIT} EQUAL 1 )
+set(CPACK_SYSTEM_NAME "${CMAKE_SYSTEM_NAME}-${CPACK_RPM_PACKAGE_ARCHITECTURE}")
+
+if ( CMAKE_SYSTEM MATCHES Linux )
+    EXECUTE_PROCESS (
+                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/distrocheck.sh
+                    OUTPUT_VARIABLE packageManagement
+                        ERROR_VARIABLE  packageManagement
+                )
+    EXECUTE_PROCESS (
+                COMMAND ${HPCC_SOURCE_DIR}/cmake_modules/getpackagerevisionarch.sh
+                    OUTPUT_VARIABLE packageRevisionArch
+                        ERROR_VARIABLE  packageRevisionArch
+                )
+    message ( "-- Auto Detecting Packaging type")
+    message ( "-- distro uses ${packageManagement}, revision is ${packageRevisionArch}" )
+
+    if ( ${packageManagement} STREQUAL "DEB" )
+        set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${CPACK_RPM_PACKAGE_VERSION}_${version}${packageRevisionArch}")
+    elseif ( ${packageManagement} STREQUAL "RPM" )
+        set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${CPACK_RPM_PACKAGE_VERSION}_${version}.${packageRevisionArch}")
+        else()
+        set(CPACK_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${CPACK_RPM_PACKAGE_VERSION}_${version}-${CPACK_SYSTEM_NAME}")
+    endif ()
+endif ( CMAKE_SYSTEM MATCHES Linux )
+MESSAGE ("-- Current release version is ${CPACK_PACKAGE_FILE_NAME}")
+set ( CPACK_SOURCE_PACKAGE_FILE_NAME "${CMAKE_PROJECT_NAME}-${CPACK_RPM_PACKAGE_VERSION}_${version}" )
+set( CPACK_SOURCE_GENERATOR TGZ )
+set(CPACK_SOURCE_IGNORE_FILES
+        "~$"
+        "\\\\.cvsignore$"
+        "^${PROJECT_SOURCE_DIR}.*/CVS/"
+        "^${PROJECT_SOURCE_DIR}.*/.svn/"
+        "^${PROJECT_SOURCE_DIR}/ln/"
+        "^${PROJECT_SOURCE_DIR}/externals/"
+        "^${PROJECT_SOURCE_DIR}.*/*.mk$"
+        "^${PROJECT_SOURCE_DIR}/makefile$"
+        "^${PROJECT_SOURCE_DIR}/make.common$"
+        "^${PROJECT_SOURCE_DIR}/make.post$"
+        "^${PROJECT_SOURCE_DIR}/build$"
+        "^${PROJECT_SOURCE_DIR}/buildall$"
+        "^${PROJECT_SOURCE_DIR}/lastbuilds$"
+        "^${PROJECT_SOURCE_DIR}/imerge$"
+        "^${PROJECT_SOURCE_DIR}/tmerge$"
+        "^${PROJECT_SOURCE_DIR}/tmerge.bat$"
+        "^${PROJECT_SOURCE_DIR}/tag$"
+        "^${PROJECT_SOURCE_DIR}/tag_build$"
+        "^${PROJECT_SOURCE_DIR}/old_tag$"
+        "^${PROJECT_SOURCE_DIR}/ecl/regress/"
+    "^${PROJECT_SOURCE_DIR}/testing/"
+        )
+
+###
+## Run file configuration to set build tag
+###
+set( BUILD_TAG "${CPACK_RPM_PACKAGE_VERSION}_${CPACK_RPM_PACKAGE_RELEASE}")
+configure_file(${HPCC_SOURCE_DIR}/build-config.h.cmake "build-config.h" )
+configure_file("initfiles/etc/DIR_NAME/version.in" "version")
+configure_file("initfiles/etc/DIR_NAME/environment.conf.in" "environment.conf")
+configure_file("initfiles/etc/DIR_NAME/environment.xml.in" "environment.xml")
+configure_file("initfiles/etc/DIR_NAME/configmgr/configmgr.conf.in" "configmgr.conf")
+configure_file("initfiles/etc/DIR_NAME/configmgr/esp.xml.in" "esp.xml")
+if ( ${HPCC_PROJECT} STREQUAL "community" )
+    configure_file("initfiles/componentfiles/configxml/buildsetCC.xml.in" "buildset.xml")
+else ()
+    configure_file("initfiles/componentfiles/configxml/buildsetEE.xml.in" "buildset.xml")
+endif()
+
+
+###
+## CPack commands in this section require cpack 2.8.1 to function.
+## When using cpack 2.8.1, the command "make package" will create
+## an RPM.
+###
+
+if (NOT "${CMAKE_VERSION}" VERSION_LESS "2.8.1")
+    if ( CMAKE_SYSTEM MATCHES Linux )
+        if ( ${packageManagement} STREQUAL "DEB" )
+            if ("${CMAKE_VERSION}" VERSION_EQUAL "2.8.2")
+                message("WARNING: CMAKE 2.8.2  would not build DEB package")
+            else ()
+                set ( CPACK_GENERATOR "${packageManagement}" )
+                message("-- Will build DEB package")
+                ###
+                ## CPack instruction required for Debian
+                ###
+                set ( CPACK_DEBIAN_PACKAGE_DEPENDS "libboost-regex-dev, libicu-dev, libxalan110, libxerces-c28, binutils, libldap2-dev, openssl, zlib1g, g++, openssh-client, openssh-server")
+                message ("-- Packing BASH installation files")
+                set ( CPACK_DEBIAN_PACKAGE_CONTROL_EXTRA "${CMAKE_BINARY_DIR}/initfiles/bash/sbin/deb/postinst;${CMAKE_BINARY_DIR}/initfiles/sbin/prerm;${CMAKE_BINARY_DIR}/initfiles/bash/sbin/deb/postrm" )
+            endif ()
+
+        elseif ( ${packageManagement} STREQUAL "RPM" )
+            set ( CPACK_GENERATOR "${packageManagement}" )
+            ###
+            ## CPack instruction required for RPM 
+            ###
+            message("-- Will build RPM package")
+            set ( CPACK_RPM_PACKAGE_REQUIRES "boost, openldap, libicu, m4, libtool, xalan-c, xerces-c, gcc-c++, openssh-server, openssh-clients")
+            message ("-- Packing BASH installation files")
+            set ( CPACK_RPM_POST_INSTALL_SCRIPT_FILE "${CMAKE_BINARY_DIR}/initfiles/bash/sbin/deb/postinst" )
+
+            set ( CPACK_RPM_PRE_UNINSTALL_SCRIPT_FILE "${CMAKE_BINARY_DIR}/initfiles/sbin/prerm" )
+            set ( CPACK_RPM_POST_UNINSTALL_SCRIPT_FILE "${CMAKE_BINARY_DIR}/initfiles/bash/sbin/deb/postrm" )
+                else()
+            message("WARNING: Unsupported package ${packageManagement}.")
+        endif ()
+
+    endif ( CMAKE_SYSTEM MATCHES Linux )
+
+else()
+    message("WARNING: CMAKE 2.8.1 or later required to create RPMs from this project")
+endif()
+
+###
+## Below are the non-compile based install scripts required for
+## the hpcc platform.
+###
+
+Install ( FILES ${HPCC_SOURCE_DIR}/${LICENSE_FILE} DESTINATION ${OSSDIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE )
+include( install_directory/overall.install )
+include( install_directory/bash_startup.install )
+include( install_directory/WsReferenceTracker.install )
+include( install_directory/dafilesrv.install )
+include( install_directory/dali.install )
+include( install_directory/dfuplus.install )
+include( install_directory/dfuserver.install )
+include( install_directory/dkcslave.install ) #: MORE - file name is wrong!
+include( install_directory/eclagent.install )       
+include( install_directory/ecldirect.install )
+include( install_directory/ecllibrary.install )
+include( install_directory/eclplus.install )
+include( install_directory/eclcc.install )      
+include( install_directory/eclccserver.install )      
+include( install_directory/eclscheduler.install )
+include( install_directory/esp.install )             
+include( install_directory/espsmc.install )          
+include( install_directory/ftslave.install )
+#include( install_directory/regressionSuite.install )
+include( install_directory/roxie.install )           
+include( install_directory/sasha.install )           
+include( install_directory/thor.install )            
+include( install_directory/ws_account.install )
+include( install_directory/ws_watchlist.install )
+include( install_directory/ws_ecl.install )
+include( install_directory/configgen.install )
+include( install_directory/configmgr.install )
+include( install_directory/agentexec.install )
+include (CPack)

LICENSE.txt

@@ -0,0 +1,3 @@
+Copyright (C) 2011 HPCC Systems.
+License Agreement: http://hpccsystems.com/hpcc-ce-binary-license-agreement
+

README

@@ -0,0 +1,50 @@
+ 
+Copyright (C) <2011> <LexisNexis Risk Data Management Inc.>
+ 
+All rights reserved. This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as
+published by the Free Software Foundation, either version 3 of the
+License, or (at your option) any later version.
+ 
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU Affero General Public License for more details.
+ 
+You should have received a copy of the GNU Affero General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+http://hpccsystems.com
+ 
+ 
+Description:
+------------
+This file is the top level handling for 
+cmake based compilation and build process.
+ 
+To build for Linux:
+1. Check out sources (for example, to directory ~/hpcc)
+2. Create a build directory - either as a child of hpcc or elsewhere
+3. cd to the build directory
+4a.To create makefiles to build native release version for local machine, run
+cmake ~/hpcc
+4b.To create makefiles to build native debug version, run
+cmake -DCMAKE_BUILD_TYPE=Debug ~/hpcc
+4c.To create makefiles to build 32-bit version from 64-bit host, run
+cmake -DCMAKE_C_FLAGS:STRING="-m32 -march=i386" -DCMAKE_CXX_FLAGS:STRING="-m32 -march=i386" ~/hpcc
+5. To build the makefiles just created above, run
+make
+6. Executables will be created in ./<releasemode>/bin and ./<releasemode>/libs
+ 
+To build for Windows:
+1. Check out sources (for example, to directory c:\hpcc)
+2. Create a build directory - either as a child of hpcc or elsewhere
+3. cd to the build directory
+4. To create a Visual Studio project, run
+cmake c:\hpcc -G "Visual Studio 9 2008"
+The sln file hpccsystems-platform.sln will be created in the current directory, and will support debug and release targets
+5. To build the project, load the solution into the visual studio IDE and build in the usual way.
+6. Executables will be created in (for example) c:\hpcc\bin\<release_mode>
+ 
+ 

baseaddr.txt

@@ -0,0 +1,119 @@
+;    Copyright (C) 2011 HPCC Systems.
+;
+;    All rights reserved. This program is free software: you can redistribute it and/or modify
+;    it under the terms of the GNU Affero General Public License as
+;    published by the Free Software Foundation, either version 3 of the
+;    License, or (at your option) any later version.
+;
+;    This program is distributed in the hope that it will be useful,
+;    but WITHOUT ANY WARRANTY; without even the implied warranty of
+;    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+;    GNU Affero General Public License for more details.
+;
+;    You should have received a copy of the GNU Affero General Public License
+;    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+; This file is directly used by the linker, and should warn about any clashes.
+; Reserved:
+;
+; C++ compiler:
+
+; cl.exe    00400000 00410000   fixed
+; c1.dll    10600000 106A4000       
+; c1xx.dll  10400000 10522000
+; c2.dll    10700000 107B5000
+; link.exe  00400000 00472000    fixed
+
+; Rest of the system: (sizes approx 2x current debug size)
+;
+; name      base-addr  max-size
+ftsslave    0x00400000 0x00100000
+
+jlib        0x00500000 0x00100000
+bcd     0x00600000 0x00040000
+eclrtl      0x00640000 0x00040000
+dab     0x00680000 0x00040000
+remote      0x006C0000 0x00090000
+dalift      0x00750000 0x00040000
+dataxfer    0x00790000 0x00020000
+
+hthor       0x00700000 0x00050000
+deftype     0x00750000 0x00050000
+thorhelper  0x007a0000 0x00140000           ; doesn't load in parallel with the following entries
+deffile     0x00820000 0x000e0000
+loader          0x00960000 0x00040000
+
+
+; HOLe specific dlls
+hoparse     0x00A00000 0x000D0000
+hons        0x00AD0000 0x00030000
+readscript  0x00B00000 0x00080000
+
+processor   0x00C00000 0x00060000
+collator    0x00C00000 0x00060000
+hoserver    0x00C00000 0x00080000
+hoagent     0x00C00000 0x00060000
+msortdll    0x00C00000 0x00100000
+pipeline    0x00D00000 0x00060000
+commonext       0x00E00000 0x00010000
+socache         0x00E20000 0x00010000
+
+querydll    0x00480000 0x00100000
+
+
+; ecl server etc.
+hql         0x00A00000 0x00300000
+hqlcpp      0x00D00000 0x00280000
+
+; eclplus etc.
+fileviewer  0x00D00000 0x00060000
+
+; eclagent style program
+fileservices    0x00C00000 0x00060000
+workunitservices    0x00C60000 0x00010000
+
+;
+; for thor (intentional overlap with hole only dlls)
+activityslaves  0x00A00000 0x000D0000
+activityslaves_lcr  0x00A00000 0x000D0000
+activitymasters 0x00A00000 0x000D0000 ; never slaves/master in same process
+activitymasters_lcr 0x00A00000 0x000D0000 ; never slaves/master in same process
+graph           0x00B00000 0x00075000
+graph_lcr       0x00B00000 0x00075000
+graphslave      0x00B80000 0x00080000 
+graphslave_lcr  0x00B80000 0x00080000 
+graphmaster     0x00B80000 0x00080000 ; never in same process as graphslave
+graphmaster_lcr 0x00B80000 0x00080000 ; never in same process as graphslave_lcr
+mfilemanager    0x00D00000 0x00075000
+mfilemanager_lcr    0x00D00000 0x00075000
+
+dalibase        0x00D80000 0x00100000
+mp              0x00E90000 0x00020000
+environment     0x00EB0000 0x00050000
+sasha           0x00F00000 0x00020000
+dfuwu           0x00F00000 0x00020000
+workunit        0x00F20000 0x00080000
+jhtree          0x00FA0000 0x00060000
+dllserver       0x01000000 0x00040000
+mysql       0x01040000 0x00040000
+boostregex      0x01080000 0x00120000
+icuuc           0x011A0000 0x000A0000
+icuin           0x01240000 0x000C0000
+icudata         0x01300000 0x00810000
+snmputils       0x014A0000 0x00040000
+schedulectrl    0x014E0000 0x00040000
+
+roxiemanager    0x01560000 0x00080000
+roxiecommlib    0x01600000 0x00080000
+
+; plugins.
+parselib        0x01500000 0x00020000
+logging         0x01520000 0x00020000
+auditlib        0x01540000 0x00020000
+
+;
+; More could add /FIXED attributes to the linker options to reduce file size
+; especially the .exe should have no effect
+;
+

build-config.h.cmake

@@ -0,0 +1,87 @@
+#ifndef PREFIX
+    #cmakedefine PREFIX "${PREFIX}"
+#endif
+
+#ifndef EXEC_PREFIX
+    #cmakedefine EXEC_PREFIX "${EXEC_PREFIX}"
+#endif
+
+#ifndef CONFIG_PREFIX
+    #cmakedefine CONFIG_PREFIX "${CONFIG_PREFIX}"
+#endif
+
+#ifndef DIR_NAME
+    #cmakedefine DIR_NAME "${DIR_NAME}"
+#endif
+
+#ifndef INSTALL_DIR
+    #define INSTALL_DIR "${INSTALL_DIR}"
+#endif
+
+#ifndef LIB_DIR
+    #cmakedefine LIB_DIR "${LIB_PATH}"
+#endif
+
+#ifndef EXEC_DIR
+    #cmakedefine EXEC_DIR "${EXEC_PATH}"
+#endif
+
+#ifndef COMPONENTFILES_DIR
+    #cmakedefine COMPONENTFILES_DIR "${COMPONENTFILES_PATH}"
+#endif
+
+#ifndef CONFIG_DIR
+    #define CONFIG_DIR "${CONFIG_DIR}"
+#endif
+
+#ifndef CONFIG_SOURCE_DIR
+    #cmakedefine CONFIG_SOURCE_DIR "${CONFIG_SOURCE_PATH}"
+#endif
+
+#ifndef ADMIN_DIR
+    #cmakedefine ADMIN_DIR "${ADMIN_PATH}"
+#endif
+
+#ifndef PLUGINS_DIR
+    #cmakedefine PLUGINS_DIR "${PLUGINS_PATH}"
+#endif
+
+#ifndef RUNTIME_DIR
+    #cmakedefine RUNTIME_DIR "${RUNTIME_PATH}"
+#endif
+
+#ifndef HOME_DIR
+    #cmakedefine HOME_DIR "${HOME_DIR}"
+#endif
+
+#ifndef LOCK_DIR
+    #cmakedefine LOCK_DIR "${LOCK_PATH}"
+#endif
+
+#ifndef PID_DIR
+    #cmakedefine PID_DIR "${PID_PATH}"
+#endif
+
+#ifndef LOG_DIR
+    #cmakedefine LOG_DIR "${LOG_PATH}"
+#endif
+
+#ifndef RUNTIME_USER
+    #cmakedefine RUNTIME_USER "${RUNTIME_USER}"
+#endif
+
+#ifndef ENV_XML_FILE
+    #cmakedefine ENV_XML_FILE "${ENV_XML_FILE}"
+#endif
+
+#ifndef ENV_CONF_FILE
+    #cmakedefine ENV_CONF_FILE "${ENV_CONF_FILE}"
+#endif
+
+#ifndef BUILD_TAG
+    #cmakedefine BUILD_TAG "${BUILD_TAG}"
+#endif
+
+#ifndef BUILD_LEVEL
+    #cmakedefine BUILD_LEVEL "${BUILD_LEVEL}"
+#endif

cmake_modules/FindBINUTILS.cmake

@@ -0,0 +1,53 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the BinUtils development library Once done this will define
+#
+#  BINUTILS_FOUND - system has the BinUtils library BINUTILS_INCLUDE_DIR - the BinUtils include directory BINUTILS_LIBRARIES - The libraries needed 
+#  to use BinUtils
+IF (NOT BINUTILS_FOUND)
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (${ARCH64BIT} EQUAL 1)
+      SET (osdir "linux64_gcc4.1.1")
+    ELSE()
+      SET (osdir "linux32_gcc4.1.1")
+    ENDIF()
+    FIND_PATH (BINUTILS_INCLUDE_DIR NAMES bfd.h PATHS "${EXTERNALS_DIRECTORY}/binutils/${osdir}/include" NO_DEFAULT_PATH)
+    FIND_LIBRARY (BINUTILS_LIBRARIES NAMES bfd PATHS "${EXTERNALS_DIRECTORY}/binutils/${osdir}/lib" NO_DEFAULT_PATH)
+    # We also need libiberty - but that is complicated by the fact that some distros ship libibery_pic that you have to use in .so's, while on others libiberty.a is PIC-friendly
+    FIND_LIBRARY (IBERTY_LIBRARIES NAMES iberty_pic PATHS "${EXTERNALS_DIRECTORY}/binutils/${osdir}/lib" NO_DEFAULT_PATH)
+    FIND_LIBRARY (IBERTY_LIBRARIES NAMES iberty PATHS "${EXTERNALS_DIRECTORY}/binutils/${osdir}/lib" NO_DEFAULT_PATH)
+  ENDIF()
+  # if we didn't find in externals, look in system include path
+  if (USE_NATIVE_LIBRARIES)
+    FIND_PATH (BINUTILS_INCLUDE_DIR NAMES bfd.h)
+    FIND_LIBRARY (BINUTILS_LIBRARIES NAMES bfd)
+    FIND_LIBRARY (IBERTY_LIBRARIES NAMES iberty_pic)
+    FIND_LIBRARY (IBERTY_LIBRARIES NAMES iberty)
+  endif()
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(BinUtils DEFAULT_MSG
+    BINUTILS_LIBRARIES
+    BINUTILS_INCLUDE_DIR
+    IBERTY_LIBRARIES
+  )
+  IF (BINUTILS_FOUND)
+    set (BINUTILS_LIBRARIES ${BINUTILS_LIBRARIES} ${IBERTY_LIBRARIES} )
+  ENDIF()
+  MARK_AS_ADVANCED(BINUTILS_INCLUDE_DIR BINUTILS_LIBRARIES)
+ENDIF()

cmake_modules/FindBOOST_REGEX.cmake

@@ -0,0 +1,78 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the Boost regex library
+# Once done this will define
+#
+#  BOOST_REGEX_FOUND - system has the Boost regex library
+#  BOOST_REGEX_INCLUDE_DIR - the Boost regex include directory
+#  BOOST_REGEX_LIBRARIES - The libraries needed to use Boost regex
+#  BOOST_REGEX_LIBRARY_DIR - The directory containing libraries needed to use Boost regex
+
+IF (NOT BOOST_REGEX_FOUND)
+  IF (UNIX)
+    SET (boost_regex_lib "boost_regex-mt")
+    IF(Boost_USE_STATIC_LIBS)
+      SET (boost_regex_lib "libboost_regex-mt.a")
+    ENDIF()
+  ELSEIF(WIN32)
+    SET (boost_regex_lib "libboost_regex-vc90-mt.lib") # note - this may not be the lib we need, but should be in same place as it...
+  ENDIF()
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux-x86_64-gcc4")
+      ELSE()
+        SET (osdir "linux-i686-gcc4")
+      ENDIF()
+    ELSEIF(WIN32)
+      SET (osdir "windows-i386-vc90")
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (BOOST_REGEX_INCLUDE_DIR NAMES boost/regex.h PATHS "${EXTERNALS_DIRECTORY}/boost/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (BOOST_REGEX_LIBRARIES NAMES ${boost_regex_lib} PATHS "${EXTERNALS_DIRECTORY}/boost/${osdir}/lib" NO_DEFAULT_PATH)
+    ENDIF() 
+  ENDIF()
+
+  # if we didn't find in externals, look in system include path
+  if (USE_NATIVE_LIBRARIES)
+     SET(Boost_ADDITIONAL_VERSIONS "1.44.0")
+     set(Boost_USE_MULTITHREADED ON)
+     find_package( Boost COMPONENTS regex )
+     if(Boost_FOUND)
+         set(BOOST_REGEX_LIBRARIES ${Boost_LIBRARIES})
+         set(BOOST_REGEX_INCLUDE_DIR ${Boost_INCLUDE_DIRS})
+     endif()
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(BOOST_REGEX DEFAULT_MSG
+    BOOST_REGEX_LIBRARIES 
+    BOOST_REGEX_INCLUDE_DIR
+  )
+
+  IF (BOOST_REGEX_FOUND)
+    STRING(REPLACE "/${boost_regex_lib}" "" BOOST_REGEX_LIBRARY_DIR "${BOOST_REGEX_LIBRARIES}")
+    IF (WIN32)
+      set (BOOST_REGEX_LIBRARIES "")  # the actual library to use is controlled by boost header files
+    ENDIF()
+  ENDIF()
+  MARK_AS_ADVANCED(BOOST_REGEX_INCLUDE_DIR BOOST_REGEX_LIBRARIES)
+ENDIF()

cmake_modules/FindCPPUNIT.cmake

@@ -0,0 +1,75 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the CppUnit unit testing library
+# Once done this will define
+#
+#  CPPUNIT_FOUND - system has the CppUnit library
+#  CPPUNIT_INCLUDE_DIR - the CppUnit include directory
+#  CPPUNIT_LIBRARIES - The libraries needed to use CppUnit
+
+IF (NOT CPPUNIT_FOUND)
+  IF (WIN32)
+    SET (cppunit_dll "cppunit_dll")
+  ELSE()
+    SET (cppunit_dll "cppunit")
+  ENDIF()
+
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+  
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux64_gcc4.1.1")
+      ELSE()
+        SET (osdir "linux32_gcc4.1.1")
+      ENDIF()
+    ELSEIF(WIN32)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "win64")
+      ELSE()
+        SET (osdir "win32")
+      ENDIF()
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (CPPUNIT_INCLUDE_DIR NAMES cppunit/TestFixture.h PATHS "${EXTERNALS_DIRECTORY}/cppunit/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (CPPUNIT_LIBRARIES NAMES ${cppunit_dll} PATHS "${EXTERNALS_DIRECTORY}/cppunit/lib/${osdir}" NO_DEFAULT_PATH)
+    ENDIF() 
+    
+  ENDIF()
+
+  # if we didn't find in externals, look in system include path
+  if (USE_NATIVE_LIBRARIES)
+    FIND_PATH (CPPUNIT_INCLUDE_DIR NAMES cppunit/TestFixture.h)
+    FIND_LIBRARY (CPPUNIT_LIBRARIES NAMES ${cppunit_dll})
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(CppUnit DEFAULT_MSG
+    CPPUNIT_LIBRARIES 
+    CPPUNIT_INCLUDE_DIR
+  )
+
+  IF (CPPUNIT_FOUND AND WIN32)
+    STRING(REPLACE "cppunit_dll" "cppunitd_dll" CPPUNIT_DEBUG_LIBRARIES "${CPPUNIT_LIBRARIES}")
+    set (CPPUNIT_LIBRARIES optimized ${CPPUNIT_LIBRARIES} debug ${CPPUNIT_DEBUG_LIBRARIES})
+  ENDIF()
+  MARK_AS_ADVANCED(CPPUNIT_INCLUDE_DIR CPPUNIT_LIBRARIES)
+ENDIF()

cmake_modules/FindICU.cmake

@@ -0,0 +1,68 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the ICU unicode library
+# Once done this will define
+#
+#  ICU_FOUND - system has the ICU library
+#  ICU_INCLUDE_DIR - the ICU include directory
+#  ICU_LIBRARIES - The libraries needed to use ICU
+
+IF (NOT ICU_FOUND)
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux64_gcc4.1.1")
+      ELSE()
+        SET (osdir "linux32_gcc4.1.1")
+      ENDIF()
+    ELSEIF(WIN32)
+      SET (osdir )
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (ICU_INCLUDE_DIR NAMES unicode/uchar.h PATHS "${EXTERNALS_DIRECTORY}/icu/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (ICU_LIBRARIES NAMES icuuc PATHS "${EXTERNALS_DIRECTORY}/icu/lib/${osdir}" NO_DEFAULT_PATH)
+    ENDIF()
+  ENDIF()
+
+  if (USE_NATIVE_LIBRARIES)
+    # if we didn't find in externals, look in system include path
+    FIND_PATH (ICU_INCLUDE_DIR NAMES unicode/uchar.h)
+    FIND_LIBRARY (ICU_LIBRARIES NAMES icuuc)
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(ICU DEFAULT_MSG
+    ICU_LIBRARIES 
+    ICU_INCLUDE_DIR
+  )
+  IF (ICU_FOUND)
+    IF (UNIX)
+      STRING(REPLACE "icuuc" "icui18n" ICU_EXTRA1 "${ICU_LIBRARIES}")
+      STRING(REPLACE "icuuc" "icudata" ICU_EXTRA2 "${ICU_LIBRARIES}")
+    ELSE()
+      STRING(REPLACE "icuuc" "icuin" ICU_EXTRA1 "${ICU_LIBRARIES}")
+    ENDIF()
+    set (ICU_LIBRARIES ${ICU_LIBRARIES} ${ICU_EXTRA1} ${ICU_EXTRA2} )
+  ENDIF()
+
+
+  MARK_AS_ADVANCED(ICU_INCLUDE_DIR ICU_LIBRARIES)
+ENDIF()

cmake_modules/FindMYSQL.cmake

@@ -0,0 +1,63 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the MYSQL library
+# Once done this will define
+#
+#  MYSQL_FOUND - system has the MYSQL library
+#  MYSQL_INCLUDE_DIR - the MYSQL include directory
+#  MYSQL_LIBRARIES - The libraries needed to use MYSQL
+
+IF (NOT MYSQL_FOUND)
+  IF (WIN32)
+    SET (mysql_lib "libmysql")
+  ELSE()
+    SET (mysql_lib "mysqlclient")
+  ENDIF()
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux64_gcc4.1.1")
+      ELSE()
+        SET (osdir "linux32_gcc4.1.1")
+      ENDIF()
+    ELSEIF(WIN32)
+      SET (osdir "lib")
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (MYSQL_INCLUDE_DIR NAMES mysql.h PATHS "${EXTERNALS_DIRECTORY}/mysql/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (MYSQL_LIBRARIES NAMES ${mysql_lib} PATHS "${EXTERNALS_DIRECTORY}/mysql/${osdir}" NO_DEFAULT_PATH)
+    ENDIF()
+  ENDIF()
+
+  if (USE_NATIVE_LIBRARIES)
+    # if we didn't find in externals, look in system include path
+    FIND_PATH (MYSQL_INCLUDE_DIR NAMES mysql.h PATH_SUFFIXES mysql)
+    FIND_LIBRARY (MYSQL_LIBRARIES NAMES ${mysql_lib} PATH_SUFFIXES mysql)
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(MYSQL DEFAULT_MSG
+    MYSQL_LIBRARIES 
+    MYSQL_INCLUDE_DIR
+  )
+
+  MARK_AS_ADVANCED(MYSQL_INCLUDE_DIR MYSQL_LIBRARIES)
+ENDIF()

cmake_modules/FindOPENLDAP.cmake

@@ -0,0 +1,79 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the OpenLDAP libraries
+# Once done this will define
+#
+#  OPENLDAP_FOUND - system has the OpenLDAP library
+#  OPENLDAP_INCLUDE_DIR - the OpenLDAP include directory
+#  OPENLDAP_LIBRARIES - The libraries needed to use OpenLDAP
+#
+#  Note that we use winldap for windows builds at present
+#
+IF (NOT OPENLDAP_FOUND)
+  IF (WIN32)
+    SET (ldap_dll "wldap32")
+    SET (ldap_inc "Winldap.h")
+  ELSE()
+    SET (ldap_dll "ldap_r")
+    SET (ldap_inc "ldap.h")
+  ENDIF()
+
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osincdir "openldap/linux64_gcc4.1.1/include")
+        SET (oslibdir "openldap/linux64_gcc4.1.1")
+      ELSE()
+        SET (osincdir "openldap/linux32_gcc4.1.1/include")
+        SET (oslibdir "openldap/linux32_gcc4.1.1")
+      ENDIF()
+    ELSEIF(WIN32)
+        SET (osincdir "winldap/include")
+        SET (oslibdir "winldap/lib")
+    ELSE()
+      SET (osincdir "unknown")
+    ENDIF()
+    IF (NOT ("${osincdir}" STREQUAL "unknown"))
+      FIND_PATH (OPENLDAP_INCLUDE_DIR NAMES ${ldap_inc} PATHS "${EXTERNALS_DIRECTORY}/${osincdir}" NO_DEFAULT_PATH)
+      FIND_LIBRARY (OPENLDAP_LIBRARIES NAMES ${ldap_dll} PATHS "${EXTERNALS_DIRECTORY}/${oslibdir}" NO_DEFAULT_PATH)
+    ENDIF()
+  ENDIF()
+
+  # if we didn't find in externals, look in system include path
+  if (USE_NATIVE_LIBRARIES)
+    FIND_PATH (OPENLDAP_INCLUDE_DIR NAMES ${ldap_inc})
+    FIND_LIBRARY (OPENLDAP_LIBRARIES NAMES ${ldap_dll})
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(OpenLDAP DEFAULT_MSG
+    OPENLDAP_LIBRARIES
+    OPENLDAP_INCLUDE_DIR
+  )
+  IF (OPENLDAP_FOUND)
+    IF (UNIX)
+      STRING(REPLACE "ldap_r" "lber" OPENLDAP_EXTRA "${OPENLDAP_LIBRARIES}")
+      set (OPENLDAP_LIBRARIES ${OPENLDAP_LIBRARIES} ${OPENLDAP_EXTRA} )
+    ELSE()
+      set (OPENLDAP_LIBRARIES ${OPENLDAP_LIBRARIES} netapi32 )
+    ENDIF()
+  ENDIF()
+
+  MARK_AS_ADVANCED(OPENLDAP_INCLUDE_DIR OPENLDAP_LIBRARIES)
+ENDIF()

cmake_modules/FindOPENSSL.cmake

@@ -0,0 +1,76 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the OPENSSL library
+# Once done this will define
+#
+#  OPENSSL_FOUND - system has the OPENSSL library
+#  OPENSSL_INCLUDE_DIR - the OPENSSL include directory
+#  OPENSSL_LIBRARIES - The libraries needed to use OPENSSL
+
+IF (NOT OPENSSL_FOUND)
+  IF (WIN32)
+    SET (ssl_lib "ssleay32.lib")
+  ELSE()
+    SET (ssl_lib "libssl.so")
+  ENDIF()
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux64_gcc4.1.1")
+      ELSE()
+        SET (osdir "linux32_gcc4.1.1")
+      ENDIF()
+    ELSEIF(WIN32)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "win64")
+      ELSE()
+        SET (osdir "win32")
+      ENDIF()
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h PATHS "${EXTERNALS_DIRECTORY}/openssl/${osdir}/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (OPENSSL_LIBRARIES NAMES ${ssl_lib} PATHS "${EXTERNALS_DIRECTORY}/openssl/${osdir}/lib" NO_DEFAULT_PATH)
+    ENDIF()
+  ENDIF()
+
+  if (USE_NATIVE_LIBRARIES)
+    # if we didn't find in externals, look in system include path
+    FIND_PATH (OPENSSL_INCLUDE_DIR NAMES openssl/ssl.h)
+    FIND_LIBRARY (OPENSSL_LIBRARIES NAMES ${ssl_lib})
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(OPENSSL DEFAULT_MSG
+    OPENSSL_LIBRARIES 
+    OPENSSL_INCLUDE_DIR
+  )
+  IF (OPENSSL_FOUND)
+    STRING(REPLACE "/${ssl_lib}" "" OPENSSL_LIBRARY_DIR "${OPENSSL_LIBRARIES}")
+    IF (UNIX)
+      STRING(REPLACE "libssl" "libcrypto" OPENSSL_EXTRA "${OPENSSL_LIBRARIES}")
+    ELSE()
+      STRING(REPLACE "ssleay32" "libeay32" OPENSSL_EXTRA "${OPENSSL_LIBRARIES}")
+    ENDIF()
+    set (OPENSSL_LIBRARIES ${OPENSSL_LIBRARIES} ${OPENSSL_EXTRA} )
+  ENDIF()
+
+  MARK_AS_ADVANCED(OPENSSL_INCLUDE_DIR OPENSSL_LIBRARIES)
+ENDIF()

cmake_modules/FindSVN.cmake

@@ -0,0 +1,94 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the SVN library
+# Once done this will define
+#
+#  SVN_FOUND - system has the SVN library
+#  SVN_INCLUDE_DIR - the SVN include directory
+#  SVN_APR_INCLUDE_DIR - the SVN APR include directory
+#  SVN_LIBRARIES - The libraries needed to use SVN
+
+IF (NOT SVN_FOUND)
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux-x86_64-gcc4")
+      ELSE()
+        SET (osdir "linux-i686-gcc4")
+      ENDIF()
+    ELSEIF(WIN32)
+      SET (osdir "windows-i386-vc90")
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      IF (UNIX)
+    FIND_PATH (SVN_INCLUDE_DIR NAMES svn_wc.h PATHS "${EXTERNALS_DIRECTORY}/subversion/${osdir}/include/subversion-1" NO_DEFAULT_PATH)
+    FIND_PATH (SVN_APR_INCLUDE_DIR NAMES apr.h PATHS "${EXTERNALS_DIRECTORY}/subversion/${osdir}/include/apr-1" NO_DEFAULT_PATH)
+      ELSEIF(WIN32)
+    FIND_PATH (SVN_INCLUDE_DIR NAMES apr.h PATHS "${EXTERNALS_DIRECTORY}/subversion/${osdir}/include" NO_DEFAULT_PATH)
+      ENDIF()
+      IF(WIN32)
+        FIND_LIBRARY (SVN_LIBRARIES NAMES libapr-1 PATHS "${EXTERNALS_DIRECTORY}/subversion/${osdir}/lib" NO_DEFAULT_PATH)
+      ELSE()
+        FIND_LIBRARY (SVN_LIBRARIES NAMES svn_client-1 PATHS "${EXTERNALS_DIRECTORY}/subversion/${osdir}/lib" NO_DEFAULT_PATH)
+      ENDIF()
+    ENDIF()
+  ENDIF()
+
+  if (USE_NATIVE_LIBRARIES)
+    # if we didn't find in externals, look in system include path
+    IF(WIN32)
+        FIND_PATH (SVN_INCLUDE_DIR NAMES apr.h)
+        FIND_LIBRARY (SVN_LIBRARIES NAMES libapr-1)
+    ELSE()
+        FIND_PATH (SVN_INCLUDE_DIR NAMES apr.h)
+        FIND_LIBRARY (SVN_LIBRARIES NAMES svn_client-1)
+    ENDIF()
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(SVN DEFAULT_MSG
+    SVN_LIBRARIES 
+    SVN_INCLUDE_DIR
+  )
+  IF (SVN_FOUND)
+    IF(WIN32)
+        STRING(REPLACE "libapr-1" "libapriconv-1" SVN_EXTRA1 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "libaprutil-1" SVN_EXTRA2 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_client-1" SVN_EXTRA3 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_delta-1" SVN_EXTRA4 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_diff-1" SVN_EXTRA5 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_fs-1" SVN_EXTRA6 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "libsvn_fs_util-1" SVN_EXTRA7 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "libsvn_fs_fs-1" SVN_EXTRA8 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_ra-1" SVN_EXTRA9 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "libsvn_ra_local-1" SVN_EXTRA10 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "libsvn_ra_svn-1" SVN_EXTRA11 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_repos-1" SVN_EXTRA12 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_subr-1" SVN_EXTRA13 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "svn_wc-1" SVN_EXTRA14 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "ShFolder" SVN_EXTRA15 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "xml" SVN_EXTRA16 "${SVN_LIBRARIES}")
+        STRING(REPLACE "libapr-1" "zlibstat" SVN_EXTRA17 "${SVN_LIBRARIES}")
+
+        set (SVN_LIBRARIES ${SVN_LIBRARIES} ${SVN_EXTRA1} ${SVN_EXTRA2} ${SVN_EXTRA3} ${SVN_EXTRA4} ${SVN_EXTRA5} ${SVN_EXTRA6} ${SVN_EXTRA7} ${SVN_EXTRA8} ${SVN_EXTRA9} ${SVN_EXTRA10} ${SVN_EXTRA11} ${SVN_EXTRA12} ${SVN_EXTRA13} ${SVN_EXTRA14} ${SVN_EXTRA15} ${SVN_EXTRA16} ${SVN_EXTRA17} )
+    ENDIF()
+  ENDIF()
+ENDIF()

cmake_modules/FindStartStopDaemon.cmake

@@ -0,0 +1,29 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+FIND_PROGRAM(START_STOP_DAEMON
+  NAMES start-stop-daemon
+  PATHS "/sbin"
+  NO_DEFAULT_PATH
+  )
+
+INCLUDE(FindPackageHandleStandardArgs)
+FIND_PACKAGE_HANDLE_STANDARD_ARGS(StartStopDaemon DEFAULT_MSG START_STOP_DAEMON)
+
+MARK_AS_ADVANCED(START_STOP_DAEMON)
+

cmake_modules/FindXALAN.cmake

@@ -0,0 +1,73 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the Xalan xml library
+# Once done this will define
+#
+#  XALAN_FOUND - system has the Xalan library
+#  XALAN_INCLUDE_DIR - the Xalan include directory
+#  XALAN_LIBRARIES - The libraries needed to use Xalan
+
+IF (NOT XALAN_FOUND)
+  IF (WIN32)
+    SET (xalan_libs "Xalan-C_1")
+  ELSE()
+    SET (xalan_libs "xalan-c")
+  ENDIF()
+
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux64_gcc4.1.1")
+      ELSE()
+        SET (osdir "linux32_gcc4.1.1")
+      ENDIF()
+    ELSEIF(WIN32)
+      SET (osdir "xalan-c/lib")
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (XALAN_INCLUDE_DIR NAMES xalanc/XPath/XObjectFactory.hpp PATHS "${EXTERNALS_DIRECTORY}/xalan/xalan-c/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (XALAN_LIBRARIES NAMES ${xalan_libs} PATHS "${EXTERNALS_DIRECTORY}/xalan/${osdir}" NO_DEFAULT_PATH)
+    ENDIF() 
+  ENDIF()
+
+  if (USE_NATIVE_LIBRARIES)
+    # if we didn't find in externals, look in system include path
+    FIND_PATH (XALAN_INCLUDE_DIR NAMES xalanc/XPath/XObjectFactory.hpp )
+    FIND_LIBRARY (XALAN_LIBRARIES NAMES ${xalan_libs})
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(Xalan DEFAULT_MSG
+    XALAN_LIBRARIES 
+    XALAN_INCLUDE_DIR
+  )
+
+  IF (XALAN_FOUND)
+    IF (WIN32)
+      STRING(REPLACE "Xalan-C_1" "Xalan-C_1D" XALAN_DEBUG_LIBRARIES "${XALAN_LIBRARIES}")
+      set (XALAN_LIBRARIES optimized ${XALAN_LIBRARIES} debug ${XALAN_DEBUG_LIBRARIES})
+    ELSE()
+      STRING(REPLACE "xalan-c" "xalanMsg" XALAN_EXTRA_LIBRARIES "${XALAN_LIBRARIES}")
+      set (XALAN_LIBRARIES ${XALAN_LIBRARIES} ${XALAN_EXTRA_LIBRARIES})
+    ENDIF()
+  ENDIF()
+  MARK_AS_ADVANCED(XALAN_INCLUDE_DIR XALAN_LIBRARIES)
+ENDIF()

cmake_modules/FindXERCES.cmake

@@ -0,0 +1,69 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# - Try to find the Xerces xml library
+# Once done this will define
+#
+#  XERCES_FOUND - system has the Xerces library
+#  XERCES_INCLUDE_DIR - the Xerces include directory
+#  XERCES_LIBRARIES - The libraries needed to use Xerces
+
+if (NOT XERCES_FOUND)
+  IF (WIN32)
+    SET (xerces_libs "xerces-c_2")
+  ELSE()
+    SET (xerces_libs "xerces-c")
+  ENDIF()
+
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux64_gcc4.1.1")
+      ELSE()
+        SET (osdir "linux32_gcc4.1.1")
+      ENDIF()
+    ELSEIF(WIN32)
+      SET (osdir "xerces-c/lib")
+    ELSE()
+      SET (osdir "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (XERCES_INCLUDE_DIR NAMES xercesc/util/XercesDefs.hpp PATHS "${EXTERNALS_DIRECTORY}/xalan/xerces-c/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (XERCES_LIBRARIES NAMES ${xerces_libs} PATHS "${EXTERNALS_DIRECTORY}/xalan/${osdir}" NO_DEFAULT_PATH)
+    ENDIF() 
+  ENDIF()
+
+  if (USE_NATIVE_LIBRARIES)
+    # if we didn't find in externals, look in system include path
+    FIND_PATH (XERCES_INCLUDE_DIR NAMES xercesc/util/XercesDefs.hpp )
+    FIND_LIBRARY (XERCES_LIBRARIES NAMES ${xerces_libs})
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(Xerces DEFAULT_MSG
+    XERCES_LIBRARIES 
+    XERCES_INCLUDE_DIR
+  )
+
+  IF (XERCES_FOUND AND WIN32)
+    STRING(REPLACE "xerces-c_2" "xerces-c_2D" XERCES_DEBUG_LIBRARIES "${XERCES_LIBRARIES}")
+    set (XERCES_LIBRARIES optimized ${XERCES_LIBRARIES} debug ${XERCES_DEBUG_LIBRARIES})
+  ENDIF()
+  MARK_AS_ADVANCED(XERCES_INCLUDE_DIR XERCES_LIBRARIES)
+ENDIF()
+

cmake_modules/FindZLIB.cmake

@@ -0,0 +1,72 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+    
+# - Try to find the ZLib compression library
+# Once done this will define
+#
+#  ZLIB_FOUND - system has the ZLib library
+#  ZLIB_INCLUDE_DIR - the ZLib include directory
+#  ZLIB_LIBRARIES - The libraries needed to use ZLib
+
+IF (NOT ZLIB_FOUND)
+  IF (WIN32)
+    SET (zlib_lib "zlib")
+  ELSE()
+    SET (zlib_lib "z")
+  ENDIF()
+
+  IF (NOT ${EXTERNALS_DIRECTORY} STREQUAL "")
+    IF (UNIX)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "linux64_gcc4.1.1")
+      ELSE()
+        SET (osdir "linux32_gcc4.1.1")
+      ENDIF()
+      SET (zlibver "1.2.5")
+    ELSEIF(WIN32)
+      IF (${ARCH64BIT} EQUAL 1)
+        SET (osdir "win64")
+        SET (zlibver "1.2.5")
+      ELSE()
+        SET (osdir "")
+        SET (zlibver "1.2.1")
+      ENDIF()  
+    ELSE()
+      SET (osdir "unknown")
+      SET (zlibver "unknown")
+    ENDIF()
+    IF (NOT ("${osdir}" STREQUAL "unknown"))
+      FIND_PATH (ZLIB_INCLUDE_DIR NAMES zlib.h PATHS "${EXTERNALS_DIRECTORY}/zlib/${zlibver}/include" NO_DEFAULT_PATH)
+      FIND_LIBRARY (ZLIB_LIBRARIES NAMES ${zlib_lib} PATHS "${EXTERNALS_DIRECTORY}/zlib/${zlibver}/lib/${osdir}" NO_DEFAULT_PATH)
+    ENDIF()
+  ENDIF()
+
+  if (USE_NATIVE_LIBRARIES)
+    # if we didn't find in externals, look in system include path
+    FIND_PATH (ZLIB_INCLUDE_DIR NAMES zlib.h)
+    FIND_LIBRARY (ZLIB_LIBRARIES NAMES ${zlib_lib})
+  endif()
+
+  include(FindPackageHandleStandardArgs)
+  find_package_handle_standard_args(ZLib DEFAULT_MSG
+    ZLIB_LIBRARIES 
+    ZLIB_INCLUDE_DIR
+  )
+
+  MARK_AS_ADVANCED(ZLIB_INCLUDE_DIR ZLIB_LIBRARIES)
+ENDIF()

cmake_modules/Version.cmake

@@ -0,0 +1,83 @@
+  MACRO(Subversion_WC_PROJNAME dir prefix)
+    EXECUTE_PROCESS(COMMAND
+          ${Subversion_SVN_EXECUTABLE} pg version:project_name ${dir}
+          OUTPUT_VARIABLE ${prefix}_WC_PROJNAME
+          ERROR_VARIABLE Subversion_svn_log_error
+          RESULT_VARIABLE Subversion_svn_log_result
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+    message("---- PROJNAME: ${${prefix}_WC_PROJNAME}")
+  ENDMACRO(Subversion_WC_PROJNAME)
+
+  MACRO(Subversion_WC_MAJOR dir prefix)
+        EXECUTE_PROCESS(COMMAND
+          ${Subversion_SVN_EXECUTABLE} pg version:major ${dir}
+          OUTPUT_VARIABLE ${prefix}_WC_MAJOR
+          ERROR_VARIABLE Subversion_svn_log_error
+          RESULT_VARIABLE Subversion_svn_log_result
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+        message("---- MAJOR: ${${prefix}_WC_MAJOR}")
+  ENDMACRO(Subversion_WC_MAJOR)
+
+  MACRO(Subversion_WC_MINOR dir prefix)
+        EXECUTE_PROCESS(COMMAND
+          ${Subversion_SVN_EXECUTABLE} pg version:minor ${dir}
+          OUTPUT_VARIABLE ${prefix}_WC_MINOR
+          ERROR_VARIABLE Subversion_svn_log_error
+          RESULT_VARIABLE Subversion_svn_log_result
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+        message("---- MINOR: ${${prefix}_WC_MINOR}")
+  ENDMACRO(Subversion_WC_MINOR)
+
+  MACRO(Subversion_WC_POINT dir prefix)
+        EXECUTE_PROCESS(COMMAND
+          ${Subversion_SVN_EXECUTABLE} pg version:point ${dir}
+          OUTPUT_VARIABLE ${prefix}_WC_POINT
+          ERROR_VARIABLE Subversion_svn_log_error
+          RESULT_VARIABLE Subversion_svn_log_result
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+        message("---- POINT: ${${prefix}_WC_POINT}")
+  ENDMACRO(Subversion_WC_POINT)
+
+  MACRO(Subversion_WC_SUFFIX dir prefix)
+        EXECUTE_PROCESS(COMMAND
+          ${Subversion_SVN_EXECUTABLE} pg version:suffix ${dir}
+          OUTPUT_VARIABLE ${prefix}_WC_SUFFIX
+          ERROR_VARIABLE Subversion_svn_log_error
+          RESULT_VARIABLE Subversion_svn_log_result
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+        message("---- SUFFIX: ${${prefix}_WC_SUFFIX}")
+  ENDMACRO(Subversion_WC_SUFFIX)
+
+  MACRO(Subversion_WC_MATURITY dir prefix)
+        EXECUTE_PROCESS(COMMAND
+          ${Subversion_SVN_EXECUTABLE} pg version:maturity ${dir}
+          OUTPUT_VARIABLE ${prefix}_WC_MATURITY
+          ERROR_VARIABLE Subversion_svn_log_error
+          RESULT_VARIABLE Subversion_svn_log_result
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+    if( ${prefix}_WC_MATURITY STREQUAL "")
+        set(${prefix}_WC_MATURITY "dev")
+    endif()
+        message("---- MATURITY: ${${prefix}_WC_MATURITY}")
+  ENDMACRO(Subversion_WC_MATURITY)
+
+  MACRO(Subversion_WC_SEQUENCE dir prefix)
+        EXECUTE_PROCESS(COMMAND
+          ${Subversion_SVN_EXECUTABLE} pg version:sequence ${dir}
+          OUTPUT_VARIABLE ${prefix}_WC_SEQUENCE
+          ERROR_VARIABLE Subversion_svn_log_error
+          RESULT_VARIABLE Subversion_svn_log_result
+          OUTPUT_STRIP_TRAILING_WHITESPACE)
+        message("---- SEQUENCE: ${${prefix}_WC_SEQUENCE}")
+  ENDMACRO(Subversion_WC_SEQUENCE)
+
+  MACRO(Subversion_WC_PG dir prefix)
+    Subversion_WC_PROJNAME(${dir} ${prefix})
+    Subversion_WC_MAJOR(${dir} ${prefix})
+    Subversion_WC_MINOR(${dir} ${prefix})
+    Subversion_WC_POINT(${dir} ${prefix})
+    Subversion_WC_SUFFIX(${dir} ${prefix})
+    Subversion_WC_MATURITY(${dir} ${prefix})
+    Subversion_WC_SEQUENCE(${dir} ${prefix})
+  ENDMACRO(Subversion_WC_PG)
+

cmake_modules/commonSetup.cmake

@@ -0,0 +1,316 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+# File  : commonSetup.cmake
+#
+#########################################################
+# Description:
+# ------------
+# sets up various cmake options. 
+#########################################################
+
+IF ("${COMMONSETUP_DONE}" STREQUAL "")
+  SET (COMMONSETUP_DONE 1)
+
+  MACRO (MACRO_ENSURE_OUT_OF_SOURCE_BUILD _errorMessage)
+    STRING(COMPARE EQUAL "${CMAKE_SOURCE_DIR}" "${CMAKE_BINARY_DIR}" insource)
+    IF (insource)
+      MESSAGE(FATAL_ERROR "${_errorMessage}")
+    ENDIF(insource)
+  ENDMACRO (MACRO_ENSURE_OUT_OF_SOURCE_BUILD)
+
+  macro_ensure_out_of_source_build("The LexisNexis Hpcc requires an out of source build.
+    Please remove the directory ${CMAKE_BINARY_DIR}/CMakeFiles 
+    and the file ${CMAKE_BINARY_DIR}/CMakeCache.txt,
+    then create a separate build directory and run 'cmake path_to_source [options]' there.")
+
+  cmake_policy ( SET CMP0011 NEW )
+
+  # For options that we don't presently support setting to off, we have a SET() below rather than an option...
+
+  option(USE_CPPUNIT "Enable unit tests (requires cppunit)" OFF)
+  set (USE_OPENLDAP 1)     # option(USE_OPENLDAP "Enable OpenLDAP support (requires OpenLDAP)" ON)
+  set (USE_ICU 1)          # option(USE_ICU "Enable unicode support (requires ICU)" ON)
+  set (USE_XALAN 1)        # option(USE_XALAN "Configure use of xalan" ON)
+  set (USE_XERCES 1)       # option(USE_XERCES "Configure use of xerces" ON)
+  option(USE_BOOST_REGEX "Configure use of boost regex" ON)
+  option(Boost_USE_STATIC_LIBS "Use boost_regex static library for RPM BUILD" OFF)
+  set (USE_OPENSSL 1)      # option(USE_OPENSSL "Configure use of OpenSSL" ON)
+  option(USE_ZLIB "Configure use of zlib" ON)
+  option(USE_NATIVE_LIBRARIES "Search standard OS locations for thirdparty libraries" ON)
+  option(LEGACYTHOR "Turn on building of Legacy Thor" OFF)
+
+  option(PORTALURL "Set url to hpccsystems portal download page")
+
+  if ( NOT PORTALURL )
+    set( PORTALURL "http://hpccsystems.com/download" )
+  endif()
+
+
+  ##########################################################
+
+  # common compiler/linker flags
+
+  if ("${CMAKE_BUILD_TYPE}" STREQUAL "")
+    set ( CMAKE_BUILD_TYPE "Release" )
+  elseif (NOT "${CMAKE_BUILD_TYPE}" MATCHES "Debug|Release")
+    message (FATAL_ERROR "Unknown build type $ENV{CMAKE_BUILD_TYPE}")
+  endif ()
+  message ("-- Making ${CMAKE_BUILD_TYPE} system")
+
+  if (CMAKE_SIZEOF_VOID_P EQUAL 8)
+    set ( ARCH64BIT 1 )
+  else ()
+    set ( ARCH64BIT 0 )
+  endif ()
+  message ("-- 64bit architecture is ${ARCH64BIT}")
+
+  set (CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -D_DEBUG -DDEBUG")
+
+  if (WIN32)
+    # On windows, the vcproj generator generates both windows and debug build capabilities, and the release mode is appended to the directory later
+    # This output location matches what our existing windows release scripts expect - might make sense to move out of tree sometime though
+    set ( EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}/bin" )
+    set ( LIBRARY_OUTPUT_PATH "${CMAKE_BINARY_DIR}/bin" )
+    # Workaround CMake's odd decision to default windows stack size to 10000000
+    STRING(REGEX REPLACE "/STACK:[0-9]+" "" CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS}")
+    SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} /LARGEADDRESSAWARE")
+
+    if (${ARCH64BIT} EQUAL 0)
+      add_definitions(/Zc:wchar_t-)
+    endif ()      
+      
+    if ("${CMAKE_BUILD_TYPE}" MATCHES "Debug")
+      add_definitions(/ZI)
+    endif ()
+  else ()
+    set ( EXECUTABLE_OUTPUT_PATH "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/bin" )
+    set ( LIBRARY_OUTPUT_PATH "${CMAKE_BINARY_DIR}/${CMAKE_BUILD_TYPE}/libs" )
+    if (${CMAKE_COMPILER_IS_GNUCXX})
+      SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -frtti -fPIC -fmessage-length=0")
+      SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --export-dynamic")
+      SET (CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -g -fno-default-inline -fno-inline-functions")
+    endif ()
+    # All of these are defined in platform.h too, but need to be defned before any system header is included
+    ADD_DEFINITIONS (-D_LARGEFILE_SOURCE=1 -D_LARGEFILE64_SOURCE=1 -D_FILE_OFFSET_BITS=64 -D__USE_LARGEFILE64=1 -D__USE_FILE_OFFSET64=1)
+    if (${CMAKE_SYSTEM_NAME} STREQUAL "FreeBSD")
+      SET (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -pthread")
+      SET (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -pthread")
+    endif ()
+  endif ()
+
+  macro(HPCC_ADD_LIBRARY target)
+    add_library(${target} ${ARGN})
+  endmacro(HPCC_ADD_LIBRARY target)
+
+  set ( SCM_GENERATED_DIR ${CMAKE_BINARY_DIR}/generated )
+  include_directories (${SCM_GENERATED_DIR})
+
+  ##################################################################
+
+  # Build tag generation from svn info
+
+  set(projname ${HPCC_PROJECT})
+  set(majorver ${HPCC_MAJOR})
+  set(minorver ${HPCC_MINOR})
+  set(point ${HPCC_POINT})
+  if ( "${HPCC_MATURITY}" STREQUAL "release" )
+    set(stagever "")
+    set(version ${majorver}.${minorver}.${point})
+  else()
+    set(stagever "${HPCC_MATURITY}${HPCC_SEQUENCE}")
+    set(version ${majorver}.${minorver}.${point}_${stagever})
+  endif()
+
+  IF ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+    set( version "${version}_Debug" )
+  ENDIF ("${CMAKE_BUILD_TYPE}" STREQUAL "Debug")
+
+  ###########################################################################
+
+  if (NOT "${EXTERNALS_DIRECTORY}" STREQUAL "")
+    message ("-- Using externals directory at ${EXTERNALS_DIRECTORY}")
+  endif()
+
+  IF ("${EXTERNALS_DIRECTORY}" STREQUAL "")
+    SET(bisoncmd "bison")
+    SET(flexcmd "flex")
+  ELSE()
+    SET(bisoncmd "${EXTERNALS_DIRECTORY}/bison/bison")
+    SET(flexcmd "${EXTERNALS_DIRECTORY}/bison/flex")
+  ENDIF()
+
+  IF ("${BISON_VERSION}" STREQUAL "")
+    IF (WIN32)
+      # cmake bug workaround - it converts path separators fine in add_custom_command but not here
+      STRING(REPLACE "/" "\\" BISON_exename "${bisoncmd}")  
+      IF (NOT "${EXTERNALS_DIRECTORY}" STREQUAL "")
+        set (BISON_exename "${BISON_exename}.bat")
+      ENDIF()
+    ELSE()
+      SET(BISON_exename "${bisoncmd}")  
+    ENDIF()
+    EXECUTE_PROCESS(COMMAND ${BISON_exename} --version
+      OUTPUT_VARIABLE BISON_version_output
+      ERROR_VARIABLE BISON_version_error
+      RESULT_VARIABLE BISON_version_result
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    STRING(REGEX REPLACE "^[^0-9]*([0-9.]+).*" "\\1" BISON_VERSION "${BISON_version_output}")
+  ENDIF()
+
+  IF ("${FLEX_VERSION}" STREQUAL "")
+    IF (WIN32)
+      # cmake bug workaround - it converts path separators fine in add_custom_command but not here
+      STRING(REPLACE "/" "\\" FLEX_exename "${flexcmd}")  
+      IF (NOT "${EXTERNALS_DIRECTORY}" STREQUAL "")
+        set (FLEX_exename "${FLEX_exename}.bat")
+      ENDIF()
+    ELSE()
+      SET(FLEX_exename "${flexcmd}")  
+    ENDIF()
+    EXECUTE_PROCESS(COMMAND ${FLEX_exename} --version
+      OUTPUT_VARIABLE FLEX_version_output
+      ERROR_VARIABLE FLEX_version_error
+      RESULT_VARIABLE FLEX_version_result
+      OUTPUT_STRIP_TRAILING_WHITESPACE)
+    STRING(REGEX REPLACE "^[^0-9]*([0-9.]+).*" "\\1" FLEX_VERSION "${FLEX_version_output}")
+  ENDIF()
+
+  IF ("${BISON_VERSION}" VERSION_LESS "2.4.1")
+    MESSAGE(FATAL_ERROR "You need bison version 2.4.1 or later to build this project (version ${BISON_VERSION} detected)")
+  ENDIF()
+
+  IF ("${FLEX_VERSION}" VERSION_LESS "2.5.35")
+    MESSAGE(FATAL_ERROR "You need flex version 2.5.35 or later to build this project (version ${FLEX_VERSION} detected)")
+  ENDIF()
+
+  IF (CMAKE_COMPILER_IS_GNUCXX)
+    EXECUTE_PROCESS(COMMAND ${CMAKE_CXX_COMPILER} -dumpversion OUTPUT_VARIABLE GNUCXX_VERSION OUTPUT_STRIP_TRAILING_WHITESPACE)
+    IF ("${GNUCXX_VERSION}" VERSION_LESS "4.1.1")
+      MESSAGE(FATAL_ERROR "You need Gnu c++ version 4.1.1 or later to build this project (version ${GNUCXX_VERSION} detected)")
+    ENDIF()
+  ENDIF()
+
+  ###########################################################################
+
+  # External library setup - some can be optionally selected based on USE_xxx flags, some are required
+
+  IF (NOT WIN32)
+    find_package(BINUTILS)
+    IF (NOT BINUTILS_FOUND)
+      message(FATAL_ERROR "BINUTILS package not found")
+    ENDIF()
+  ENDIF()
+
+  IF (USE_OPENLDAP)
+    find_package(OPENLDAP)
+    IF (OPENLDAP_FOUND)
+      add_definitions (-D_USE_OPENLDAP)
+    ELSE()
+      message(FATAL_ERROR "OPENLDAP requested but package not found")
+    ENDIF()
+  ENDIF(USE_OPENLDAP)
+
+  IF (USE_CPPUNIT)
+    find_package(CPPUNIT)
+    IF (CPPUNIT_FOUND)
+      add_definitions (-D_USE_CPPUNIT)
+      include_directories(${CPPUNIT_INCLUDE_DIR})
+    ELSE()
+      message(FATAL_ERROR "CPPUNIT requested but package not found")
+    ENDIF()
+  ENDIF(USE_CPPUNIT)
+
+  IF (USE_ICU)
+    find_package(ICU)
+    IF (ICU_FOUND)
+      add_definitions (-D_USE_ICU)
+      include_directories(${ICU_INCLUDE_DIR})
+    ELSE()
+      message(FATAL_ERROR "ICU requested but package not found")
+    ENDIF()
+  ENDIF(USE_ICU)
+
+  if(USE_XALAN)
+    find_package(XALAN)
+    if (XALAN_FOUND)
+      add_definitions (-D_USE_XALAN)
+    else()
+      message(FATAL_ERROR "XALAN requested but package not found")
+    endif()
+  endif(USE_XALAN)
+
+  if(USE_XERCES)
+    find_package(XERCES)
+    if (XERCES_FOUND)
+      add_definitions (-D_USE_XERCES)
+    else()
+      message(FATAL_ERROR "XERCES requested but package not found")
+    endif()
+  endif(USE_XERCES)
+
+  if(USE_ZLIB)
+    find_package(ZLIB)
+    if (ZLIB_FOUND)
+      add_definitions (-D_USE_ZLIB)
+    else()
+      message(FATAL_ERROR "ZLIB requested but package not found")
+    endif()
+  endif(USE_ZLIB)
+
+  if(USE_BOOST_REGEX)
+    find_package(BOOST_REGEX)
+    if (BOOST_REGEX_FOUND)
+      add_definitions (-D_USE_BOOST_REGEX)
+    else()
+      message(FATAL_ERROR "BOOST_REGEX requested but package not found")
+    endif()
+  endif(USE_BOOST_REGEX)
+
+  if(USE_OPENSSL)
+    find_package(OPENSSL)
+    if (OPENSSL_FOUND)
+      add_definitions (-D_USE_OPENSSL)
+      include_directories(${OPENSSL_INCLUDE_DIR})
+      link_directories(${OPENSSL_LIBRARY_DIR})
+    else()
+      message(FATAL_ERROR "OPENSSL requested but package not found")
+    endif()
+  endif(USE_OPENSSL)
+
+  if(USE_MYSQL)
+    find_package(MYSQL)
+    if (MYSQL_FOUND)
+      add_definitions (-D_USE_MYSQL)
+    else()
+      message(FATAL_ERROR "MYSQL requested but package not found")
+    endif()
+  else()
+    add_definitions (-D_NO_MYSQL)
+  endif(USE_MYSQL)
+
+  if(USE_SVN)
+    find_package(SVN)
+    if (SVN_FOUND)
+      add_definitions (-D_USE_SVN)
+    else()
+      message(FATAL_ERROR "SVN requested but package not found")
+    endif()
+  endif(USE_SVN)
+
+  ###########################################################################
+endif ("${COMMONSETUP_DONE}" STREQUAL "")

cmake_modules/distrocheck.sh

@@ -0,0 +1,86 @@
+#!/bin/sh
+
+#############################################
+ 
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with All rights reserved. This program is free software: you can redistribute program.  If not, see <http://www.gnu.org/licenses/>.
+    
+#############################################
+
+cat /etc/*release > temp.txt
+
+############### RPM DISTROS ##################
+
+# Distribution is openSuse
+VALUE=`grep -c -i 'suse' temp.txt` 
+if [ $VALUE -ge 1 ]; then
+    echo -n "RPM"
+    rm temp.txt
+    exit 1;
+fi
+
+# Distribution is Centos
+VALUE=`grep -c -i 'centos' temp.txt` 
+if [ $VALUE -ge 1 ]; then
+    echo -n "RPM"
+    rm temp.txt
+    exit 1;
+fi
+
+# Distribution is Mandriva 
+VALUE=`grep -c -i 'mandr' temp.txt` 
+if [ $VALUE -ge 1 ]; then
+    echo -n "RPM"
+    rm temp.txt
+    exit 1;
+fi
+
+# Distribution is Redhat
+VALUE=`grep -c -i 'redhat' temp.txt` 
+if [ $VALUE -ge 1 ]; then
+    echo -n "RPM"
+    rm temp.txt
+    exit 1;
+fi
+
+############### DEB DISTROS ##################
+
+# Distribution is Ubuntu
+VALUE=`grep -c -i 'ubuntu' temp.txt` 
+if [ $VALUE -ge 1 ]; then
+    echo -n "DEB"
+    rm temp.txt
+    exit 2;
+fi
+
+# Distribution is Debian
+VALUE=`grep -c -i 'DEBian' temp.txt` 
+if [ $VALUE -ge 1 ]; then
+    echo -n "DEB"
+    rm temp.txt
+    exit 2;
+fi
+
+
+############## OTHER DISTROS #################
+
+# Distribution is Gentoo
+VALUE=`grep -c -i 'gentoo' temp.txt` 
+if [ $VALUE -ge 1 ]; then
+    echo -n "TGZ"
+    rm temp.txt
+    exit 4;
+fi
+

cmake_modules/getpackagerevisionarch.sh

@@ -0,0 +1,64 @@
+#!/bin/bash
+
+OUTPUT=""
+ARCH=`uname -m`
+ARCH2=${ARCH}
+case "$ARCH" in
+  "x86_64")
+     ARCH="x86_64"
+     ARCH2="amd64"
+     ;;
+   "i386" | "i686")
+     ARCH="i386"
+     ARCH2="i386"
+     ;;
+esac
+
+
+if [ -e /etc/debian_version ]; then
+  if [ -e /etc/lsb-release ]; then
+    . /etc/lsb-release
+    OUTPUT="${DISTRIB_CODENAME}_${ARCH2}"
+  else
+    case `cat /etc/debian_version` in
+      5.*)
+        OUTPUT="lenny_${ARCH2}"
+        ;;
+      "sid")
+        OUTPUT="sid_${ARCH2}"
+        ;;
+    esac
+  fi
+elif [ -e /etc/redhat-release ]; then
+  if [ -x /bin/rpm ]; then
+    OUTPUT="${OUTPUT}\npackage=rpm"
+    OS_GROUP=`/bin/rpm -q --qf "%{NAME}" --whatprovides /etc/redhat-release | sed 's/-release.*//' |  tr '[A-Z]' '[a-z]'`
+    REDHAT_VERSION=`/bin/rpm -q --qf "%{VERSION}" --whatprovides /etc/redhat-release`
+    case "$OS_GROUP" in
+      "centos" | "fedora")
+        OUTPUT="el${REDHAT_VERSION}.${ARCH}"
+        ;;
+      "redhat")
+        REDHAT_RELEASE=`/bin/rpm -q --qf "%{RELEASE}" --whatprovides /etc/redhat-release| cut -d. -f1`
+        OUTPUT="el${REDHAT_VERSION}.${ARCH}"
+        ;;
+      esac
+    fi
+elif [ -e /etc/SuSE-release ]; then
+  if [ -x /bin/rpm ]; then
+    OS_GROUP=`/bin/rpm -q --qf "%{NAME}" --whatprovides /etc/SuSE-release | sed 's/-release.*//' |  tr '[A-Z]' '[a-z]'`
+    REDHAT_VERSION=`/bin/rpm -q --qf "%{VERSION}" --whatprovides /etc/SuSE-release`
+      case "$OS_GROUP" in
+        "opensuse" )
+          OUTPUT="suse${REDHAT_VERSION}.${ARCH}"
+          ;;
+      esac
+  fi
+elif [ -e /etc/gentoo-release ]; then
+  OUTPUT="gentoo"
+else
+  exit 0
+fi
+
+echo -n $OUTPUT
+

cmake_modules/optionDefaults.cmake

@@ -0,0 +1,89 @@
+
+if ( NOT PREFIX )
+    set( PREFIX "/opt" )
+endif()
+
+if ( NOT EXEC_PREFIX )
+    set( EXEC_PREFIX "/var" )
+endif()
+
+if ( NOT CONFIG_PREFIX )
+    set( CONFIG_PREFIX "/etc" )
+endif()
+
+if ( NOT DIR_NAME )
+    set( DIR_NAME "HPCCSystems" )
+endif()
+
+if ( NOT LIB_DIR )
+    set( LIB_DIR "lib" )
+endif()
+
+if ( NOT EXEC_DIR )
+    set( EXEC_DIR "bin" )
+endif()
+
+if ( NOT COMPONENTFILES_DIR )
+    set( COMPONENTFILES_DIR "componentfiles" )
+endif()
+
+if ( NOT ADMIN_DIR )
+    set( ADMIN_DIR "sbin" )
+endif()
+
+if ( NOT PLUGINS_DIR )
+    set( PLUGINS_DIR "plugins" )
+endif()
+
+if ( NOT CONFIG_SOURCE_DIR )
+    set( CONFIG_SOURCE_DIR "source" )
+endif()
+
+if ( NOT RUNTIME_DIR )
+    set( RUNTIME_DIR "lib" )
+endif()
+
+if ( NOT HOME_DIR )
+    set( HOME_DIR "/home" )
+endif()
+
+if ( NOT LOCK_DIR )
+    set( LOCK_DIR "lock" )
+endif()
+
+if ( NOT PID_DIR )
+    set( PID_DIR "run" )
+endif()
+
+if ( NOT LOG_DIR )
+    set( LOG_DIR "log" )
+endif()
+
+if ( NOT RUNTIME_USER )
+    set( RUNTIME_USER "hpcc" )
+endif()
+
+if ( NOT RUNTIME_GROUP )
+    set( RUNTIME_GROUP "hpcc" )
+endif()
+
+if ( NOT ENV_XML_FILE )
+    set( ENV_XML_FILE "environment.xml" )
+endif()
+
+if ( NOT ENV_CONF_FILE )
+    set( ENV_CONF_FILE "environment.conf" )
+endif()
+
+set( INSTALL_DIR "${PREFIX}/${DIR_NAME}" )
+set( CONFIG_DIR "${CONFIG_PREFIX}/${DIR_NAME}" )
+set( RUNTIME_PATH "${EXEC_PREFIX}/${RUNTIME_DIR}/${DIR_NAME}" )
+set( LOG_PATH "${EXEC_PREFIX}/${LOG_DIR}/${DIR_NAME}" )
+set( LOCK_PATH "${EXEC_PREFIX}/${LOCK_DIR}/${DIR_NAME}" )
+set( PID_PATH "${EXEC_PREFIX}/${PID_DIR}/${DIR_NAME}" )
+set( CONFIG_SOURCE_PATH "${CONFIG_DIR}/${CONFIG_SOURCE_DIR}" )
+set( COMPONENTFILES_PATH "${INSTALL_DIR}/${COMPONENTFILES_DIR}" )
+set( PLUGINS_PATH "${INSTALL_DIR}/${PLUGINS_DIR}" )
+set( LIB_PATH "${INSTALL_DIR}/${LIB_DIR}" )
+set( EXEC_PATH "${INSTALL_DIR}/${EXEC_DIR}" )
+set( ADMIN_PATH "${INSTALL_DIR}/${ADMIN_DIR}" )

cmake_modules/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>cmake_modules</title>
+
+    <para>
+        The cmake_modules directory contains the sources for the cmake_modules library.
+    </para>
+</section>

common/commonext/CMakeLists.txt

@@ -0,0 +1,51 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: commonext 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for commonext
+#####################################################
+
+project( commonext ) 
+
+set (    SRCS 
+         thorport.cpp 
+         commonext.cpp 
+         
+         commonext.hpp 
+    )
+
+include_directories ( 
+         ./../../rtl/include 
+         ./../../system/include 
+         ./../workunit 
+         ./../../system/jlib 
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DCOMMONEXT_EXPORTS )
+
+HPCC_ADD_LIBRARY( commonext SHARED ${SRCS} )
+add_dependencies (commonext espscm )
+install ( TARGETS commonext DESTINATION ${OSSDIR}/lib )
+target_link_libraries ( commonext 
+         jlib 
+    )
+
+

common/commonext/commonext.cpp

@@ -0,0 +1,224 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "platform.h"
+#define CHEAP_UCHAR_DEF
+#ifdef _WIN32
+typedef wchar_t UChar;
+#else 
+typedef unsigned short UChar;
+#endif 
+#include "eclhelper.hpp"
+
+#include "jmisc.hpp"
+
+#include "commonext.hpp"
+
+static const char **kindArray;
+
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    kindArray = (const char **)malloc(TAKlast * sizeof(const char *));
+    memset(kindArray, 0, TAKlast * sizeof(const char *));
+
+    kindArray[TAKnone] = "none";
+    kindArray[TAKdiskwrite] = "diskwrite" ;
+    kindArray[TAKsort] = "sort" ;
+    kindArray[TAKdedup] = "dedup" ;
+    kindArray[TAKfilter] = "filter" ;
+    kindArray[TAKsplit] = "split" ;
+    kindArray[TAKproject] = "project" ;
+    kindArray[TAKrollup] = "rollup";
+    kindArray[TAKiterate] = "iterate";
+    kindArray[TAKaggregate] = "aggregate";
+    kindArray[TAKhashaggregate] = "hashaggregate";
+    kindArray[TAKfirstn] = "firstn";
+    kindArray[TAKsample] = "sample";
+    kindArray[TAKdegroup] = "degroup";
+    kindArray[TAKjoin] = "join";
+    kindArray[TAKhashjoin] = "hashjoin";
+    kindArray[TAKlookupjoin] = "lookupjoin";
+    kindArray[TAKselfjoin] = "selfjoin";
+    kindArray[TAKkeyedjoin] = "keyedjoin";
+    kindArray[TAKgroup] = "group";
+    kindArray[TAKworkunitwrite] = "workunitwrite";
+    kindArray[TAKfunnel] = "funnel";
+    kindArray[TAKapply] = "apply";
+    kindArray[TAKtemptable] = "temptable";
+    kindArray[TAKtemprow] = "temprow";
+    kindArray[TAKhashdistribute] = "hashdistribute";
+    kindArray[TAKhashdedup] = "hashdedup";
+    kindArray[TAKnormalize] = "normalize";
+    kindArray[TAKremoteresult] = "remoteresult";
+    kindArray[TAKpull] = "pull";
+    kindArray[TAKdenormalize] = "denormalize";
+    kindArray[TAKnormalizechild] = "normalizechild";
+    kindArray[TAKchilddataset] = "childdataset";
+    kindArray[TAKselectn] = "selectn";
+    kindArray[TAKenth] = "enth";
+    kindArray[TAKif] = "if";
+    kindArray[TAKnull] = "null";
+    kindArray[TAKdistribution] = "distribution";
+    kindArray[TAKcountproject] = "countproject";
+    kindArray[TAKchoosesets] = "choosesets";
+    kindArray[TAKpiperead] = "piperead";
+    kindArray[TAKpipewrite] = "pipewrite";
+    kindArray[TAKcsvwrite] = "csvwrite";
+    kindArray[TAKpipethrough] = "pipethrough";
+    kindArray[TAKindexwrite] = "indexwrite";
+    kindArray[TAKchoosesetsenth] = "choosesetsenth";
+    kindArray[TAKchoosesetslast] = "choosesetslast";
+    kindArray[TAKfetch] = "fetch";
+    kindArray[TAKhashdenormalize] = "hashdenormalize";
+    kindArray[TAKworkunitread] = "workunitread";
+    kindArray[TAKthroughaggregate] = "throughaggregate";
+    kindArray[TAKspill] = "spill";
+    kindArray[TAKcase] = "case";
+    kindArray[TAKlimit] = "limit";
+    kindArray[TAKcsvfetch] = "csvfetch";
+    kindArray[TAKxmlwrite] = "xmlwrite";
+    kindArray[TAKparse] = "parse";
+    kindArray[TAKtopn] = "topn";
+    kindArray[TAKmerge] = "merge";
+    kindArray[TAKxmlfetch] = "xmlfetch";
+    kindArray[TAKxmlparse] = "xmlparse";
+    kindArray[TAKkeyeddistribute] = "keyeddistribute";
+    kindArray[TAKjoinlight] = "joinlight";
+    kindArray[TAKalljoin] = "alljoin";
+    kindArray[TAKsoap_rowdataset] = "SOAP_rowdataset";
+    kindArray[TAKsoap_rowaction] = "SOAP_rowaction";
+    kindArray[TAKsoap_datasetdataset] = "SOAP_datasetdataset";
+    kindArray[TAKsoap_datasetaction] = "SOAP_datasetaction";
+    kindArray[TAKkeydiff] = "keydiff";
+    kindArray[TAKkeypatch] = "keypatch";
+    kindArray[TAKkeyeddenormalize] = "keyeddenormalize";
+    kindArray[TAKchilditerator] = "Child Dataset";
+    kindArray[TAKdatasetresult] = "Dataset Result";
+    kindArray[TAKrowresult] = "Row Result";
+    kindArray[TAKchildif] = "childif";
+    kindArray[TAKpartition] = "partition";
+    kindArray[TAKlocalgraph] = "local graph";
+    kindArray[TAKifaction] = "if action";
+    kindArray[TAKsequential] = "sequential";
+    kindArray[TAKparallel] = "parallel";
+    kindArray[TAKemptyaction] = "emptyaction";
+    kindArray[TAKskiplimit] = "skip_limit";
+    kindArray[TAKdiskread] = "diskread";
+    kindArray[TAKdisknormalize] = "disknormalize";
+    kindArray[TAKdiskaggregate] = "diskaggregate";
+    kindArray[TAKdiskcount] = "diskcount";
+    kindArray[TAKdiskgroupaggregate] = "diskgroupaggregate";
+    kindArray[TAKindexread] = "indexread";
+    kindArray[TAKindexnormalize] = "indexnormalize";
+    kindArray[TAKindexaggregate] = "indexaggregate";
+    kindArray[TAKindexcount] = "indexcount";
+    kindArray[TAKindexgroupaggregate] = "indexgroupaggregate";
+    kindArray[TAKchildnormalize] = "childnormalize";
+    kindArray[TAKchildaggregate] = "childaggregate";
+    kindArray[TAKchildgroupaggregate] = "childgroupaggregate";
+    kindArray[TAKchildthroughnormalize] = "childthroughnormalize";
+    kindArray[TAKcsvread] = "csvread";
+    kindArray[TAKxmlread] = "xmlread";
+    kindArray[TAKlocalresultread] = "localresultread";
+    kindArray[TAKlocalresultwrite] = "localresultwrite";
+    kindArray[TAKcombine] = "combine";
+    kindArray[TAKregroup] = "regroup";
+    kindArray[TAKrollupgroup] = "rollupgroup";
+    kindArray[TAKcombinegroup] = "combinegroup";
+    kindArray[TAKlookupdenormalize] = "lookupdenormalize";
+    kindArray[TAKalldenormalize] = "alldenormalize";
+    kindArray[TAKdenormalizegroup] = "denormalizegroup";
+    kindArray[TAKhashdenormalizegroup] = "hashdenormalizegroup";
+    kindArray[TAKlookupdenormalizegroup] = "lookupdenormalizegroup";
+    kindArray[TAKkeyeddenormalizegroup] = "keyeddenormalizegroup";
+    kindArray[TAKalldenormalizegroup] = "alldenormalizegroup";
+    kindArray[TAKlocalresultspill] = "localresultspill";
+    kindArray[TAKsimpleaction] = "simpleaction";
+    kindArray[TAKloopcount] = "loop";
+    kindArray[TAKlooprow] = "loop";
+    kindArray[TAKloopdataset] = "loop";
+    kindArray[TAKchildcase] = "childcase";
+    kindArray[TAKremotegraph] = "remote";
+    kindArray[TAKlibrarycall] = "librarycall";
+    kindArray[TAKrawiterator] = "Child Dataset";
+    kindArray[TAKlocalstreamread] = "localstreamread";
+    kindArray[TAKprocess] = "process";
+    kindArray[TAKgraphloop] ="graph";
+    kindArray[TAKparallelgraphloop] = "graph";
+    kindArray[TAKgraphloopresultread] = "graphloopread";
+    kindArray[TAKgraphloopresultwrite] = "graphloopwrite";
+    kindArray[TAKgrouped] = "grouped";
+    kindArray[TAKsorted] = "sorted";
+    kindArray[TAKdistributed] = "distributed";
+    kindArray[TAKnwayjoin] = "nwayjoin";
+    kindArray[TAKnwaymerge] = "nwaymerge";
+    kindArray[TAKnwaymergejoin] = "nwaymergejoin";
+    kindArray[TAKnwayinput] = "nwayinput";
+    kindArray[TAKnwaygraphloopresultread] = "nwaygraphloopread";
+    kindArray[TAKnwayselect] = "nwayselect";
+    kindArray[TAKnonempty] = "nonempty";
+    kindArray[TAKcreaterowlimit] = "createrow_limit";
+    kindArray[TAKexistsaggregate] = "existsaggregate";
+    kindArray[TAKcountaggregate] = "countaggregate";
+    kindArray[TAKprefetchproject] = "prefetchproject";
+    kindArray[TAKprefetchcountproject] = "prefetchcountproject";
+    kindArray[TAKfiltergroup] = "filtergroup";
+    kindArray[TAKmemoryspillread] = "memoryspillread";
+    kindArray[TAKmemoryspillwrite] = "memoryspillwrite";
+    kindArray[TAKmemoryspillsplit] = "memoryspillsplit";
+    kindArray[TAKsection] = "section";
+    kindArray[TAKlinkedrawiterator] = "linkedrawiterator";
+    kindArray[TAKnormalizelinkedchild] = "normalizelinkedchild";
+    kindArray[TAKfilterproject] = "filterproject";
+    kindArray[TAKcatch] = "catch";
+    kindArray[TAKskipcatch] = "skip_catch";
+    kindArray[TAKcreaterowcatch] = "createrow_catch";
+    kindArray[TAKsectioninput] = "sectioninput";
+    kindArray[TAKindexgroupexists] = "indexgroupexists";
+    kindArray[TAKindexgroupcount] = "indexgroupcount";
+    kindArray[TAKhashdistributemerge] = "hashdistributemerge";
+    kindArray[TAKselfjoinlight] = "selfjoinlight";
+    kindArray[TAKwhen] = "when";
+    kindArray[TAKhttp_rowdataset] = "http";
+    kindArray[TAKstreamediterator] = "streamediterator";
+    kindArray[TAKexternalsource] = "externalsource";
+    kindArray[TAKexternalsink] = "externalsink";
+    kindArray[TAKexternalprocess] = "externalprocess";
+
+//Non standard
+    kindArray[TAKcountindex] = "countindex";
+    kindArray[TAKcountdisk] = "countdisk";
+    kindArray[TAKsubgraph] = "subgraph";
+
+    return true;
+}
+
+MODULE_EXIT()
+{
+    free(kindArray);
+}
+
+const char *activityKindStr(ThorActivityKind kind) 
+{
+    const char *ret = kind<TAKlast?kindArray[kind]:NULL;
+    if (ret) return ret;
+    static char num[32];
+    itoa(kind,num,10);
+    return num;
+}
+

common/commonext/commonext.hpp

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _COMMONEXT_
+#define _COMMONEXT_
+
+#ifdef _WIN32
+    #ifdef COMMONEXT_EXPORTS
+        #define thcommonext_decl __declspec(dllexport)
+    #else
+        #define thcommonext_decl __declspec(dllimport)
+    #endif
+#else
+    #define thcommonext_decl
+#endif
+
+#include "eclhelper.hpp"
+
+extern thcommonext_decl const char *activityKindStr(ThorActivityKind kind);
+
+#endif
+

common/commonext/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/commonext</title>
+
+    <para>
+        The common/commonext directory contains the sources for the common/commonext library.
+    </para>
+</section>

common/commonext/thorport.cpp

@@ -0,0 +1,143 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+// Thor Port Allocation
+
+#include "platform.h"
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <stdio.h> 
+#include <stdlib.h> 
+
+#include "jlib.hpp"
+#include "jdebug.hpp"
+#include "jexcept.hpp"
+#include "jmutex.hpp"
+#include "jset.hpp"
+#include "jmisc.hpp"
+
+#include "portlist.h"
+#include "thorport.hpp"
+
+static CBuildVersion _bv("$HeadURL: https://svn.br.seisint.com/ecl/trunk/thorlcr/shared/thorport.cpp $ $Id: thorport.cpp 62376 2011-02-04 21:59:58Z sort $");
+
+#define WATCHDOGINC        1
+
+static CriticalSection *portallocsection;
+static IBitSet *portmap;
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    portallocsection = new CriticalSection;
+    portmap = createBitSet();
+    portmap->set(THOR_MP_INC, true);
+    portmap->set(WATCHDOGINC, true);
+    return true;
+}
+MODULE_EXIT()
+{
+    portmap->Release();
+    delete portallocsection;
+}
+
+static unsigned short masterportbase=0;
+static unsigned short machineportbase=0;
+
+
+unsigned short getFixedPort(ThorPortKind category)
+{
+    return getFixedPort(machineportbase, category);
+}
+
+unsigned short getFixedPort(unsigned short machineBase, ThorPortKind category)
+{
+    return getExternalFixedPort(masterportbase, machineBase, category);
+}
+
+unsigned short getExternalFixedPort(unsigned short masterBase, unsigned short machineBase, ThorPortKind category)
+{
+    if (!masterBase) masterBase = THOR_BASE_PORT;
+    if (!machineBase) machineBase = THOR_BASESLAVE_PORT;
+    switch (category) {
+    case TPORT_watchdog:
+        return machineBase+WATCHDOGINC;
+    case TPORT_mp:
+        return machineBase+THOR_MP_INC; 
+    }
+    LOG(MCerror,unknownJob,"getFixedPort: Unknown Port Kind!");
+    return 0;
+}
+
+unsigned short allocPort(unsigned num)
+{
+    CriticalBlock proc(*portallocsection);
+    if (num==0)
+        num = 1;
+    unsigned sp=0;
+    unsigned p;
+    loop {
+        p = portmap->scan(sp,false);
+        unsigned q;
+        for (q=p+1;q<p+num;q++) {
+            if (portmap->test(q))
+                break;
+        }
+        if (q==p+num) {
+            while (q!=p)
+                portmap->set(--q);
+            break;
+        }
+        sp=p+1;
+    }
+
+    return (unsigned short)(p+machineportbase);
+}
+
+void freePort(unsigned short p,unsigned num)
+{
+    CriticalBlock proc(*portallocsection);
+    if (!p)
+        return;
+    if (!portmap) 
+        return;
+    if (num==0)
+        num = 1;
+    while (num--) 
+        portmap->set(p-machineportbase+num,false);
+}
+        
+void setMachinePortBase(unsigned short base)
+{
+    machineportbase = base?base:THOR_BASESLAVE_PORT;
+}
+
+void setMasterPortBase(unsigned short base)
+{
+    masterportbase = base?base:THOR_BASE_PORT;
+}
+
+unsigned short getMasterPortBase()
+{
+    return masterportbase?masterportbase:THOR_BASE_PORT;
+}
+
+unsigned short getMachinePortBase()
+{
+    return machineportbase?machineportbase:THOR_BASESLAVE_PORT;
+}

common/commonext/thorport.hpp

@@ -0,0 +1,82 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef __THORPORT__
+#define __THORPORT__
+
+#include "jarray.hpp"
+#include "jutil.hpp"
+#include "jsocket.hpp"
+
+#ifdef _WIN32
+    #ifdef COMMONEXT_EXPORTS
+        #define thcommonext_decl __declspec(dllexport)
+    #else
+        #define thcommonext_decl __declspec(dllimport)
+    #endif
+#else
+    #define thcommonext_decl
+#endif
+
+enum ThorPortKind
+{
+    TPORT_watchdog,
+    TPORT_mp
+};
+
+thcommonext_decl unsigned short getFixedPort(ThorPortKind category);
+thcommonext_decl unsigned short getFixedPort(unsigned short base, ThorPortKind category);
+thcommonext_decl unsigned short getExternalFixedPort(unsigned short masterbase, unsigned short machinebase, ThorPortKind category);
+thcommonext_decl unsigned short allocPort(unsigned num=1);
+thcommonext_decl void           freePort(unsigned short,unsigned num=1);
+thcommonext_decl void           setMachinePortBase(unsigned short base);
+thcommonext_decl void           setMasterPortBase(unsigned short base);
+thcommonext_decl unsigned short         getMasterPortBase();
+thcommonext_decl unsigned short         getMachinePortBase();
+
+typedef UnsignedShortArray PortArray;
+
+class CPortGroup
+{
+public:
+    unsigned short allocPort(unsigned n=1)
+    {
+        unsigned short p=::allocPort(n);
+        while (n--)
+            portsinuse.append(p+n);
+        return p;
+    }
+    void freePort(unsigned short p,unsigned n=1)
+    {
+        unsigned i;
+        for (i=0;i<n;i++)
+            portsinuse.zap(p+i);
+        ::freePort(p,n);
+    }
+    virtual ~CPortGroup()
+    {
+        ForEachItemIn(i,portsinuse) {
+            freePort(portsinuse.item(i));
+        }
+    }
+protected:
+    PortArray portsinuse;
+};
+
+
+#endif

common/dataxfer/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/dataxfer</title>
+
+    <para>
+        The common/dataxfer directory contains the sources for the common/dataxfer library.
+    </para>
+</section>

common/deftype/CMakeLists.txt

@@ -0,0 +1,56 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: deftype 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for deftype
+#####################################################
+
+project( deftype ) 
+
+set (    SRCS 
+         deffield.cpp 
+         deftype.cpp 
+         defvalue.cpp 
+         
+         deffield.hpp
+         deftype.hpp
+         defvalue.hpp
+         
+    )
+
+include_directories ( 
+         ./../../common/deftype 
+         ./../../rtl/eclrtl 
+         ./../../rtl/nbcd 
+         ./../../system/include 
+         ./../../system/jlib 
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DDEFTYPE_EXPORTS )
+
+HPCC_ADD_LIBRARY( deftype SHARED ${SRCS} )
+install ( TARGETS deftype DESTINATION ${OSSDIR}/lib )
+target_link_libraries ( deftype 
+         jlib 
+         nbcd
+         eclrtl 
+    )
+

common/deftype/deffield.cpp

@@ -0,0 +1,311 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jlib.hpp"
+#include "jstream.hpp"
+#include "jmisc.hpp"
+#include "defvalue.hpp"
+#include "jexcept.hpp"
+#include "jlzw.hpp"
+
+#include "deffield.ipp"
+
+//------------------------------------------------------------------------------------------------
+
+CDefRecordElement::CDefRecordElement(DefElemKind _kind, _ATOM _name, ITypeInfo * _type, size32_t _maxSize)
+{
+    kind = _kind;
+    name = _name;
+    type.set(_type);
+    maxSize = _maxSize;
+    closed = false;
+}
+
+bool CDefRecordElement::operator==(IDefRecordElement const & other) const
+{
+    if((kind != other.getKind()) || (name != other.queryName()) || (maxSize != other.getMaxSize()) || (queryCompareValue() != other.queryCompareValue()) || (children.ordinality() != other.numChildren()))
+            return false;
+    if(type)
+    {
+        if((!other.queryType()) || !isSameBasicType(type, other.queryType()))
+            return false;
+    }
+    else
+    {
+        if(other.queryType())
+            return false;
+    }
+    ForEachItemIn(idx, children)
+        if(!(children.item(idx) == *other.queryChild(idx)))
+            return false;
+    return true;
+}
+
+void CDefRecordElement::appendChild(IDefRecordElement * elem)
+{
+    assertex(!closed);
+    CDefRecordElement * cast = dynamic_cast<CDefRecordElement *>(elem);
+    children.append(*LINK(cast));
+}
+
+IDefRecordElement * CDefRecordElement::close()
+{
+    assertex(!closed);
+    closed = true;
+    return this;
+}
+
+//------------------------------------------------------------------------------------------------
+
+CDefRecordBuilder::CDefRecordBuilder(unsigned maxSize)
+{
+    Owned<ITypeInfo> type = makeRecordType();
+    elem.setown(new CDefRecordElement(DEKrecord, NULL, type, maxSize));
+}
+
+void CDefRecordBuilder::addChild(IDefRecordElement * element)
+{
+    elem->appendChild(element);
+}
+
+void CDefRecordBuilder::addChildOwn(IDefRecordElement * element)
+{
+    elem->appendChild(element);
+    element->Release();
+}
+
+IDefRecordElement * CDefRecordBuilder::close()
+{
+    return elem.getClear()->close();
+}
+
+//------------------------------------------------------------------------------------------------
+
+CDefRecordMeta::CDefRecordMeta(IDefRecordElement * _record, unsigned _numKeyed) : record(_record), numKeyed(_numKeyed)
+{
+}
+
+//------------------------------------------------------------------------------------------------
+
+IDefRecordElement * createDEfield(_ATOM name, ITypeInfo * type, IDefRecordElement * record, size32_t maxSize)
+{
+    CDefRecordElement * elem = new CDefRecordElement(DEKfield, name, type, maxSize);
+    if (record)
+        elem->appendChild(record);
+    return elem->close();
+}
+
+IDefRecordBuilder * createDErecord(size32_t maxSize)
+{
+    return new CDefRecordBuilder(maxSize);
+}
+
+IDefRecordElement * createDEifblock(IDefRecordElement * field, IValue * value, IDefRecordElement * record)
+{
+    CDefRecordElement * elem = new CDefIfBlock(value);
+    elem->appendChild(field);
+    elem->appendChild(record);
+    return elem->close();
+}
+
+IDefRecordMeta * createDefRecordMeta(IDefRecordElement * record, unsigned numKeyed)
+{
+    return new CDefRecordMeta(record, numKeyed);
+}
+
+//------------------------------------------------------------------------------------------------
+
+static void serializeElement(MemoryBuffer & target, IDefRecordElement * elem)
+{
+    byte kind = elem ? elem->getKind() : DEKnone;
+    target.append(kind);
+    switch (kind)
+    {
+    case DEKnone:
+        break;
+    case DEKrecord:
+        {
+            size32_t maxSize = elem->getMaxSize();
+            unsigned numChildren = elem->numChildren();
+            target.append(maxSize).append(numChildren);
+            for (unsigned i=0; i < numChildren; i++)
+                serializeElement(target, elem->queryChild(i));
+            break;
+        }
+    case DEKifblock:
+        {
+            IValue * value = elem->queryCompareValue();
+            serializeValue(target, value);
+            serializeElement(target, elem->queryChild(0));
+            serializeElement(target, elem->queryChild(1));
+            break;
+        }
+    case DEKfield:
+        {
+            _ATOM name = elem->queryName();
+            ITypeInfo * type = elem->queryType();
+            size32_t maxSize = elem->getMaxSize();
+            serializeAtom(target, name);
+            type->serialize(target);
+            serializeElement(target, elem->queryChild(0));
+            target.append(maxSize);
+            break;
+        }
+    default:
+        throwUnexpected();
+    }
+}
+
+static void doSerializeRecordMeta(MemoryBuffer & target, IDefRecordMeta * meta)
+{
+    serializeElement(target, meta->queryRecord());
+    target.append(meta->numKeyedFields());
+}
+
+extern DEFTYPE_API void serializeRecordMeta(MemoryBuffer & target, IDefRecordMeta * meta, bool compress)
+{
+    if (compress)
+    {
+        MemoryBuffer temp;
+        doSerializeRecordMeta(temp, meta);
+        compressToBuffer(target, temp.length(), temp.toByteArray());
+    }
+    else
+        doSerializeRecordMeta(target, meta);
+}
+
+static IDefRecordElement * deserializeElement(MemoryBuffer & source)
+{
+    byte kind;
+    source.read(kind);
+    switch (kind)
+    {
+    case DEKnone:
+        return NULL;
+    case DEKrecord:
+        {
+            size32_t maxSize;
+            unsigned numChildren;
+            source.read(maxSize).read(numChildren);
+            Owned<IDefRecordBuilder> builder = createDErecord(maxSize);
+            for (unsigned i=0; i < numChildren; i++)
+                builder->addChildOwn(deserializeElement(source));
+            return builder->close();
+        }
+    case DEKifblock:
+        {
+            Owned<IValue> value = deserializeValue(source);
+            Owned<IDefRecordElement> field = deserializeElement(source);
+            Owned<IDefRecordElement> record = deserializeElement(source);
+            return createDEifblock(field, value, record);
+        }
+    case DEKfield:
+        {
+            _ATOM name = deserializeAtom(source);
+            Owned<ITypeInfo> type = deserializeType(source);
+            Owned<IDefRecordElement> record = deserializeElement(source);
+            size32_t maxSize;
+            source.read(maxSize);
+            return createDEfield(name, type, record, maxSize);
+        }
+    default:
+        throwUnexpected();
+    }
+}
+
+static IDefRecordMeta * doDeserializeRecordMeta(MemoryBuffer & source)
+{
+    Owned<IDefRecordElement> record = deserializeElement(source);
+    unsigned numKeyed;
+    source.read(numKeyed);
+    return createDefRecordMeta(record, numKeyed);
+}
+
+extern DEFTYPE_API IDefRecordMeta * deserializeRecordMeta(MemoryBuffer & source, bool compress)
+{
+    if (compress)
+    {
+        MemoryBuffer temp;
+        decompressToBuffer(temp, source);
+        return doDeserializeRecordMeta(temp);
+    }
+    else
+        return doDeserializeRecordMeta(source);
+}
+
+extern DEFTYPE_API void serializeRecordMeta(IPropertyTree * target, IDefRecordMeta * meta);
+extern DEFTYPE_API IDefRecordMeta * deserializeRecordMeta(const IPropertyTree * source);
+
+static void addElementToPTree(IPropertyTree * root, IDefRecordElement * elem)
+{
+    byte kind = elem ? elem->getKind() : DEKnone;
+    Owned<IPTree> branch = createPTree(false);
+    StringAttr branchName;
+    switch (kind)
+    {
+    case DEKnone:
+        branchName.set("None");
+        assertex(elem->numChildren() == 0);
+        break;
+    case DEKrecord:
+        {
+            branchName.set("Record");
+            branch->setPropInt("@maxSize", elem->getMaxSize());
+            unsigned numChildren = elem->numChildren();
+            for (unsigned i=0; i < numChildren; i++)
+                addElementToPTree(branch, elem->queryChild(i));
+            break;
+        }
+    case DEKifblock:
+        {
+            branchName.set("IfBlock");
+            StringBuffer value;
+            elem->queryCompareValue()->getStringValue(value);
+            branch->setProp("@compareValue", value.str());
+            assertex(elem->numChildren() == 2);
+            addElementToPTree(branch, elem->queryChild(0));
+            addElementToPTree(branch, elem->queryChild(0));
+            break;
+        }
+    case DEKfield:
+        {
+            branchName.set("Field");
+            branch->setProp("@name", elem->queryName()->str());
+            branch->setPropInt("@maxSize", elem->getMaxSize());
+            StringBuffer type;
+            elem->queryType()->getDescriptiveType(type);
+            branch->setProp("@type", type.str());
+            assertex(elem->numChildren() <= 1);
+            if(elem->numChildren())
+                addElementToPTree(branch, elem->queryChild(0));
+            break;
+        }
+    default:
+        throwUnexpected();
+    }
+    root->addPropTree(branchName.get(), branch.getClear());
+}
+
+extern DEFTYPE_API StringBuffer & getRecordMetaAsString(StringBuffer & out, IDefRecordMeta const * meta)
+{
+    Owned<IPropertyTree> tree = createPTree("RecordMeta", false);
+    tree->setPropInt("@numKeyedFields", meta->numKeyedFields());
+    addElementToPTree(tree, meta->queryRecord());
+    toXML(tree, out);
+    return out;
+}

common/deftype/deffield.hpp

@@ -0,0 +1,75 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _DEFFIELD_HPP
+#define _DEFFIELD_HPP
+
+#include "jptree.hpp"
+#include "deftype.hpp"
+#include "defvalue.hpp"
+
+enum DefElemKind { DEKnone, DEKrecord, DEKifblock, DEKfield, DEKattr };
+
+interface IDefRecordElement : public IInterface
+{
+    virtual DefElemKind getKind() const = 0;
+    virtual size32_t getMaxSize() const = 0;
+    virtual unsigned numChildren() const = 0;
+    virtual ITypeInfo *queryType() const = 0;
+    virtual _ATOM queryName() const = 0;
+    virtual IDefRecordElement * queryChild(unsigned i) const = 0;
+    virtual IValue * queryCompareValue() const = 0;
+    virtual bool operator==(IDefRecordElement const & other) const = 0;
+};
+typedef IArrayOf<IDefRecordElement> IDefRecordElementArray;
+
+//fields of type record/dataset have queryChild(0) as the dataset
+//ifblocks have queryChild(0) as the condition field, queryChild(1) as the child record
+//unnamed records (used for inheritance) are already expanded inline 
+//records are not commoned up - even if the same structure is used with two names.  This ensures each field has a unique IDefRecordElement
+//Whether a field needs biasing is determined by the numKeyedFields() and the number of root fields
+
+interface IDefRecordMeta : public IInterface
+{
+    virtual IDefRecordElement * queryRecord() const = 0;
+    virtual unsigned numKeyedFields() const = 0;
+    virtual bool equals(const IDefRecordMeta *other) const = 0;
+    virtual bool IsShared() const = 0;
+};
+
+//Interfaces used by compiler to generate the meta information.
+interface IDefRecordBuilder : public IInterface
+{
+    virtual void addChild(IDefRecordElement * element) = 0;
+    virtual void addChildOwn(IDefRecordElement * element) = 0;
+    virtual IDefRecordElement * close() = 0;
+};
+
+extern DEFTYPE_API IDefRecordElement * createDEfield(_ATOM name, ITypeInfo * type, IDefRecordElement * record, size32_t maxSize=0);
+extern DEFTYPE_API IDefRecordBuilder * createDErecord(size32_t maxSize);
+extern DEFTYPE_API IDefRecordElement * createDEifblock(IDefRecordElement * field, IValue * value, IDefRecordElement * record);
+extern DEFTYPE_API IDefRecordMeta * createDefRecordMeta(IDefRecordElement * record, unsigned numKeyed);
+
+extern DEFTYPE_API void serializeRecordMeta(MemoryBuffer & target, IDefRecordMeta * meta, bool compress);
+extern DEFTYPE_API IDefRecordMeta * deserializeRecordMeta(MemoryBuffer & source, bool compress);
+extern DEFTYPE_API void serializeRecordMeta(IPropertyTree * target, IDefRecordMeta * meta);
+extern DEFTYPE_API IDefRecordMeta * deserializeRecordMeta(const IPropertyTree * source);
+
+extern DEFTYPE_API StringBuffer & getRecordMetaAsString(StringBuffer & out, IDefRecordMeta const * meta);
+
+#endif

common/deftype/deffield.ipp

@@ -0,0 +1,130 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _DEFFIELD_IPP
+#define _DEFFIELD_IPP
+
+#include "deffield.hpp"
+
+
+class CDefRecordElement : public CInterface, implements IDefRecordElement
+{
+public:
+    CDefRecordElement(DefElemKind _kind, _ATOM _name, ITypeInfo * _type, size32_t _maxSize = 0);
+    IMPLEMENT_IINTERFACE
+
+    virtual DefElemKind getKind() const
+    {
+        return (DefElemKind)kind;
+    }
+    virtual size32_t getMaxSize() const
+    {
+        return maxSize;
+    }
+    virtual ITypeInfo *queryType() const
+    {
+        return type;
+    }
+    virtual _ATOM queryName() const
+    {
+        return name;
+    }
+    virtual IValue * queryCompareValue() const
+    {
+        return NULL;
+    }
+    virtual unsigned numChildren() const
+    {
+        return children.ordinality();
+    }
+    virtual IDefRecordElement * queryChild(unsigned i) const
+    {
+        if (children.isItem(i))
+            return &children.item(i);
+        return NULL;
+    }
+    virtual bool operator==(IDefRecordElement const & other) const;
+
+//internal public interface
+    virtual void appendChild(IDefRecordElement * elem);
+    virtual IDefRecordElement * close();
+
+protected:
+    byte kind;
+    _ATOM name;
+    Owned<ITypeInfo> type;
+    CIArrayOf<CDefRecordElement> children;
+    size32_t maxSize;
+    bool closed;
+};
+
+class CDefIfBlock : public CDefRecordElement
+{
+public:
+    CDefIfBlock(IValue * _value) : CDefRecordElement(DEKifblock, NULL, NULL)
+    {
+        value.set(_value);
+    }
+    
+    virtual IValue * queryCompareValue() const
+    {
+        return value;
+    }
+
+protected:
+    Owned<IValue> value;
+};
+
+
+class CDefRecordMeta : public CInterface, implements IDefRecordMeta
+{
+public:
+    CDefRecordMeta(IDefRecordElement * _record, unsigned _numKeyed);
+    IMPLEMENT_IINTERFACE
+
+    virtual IDefRecordElement * queryRecord() const { return record; }
+    virtual unsigned numKeyedFields() const { return numKeyed; }
+    virtual bool equals(const IDefRecordMeta *other) const
+    {
+        return other && numKeyed==other->numKeyedFields() && *record==*other->queryRecord();
+    }
+    virtual bool IsShared() const { return CInterface::IsShared(); }
+
+protected:
+    Linked<IDefRecordElement> record;
+    unsigned numKeyed;
+};
+
+class CDefRecordBuilder : public CInterface, implements IDefRecordBuilder
+{
+public:
+    CDefRecordBuilder(unsigned maxSize);
+    IMPLEMENT_IINTERFACE
+
+    virtual void addChild(IDefRecordElement * element);
+    virtual void addChildOwn(IDefRecordElement * element);
+    virtual IDefRecordElement * close();
+
+protected:
+    Owned<CDefRecordElement> elem;
+};
+
+
+
+
+#endif

common/deftype/deftype.hpp

@@ -0,0 +1,417 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _DEFTYPE_INCL
+#define _DEFTYPE_INCL
+
+#include "jcomp.hpp"
+
+#ifdef _WIN32
+#ifdef DEFTYPE_EXPORTS
+#define DEFTYPE_API __declspec(dllexport)
+#else
+#define DEFTYPE_API __declspec(dllimport)
+#endif
+#else
+#define DEFTYPE_API
+#endif
+
+#ifdef _DEBUG
+    #define ENABLE_TYPET_ENUM_TYPE
+#endif
+
+#define CHEAP_UCHAR_DEF
+#ifdef _WIN32
+typedef wchar_t UChar;
+#else //__WIN32
+typedef unsigned short UChar;
+#endif //__WIN32
+
+interface ITypeInfo;
+interface IValue;
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+#define type_bigendianint       type_swapint
+#define type_littleendianint    type_int
+#else
+#define type_bigendianint       type_int
+#define type_littleendianint    type_swapint
+#endif
+
+//These values correspond to the maximums for fields in a dataset
+//Temporary values can be larger (see nbcd.h TempDecimal for details)
+#define MAX_DECIMAL_LENGTH      32
+
+// NOTE - do not change the values here - they are also used in Clarion and stored in a database!!
+// Add new types to the end
+
+enum type_vals 
+{
+    type_boolean        = 0, 
+    type_int            = 1,    
+    type_real           = 2, 
+    type_decimal        = 3, 
+    type_string         = 4, 
+type_unused1            = 5, 
+    type_date           = 6, 
+type_unused2            = 7, 
+type_unused3            = 8, 
+    type_bitfield       = 9, 
+type_unused4            = 10, 
+    type_char           = 11,
+    type_enumerated     = 12, 
+    type_record         = 13, 
+    type_varstring      = 14,
+    type_blob           = 15, 
+    type_data           = 16, 
+    type_pointer        = 17, 
+    type_class          = 18, 
+    type_array          = 19,
+    type_table          = 20, 
+    type_set            = 21, 
+    type_row            = 22, 
+    type_groupedtable   = 23,
+    type_void           = 24,
+    type_alien          = 25,
+    type_swapint        = 26,
+    type_none           = 27,
+    type_packedint      = 28,
+type_unused5            = 29,
+    type_qstring        = 30,
+    type_unicode        = 31,
+    type_any            = 32,
+    type_varunicode     = 33,
+    type_pattern        = 34,
+    type_rule           = 35,
+    type_token          = 36,
+    type_feature        = 37,
+    type_event          = 38,
+    type_null           = 39,       // not the same as type_void, which should be reserved for actions.
+    type_scope          = 40,
+    type_utf8           = 41,
+    type_transform      = 42,
+    type_ifblock        = 43,       // not a real type -but used for the rtlfield serialization
+    type_function       = 44,
+    type_sortlist       = 45,
+
+    type_modifier       = 0xff,     // used by getKind()
+    type_unsigned       = 0x100,  // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
+    type_ebcdic         = 0x200,   // combined with some of the above, when returning summary type information. Not returned by getTypeCode()
+
+//Some pseudo types - never actually created
+    type_stringorunicode= 0xfc, // any string/unicode variant
+    type_numeric        = 0xfd,
+    type_scalar         = 0xfe,
+};
+
+enum typemod_t
+{
+    typemod_none        = 0,
+    typemod_const       = 1,
+    typemod_ref         = 2,
+    typemod_wrapper     = 3,
+    typemod_builder     = 4,
+    typemod_original    = 5,
+    typemod_member      = 6,
+    typemod_serialized  = 7,
+    typemod_outofline   = 8,
+    typemod_attr        = 9,
+    typemod_indirect    = 10,       // type definition needs to go via an ecl definition
+    typemod_max
+};
+
+#define INFINITE_LENGTH         0xFFFFFFF0
+#define UNKNOWN_LENGTH          0xFFFFFFF1
+
+#ifdef ENABLE_TYPET_ENUM_TYPE
+    typedef enum type_vals type_t;
+#else
+    typedef unsigned char type_t;
+#endif
+
+//MORE: Something like this should replace caseSensitive for strings...
+interface ICollationInfo;
+interface ICharsetInfo : public serializable
+{
+public:
+    virtual _ATOM queryName() = 0;
+    virtual ICollationInfo * queryDefaultCollation() = 0;
+    virtual unsigned char queryFillChar() = 0;
+    virtual char const * queryCodepageName() = 0;
+};
+
+interface ICollationInfo : public serializable
+{
+public:
+    virtual _ATOM queryName() = 0;
+    virtual ICharsetInfo * getCharset() = 0;
+    virtual int compare(const char * left, const char * right, unsigned len) = 0;
+    virtual const char * getCompareName(bool varLength) = 0;
+};
+
+interface ITranslationInfo : public IInterface
+{
+public:
+    virtual _ATOM queryName() = 0;
+    virtual const char * queryRtlFunction() = 0;
+    virtual const char * queryVarRtlFunction() = 0;
+    virtual ICharsetInfo * querySourceCharset() = 0;
+    virtual ICharsetInfo * queryTargetCharset() = 0;
+    virtual StringBuffer & translate(StringBuffer & tgt, unsigned len, const char * src) = 0;
+};
+
+interface IValue;
+interface ITypeInfo : public serializable
+{
+public:
+    virtual type_t getTypeCode() const = 0;
+    virtual size32_t getSize() = 0;
+    virtual unsigned getAlignment() = 0;
+    virtual unsigned getBitSize() = 0;
+    virtual unsigned getPrecision() = 0;
+    virtual unsigned getStringLen() = 0;
+    virtual unsigned getDigits() = 0;
+    virtual bool assignableFrom(ITypeInfo * source) = 0;
+    virtual IValue * castFrom(bool isSignedValue, __int64 value) = 0;
+    virtual IValue * castFrom(double value) = 0;
+    virtual IValue * castFrom(size32_t len, const char * text) = 0;
+    virtual IValue * castFrom(size32_t len, const UChar * text) = 0;
+    virtual StringBuffer &getECLType(StringBuffer & out) = 0;
+    virtual StringBuffer &getDescriptiveType(StringBuffer & out) = 0;
+    virtual const char *queryTypeName() = 0;
+    virtual unsigned getCardinality() = 0;
+    virtual bool isInteger() = 0;
+    virtual bool isReference() = 0;
+    virtual bool isScalar() = 0;
+    virtual bool isSigned() = 0;
+    virtual bool isSwappedEndian() = 0;
+    virtual ITypeInfo * queryChildType() = 0;
+    virtual ICharsetInfo * queryCharset() = 0;
+    virtual ICollationInfo * queryCollation() = 0;
+    virtual _ATOM queryLocale() = 0;
+    virtual IInterface * queryDistributeInfo() = 0;
+    virtual IInterface * queryGroupInfo() = 0;
+    virtual IInterface * queryGlobalSortInfo() = 0;
+    virtual IInterface * queryLocalUngroupedSortInfo() = 0;
+    virtual IInterface * queryGroupSortInfo() = 0;
+    virtual ITypeInfo * queryPromotedType() = 0;
+    virtual ITypeInfo * queryTypeBase() = 0;
+    virtual unsigned getCrc() = 0;      // must be run independant.
+    virtual typemod_t queryModifier() = 0;
+    virtual IInterface * queryModifierExtra() = 0;
+
+    inline bool isBoolean() const { return getTypeCode() == type_boolean; }
+
+private:
+    inline IValue * castFrom(__int64 value) { return NULL; }
+};
+
+interface IFunctionTypeExtra : public IInterface
+{
+    virtual IInterface * queryParameters() = 0;
+    virtual IInterface * queryDefaults() = 0;
+};
+
+interface IEnumeratedTypeBuilder : public IInterface
+{
+public:
+    virtual ITypeInfo *getTypeInfo() = 0;
+    virtual int addValue(IValue *val, size32_t frequency) = 0;
+};
+
+extern DEFTYPE_API ITypeInfo *makeStringType(unsigned size, ICharsetInfo * _charset = NULL, ICollationInfo * _collation = NULL);
+extern DEFTYPE_API ITypeInfo *makeVarStringType(unsigned size, ICharsetInfo * _charset = NULL, ICollationInfo * _collation = NULL);    //NB: size is numchars+1
+extern DEFTYPE_API ITypeInfo *makeQStringType(int len);
+extern DEFTYPE_API ITypeInfo *makeUnicodeType(unsigned len, _ATOM locale); // takes length in UChars, i.e. bytes/2 if known; locale is like fr_BE_EURO, or 0 for default
+extern DEFTYPE_API ITypeInfo *makeVarUnicodeType(unsigned len, _ATOM locale); // takes length in UChars + 1, i.e. bytes/2 + 1 if known; locale is like fr_BE_EURO, or 0 for default
+extern DEFTYPE_API ITypeInfo *makeUtf8Type(unsigned len, _ATOM locale);       // takes length in UChars, i.e. bytes/4 if known; locale is like fr_BE_EURO, or 0 for default
+extern DEFTYPE_API ITypeInfo *makeCharType(bool caseSensitive = false);
+extern DEFTYPE_API ITypeInfo *makeIntType(int size, bool isSigned);
+extern DEFTYPE_API ITypeInfo *makeSwapIntType(int size, bool isSigned);
+extern DEFTYPE_API ITypeInfo *makePackedIntType(ITypeInfo * basetype);
+extern DEFTYPE_API ITypeInfo *makePackedIntType(int size, bool isSigned);
+extern DEFTYPE_API ITypeInfo *makeRealType(int size);
+extern DEFTYPE_API ITypeInfo *makeDataType(int size);
+extern DEFTYPE_API ITypeInfo *makeBitfieldType(int sizeInBits, ITypeInfo * basetype = NULL);
+extern DEFTYPE_API ITypeInfo *makeBoolType();
+extern DEFTYPE_API ITypeInfo *makeBlobType();
+extern DEFTYPE_API ITypeInfo *makeRecordType();     // not used by any IHqlExpressions
+extern DEFTYPE_API ITypeInfo *makeVoidType();
+extern DEFTYPE_API ITypeInfo *makeNullType();
+extern DEFTYPE_API ITypeInfo *makeEventType();
+extern DEFTYPE_API ITypeInfo *makeAnyType();
+extern DEFTYPE_API ITypeInfo *makeType(type_t type, int size);
+extern DEFTYPE_API IEnumeratedTypeBuilder *makeEnumeratedTypeBuilder(ITypeInfo *base, aindex_t numvalues);
+extern DEFTYPE_API ITypeInfo *makeDecimalType(int size, unsigned char prec, bool isSigned);
+extern DEFTYPE_API ITypeInfo *makeTableType(ITypeInfo *basetype, IInterface * distributeinfo, IInterface *gloalSortinfo, IInterface * localSortInfo);
+extern DEFTYPE_API ITypeInfo *makeGroupedTableType(ITypeInfo *basetype, IInterface *groupinfo, IInterface *sortinfo);
+extern DEFTYPE_API ITypeInfo *makeRowType(ITypeInfo *basetype);
+extern DEFTYPE_API ITypeInfo *makeSetType(ITypeInfo *basetype);
+extern DEFTYPE_API ITypeInfo *makeTransformType(ITypeInfo *basetype);
+extern DEFTYPE_API ITypeInfo *makeSortListType(ITypeInfo *basetype);
+extern DEFTYPE_API ITypeInfo *makeFunctionType(ITypeInfo *basetype, IInterface * args, IInterface * defaults);
+
+extern DEFTYPE_API ITypeInfo * makePointerType(ITypeInfo * basetype);
+extern DEFTYPE_API ITypeInfo * makeArrayType(ITypeInfo * basetype, unsigned size=0);
+extern DEFTYPE_API ITypeInfo * makeClassType(const char * className);
+extern DEFTYPE_API ITypeInfo * makeConstantModifier(ITypeInfo * basetype);
+extern DEFTYPE_API ITypeInfo * makeReferenceModifier(ITypeInfo * basetype);
+extern DEFTYPE_API ITypeInfo * makeWrapperModifier(ITypeInfo * basetype);
+extern DEFTYPE_API ITypeInfo * makeModifier(ITypeInfo * basetype, typemod_t modifier, IInterface * extra=NULL);
+extern DEFTYPE_API ITypeInfo * makePatternType();
+extern DEFTYPE_API ITypeInfo * makeRuleType(ITypeInfo * attrType);
+extern DEFTYPE_API ITypeInfo * makeTokenType();
+extern DEFTYPE_API ITypeInfo * makeFeatureType();
+
+inline ITypeInfo * makeOutOfLineModifier(ITypeInfo * basetype) { return makeModifier(basetype, typemod_outofline); }
+inline ITypeInfo * makeOriginalModifier(ITypeInfo * basetype, IInterface * extra) { return makeModifier(basetype, typemod_original, extra); }
+inline ITypeInfo * makeAttributeModifier(ITypeInfo * basetype, IInterface * extra) { return makeModifier(basetype, typemod_attr, extra); }
+
+extern DEFTYPE_API MemoryBuffer & appendBufferFromMem(MemoryBuffer & mem, ITypeInfo * type, const void * data);
+
+extern DEFTYPE_API void ClearTypeCache();
+
+extern DEFTYPE_API ICharsetInfo * getCharset(_ATOM charset);
+extern DEFTYPE_API ICollationInfo * getCollation(_ATOM collation);
+extern DEFTYPE_API ITranslationInfo * getDefaultTranslation(ICharsetInfo * tgt, ICharsetInfo * src);
+extern DEFTYPE_API ITranslationInfo * queryDefaultTranslation(ICharsetInfo * tgt, ICharsetInfo * src);
+
+//---------------------------------------------------------------------------
+
+extern DEFTYPE_API ITypeInfo * getStretchedType(unsigned newLen, ITypeInfo * type);
+extern DEFTYPE_API ITypeInfo * getNumericType(ITypeInfo * type);
+extern DEFTYPE_API ITypeInfo * getStringType(ITypeInfo * type);
+extern DEFTYPE_API ITypeInfo * getVarStringType(ITypeInfo * type);
+extern DEFTYPE_API ITypeInfo * getPromotedType(ITypeInfo * l_type, ITypeInfo * r_type);
+extern DEFTYPE_API ITypeInfo * getPromotedNumericType(ITypeInfo * l_type, ITypeInfo * r_type);
+extern DEFTYPE_API unsigned getClarionResultType(ITypeInfo *type);
+extern DEFTYPE_API ITypeInfo * getAsciiType(ITypeInfo * type);
+extern DEFTYPE_API ITypeInfo * getBandType(ITypeInfo * type1, ITypeInfo * type2);
+extern DEFTYPE_API ITypeInfo * getBorType(ITypeInfo * type1, ITypeInfo * type2);
+extern DEFTYPE_API bool haveCommonLocale(ITypeInfo * type1, ITypeInfo * type2);
+extern DEFTYPE_API _ATOM getCommonLocale(ITypeInfo * type1, ITypeInfo * type2);
+extern DEFTYPE_API ITypeInfo * getPromotedCompareType(ITypeInfo * left, ITypeInfo * right);
+
+extern DEFTYPE_API bool isNumericType(ITypeInfo * type);
+extern DEFTYPE_API bool isStringType(ITypeInfo * type);
+extern DEFTYPE_API bool isSimpleStringType(ITypeInfo * type);
+extern DEFTYPE_API bool isIntegralType(ITypeInfo * type);
+extern DEFTYPE_API bool isPatternType(ITypeInfo * type);
+extern DEFTYPE_API bool isUnicodeType(ITypeInfo * type);
+extern DEFTYPE_API bool isLittleEndian(ITypeInfo * type);
+extern DEFTYPE_API bool isDatasetType(ITypeInfo * type);
+extern DEFTYPE_API bool isSingleValuedType(ITypeInfo * type);
+inline bool isFixedSize(ITypeInfo * type) { return type && (type->getSize() != UNKNOWN_LENGTH); }
+inline bool isAnyType(ITypeInfo * type) { return type && (type->getTypeCode() == type_any); }
+
+
+//If casting a value from type before to type after is the value preserved.
+//If the value is not preserved then it means more than one source value can match a target value.
+extern DEFTYPE_API bool preservesValue(ITypeInfo * after, ITypeInfo * before);
+extern DEFTYPE_API bool preservesOrder(ITypeInfo * after, ITypeInfo * before);
+// not quite the same as !preservesValue() since cast between integers of the same size are ok
+// if it loses information then multiple before can may to the same after
+extern DEFTYPE_API bool castLosesInformation(ITypeInfo * after, ITypeInfo * before);
+
+extern DEFTYPE_API ICharsetInfo * deserializeCharsetInfo(MemoryBuffer &src);
+extern DEFTYPE_API ICollationInfo * deserializeCollationInfo(MemoryBuffer &src);
+extern DEFTYPE_API ITypeInfo * deserializeType(MemoryBuffer &src);
+extern DEFTYPE_API void serializeType(MemoryBuffer &src, ITypeInfo * type);
+
+extern DEFTYPE_API bool getNormalizedLocaleName(unsigned len, char const * str, StringBuffer & buff);
+extern DEFTYPE_API bool isSameBasicType(ITypeInfo * left, ITypeInfo * right);
+extern DEFTYPE_API ITypeInfo * queryRecordType(ITypeInfo * t);
+extern DEFTYPE_API ITypeInfo * queryRowType(ITypeInfo * t);
+extern DEFTYPE_API ITypeInfo * queryUnqualifiedType(ITypeInfo * t);
+extern DEFTYPE_API ITypeInfo * getFullyUnqualifiedType(ITypeInfo * t);
+extern DEFTYPE_API ITypeInfo * removeModifier(ITypeInfo * t, typemod_t modifier);               // don't link inputs
+extern DEFTYPE_API ITypeInfo * cloneModifier(ITypeInfo * donorType, ITypeInfo * srcType);       // don't link inputs
+extern DEFTYPE_API ITypeInfo * cloneModifiers(ITypeInfo * donorType, ITypeInfo * srcType);      // don't link inputs
+extern DEFTYPE_API bool hasModifier(ITypeInfo * t, typemod_t modifier);
+extern DEFTYPE_API ITypeInfo * queryModifier(ITypeInfo * t, typemod_t modifier);
+extern DEFTYPE_API ITypeInfo * replaceChildType(ITypeInfo * type, ITypeInfo * newChild);
+extern DEFTYPE_API ITypeInfo * getRoundType(ITypeInfo * type);
+extern DEFTYPE_API ITypeInfo * getRoundToType(ITypeInfo * type);
+
+inline bool hasConstModifier(ITypeInfo * t)      { return hasModifier(t, typemod_const); }
+inline bool hasReferenceModifier(ITypeInfo * t)  { return hasModifier(t, typemod_ref); }
+inline bool hasWrapperModifier(ITypeInfo * t)    { return hasModifier(t, typemod_wrapper); }
+inline bool hasOutOfLineModifier(ITypeInfo * t)  { return hasModifier(t, typemod_outofline); }
+inline bool sameUnqualifiedType(ITypeInfo * t1, ITypeInfo * t2)  { return queryUnqualifiedType(t1) == queryUnqualifiedType(t2); }
+inline ITypeInfo * stripFunctionType(ITypeInfo * type)
+{
+    if (type->getTypeCode() == type_function)
+        return type->queryChildType();
+    return type;
+}
+ 
+typedef Linked<ITypeInfo> TypeInfoAttr;
+typedef Owned<ITypeInfo> OwnedITypeInfo;
+typedef IArrayOf<ITypeInfo> TypeInfoArray;
+
+interface ISchemaBuilder
+{
+public:
+    virtual void addField(const char * name, ITypeInfo & type) = 0;
+    virtual void addSetField(const char * name, const char * itemname, ITypeInfo & type) = 0;
+    virtual void beginIfBlock() = 0;
+    virtual bool beginDataset(const char * name, const char * childname) = 0;
+    virtual void beginRecord(const char * name) = 0;
+    virtual void endIfBlock() = 0;
+    virtual void endDataset(const char * name, const char * childname) = 0;
+    virtual void endRecord(const char * name) = 0;
+    virtual bool addSingleFieldDataset(const char * name, const char * childname, ITypeInfo & type) = 0;        // return true if supported
+};
+
+class DEFTYPE_API XmlSchemaBuilder : public ISchemaBuilder
+{
+public:
+    XmlSchemaBuilder(bool _addHeader) { optionalNesting = 0; addHeader = _addHeader; }
+
+    virtual void addField(const char * name, ITypeInfo & type);
+    virtual void addSetField(const char * name, const char * itemname, ITypeInfo & type);
+    virtual void beginIfBlock()                                     { optionalNesting++; }
+    virtual bool beginDataset(const char * name, const char * childname);
+    virtual void beginRecord(const char * name);
+    virtual void endIfBlock()                                       { optionalNesting--; }
+    virtual void endDataset(const char * name, const char * childname);
+    virtual void endRecord(const char * name);
+    virtual bool addSingleFieldDataset(const char * name, const char * childname, ITypeInfo & type);
+
+    void getXml(StringBuffer & results);
+    void getXml(IStringVal & results);
+
+private:
+    void addSchemaPrefix();
+    void addSchemaSuffix();
+    void clear();
+    void getXmlTypeName(StringBuffer & xmlType, ITypeInfo & type);
+
+protected:
+    StringBuffer xml;
+    StringBuffer typesXml;
+    IntArray dataSizes;
+    IntArray stringSizes;
+    IntArray decimalSizes;
+    UnsignedArray nesting;
+    CopyArray setTypes;
+    unsigned optionalNesting;
+    bool addHeader;
+};
+
+#endif

common/deftype/defvalue.hpp

@@ -0,0 +1,155 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _DEFVALUE_INCL
+#define _DEFVALUE_INCL
+
+#include "deftype.hpp"
+
+#ifdef _WIN32
+#ifdef DEFTYPE_EXPORTS
+#define DEFTYPE_API __declspec(dllexport)
+#else
+#define DEFTYPE_API __declspec(dllimport)
+#endif
+#else
+#define DEFTYPE_API
+#endif
+
+interface IValue : public serializable
+{
+public:
+    virtual ITypeInfo *getType() = 0;
+    virtual ITypeInfo *queryType() const = 0;
+    virtual type_t getTypeCode() = 0;
+    virtual size32_t getSize() = 0;
+
+    virtual IValue * castTo(ITypeInfo * type) = 0;
+    virtual const char *generateCPP(StringBuffer & out, CompilerType compiler = DEFAULT_COMPILER) = 0;
+    virtual const char *generateECL(StringBuffer & out) = 0;
+    virtual int compare(IValue * other) = 0;
+    virtual int compare(const void *mem) = 0;
+    virtual int rangeCompare(ITypeInfo * targetType) = 0;                       // can the value be represented in the type -1-too small,0-yes,+1-too big
+
+    virtual const void *queryValue() const = 0;
+    virtual bool getBoolValue() = 0;    
+    virtual __int64 getIntValue() = 0;  
+    virtual double getRealValue() = 0;
+    virtual const char *getStringValue(StringBuffer & out) = 0; 
+    virtual void * getUCharStringValue(unsigned len, void * out) = 0; //copies into buffer of outlen UChars at out; null-terminates if there is room
+    virtual const char *getUTF8Value(StringBuffer & out) = 0;
+    virtual const char *getCodepageValue(StringBuffer & out, char const * codepage) = 0; // codepages: see system/icu/include/codepages.txt
+    virtual void pushDecimalValue() = 0;
+    virtual void toMem(void * ptr) = 0;
+
+    virtual unsigned getHash(unsigned initval)=0;
+};
+
+typedef Owned<IValue> OwnedIValue;
+
+extern DEFTYPE_API void serializeValue(MemoryBuffer & target, IValue * value);
+extern DEFTYPE_API IValue * deserializeValue(MemoryBuffer & source);
+
+extern DEFTYPE_API void appendValueToBuffer(MemoryBuffer & mem, IValue * value);
+
+extern DEFTYPE_API IValue * createValueFromMem(ITypeInfo * type, const void * ptr);
+
+extern DEFTYPE_API IValue * createStringValue(const char * value, unsigned size);
+extern DEFTYPE_API IValue * createStringValue(const char * value, ITypeInfo *type);
+extern DEFTYPE_API IValue * createStringValue(const char *val, ITypeInfo *type, int srcLength, ICharsetInfo *srcCharset);
+extern DEFTYPE_API IValue * createUnicodeValue(char const * value, unsigned size, char const * locale, bool utf8, bool unescape = false); // size is length of ascii or utf8 string; locale is like fr_BE_EURO
+extern DEFTYPE_API IValue * createUnicodeValue(char const * value, ITypeInfo *type); // only for ascii string, use above for utf8
+extern DEFTYPE_API IValue * createUnicodeValue(char const * value, ITypeInfo *type, unsigned srclen); // only for ascii string
+extern DEFTYPE_API IValue * createUnicodeValue(size32_t len, const void * text, ITypeInfo * type);
+extern DEFTYPE_API IValue * createVarUnicodeValue(char const * value, unsigned size, char const * locale, bool utf8, bool unescape = false); // size is length of ascii or utf8 string; locale is like fr_BE_EURO
+extern DEFTYPE_API IValue * createVarUnicodeValue(size32_t len, const void * text, ITypeInfo * type);
+extern DEFTYPE_API IValue * createUtf8Value(char const * value, ITypeInfo *type);
+extern DEFTYPE_API IValue * createUtf8Value(unsigned srclen, char const * value, ITypeInfo *type);
+extern DEFTYPE_API IValue * createDataValue(const char * value, unsigned size);
+extern DEFTYPE_API IValue * createDataValue(const char * value, ITypeInfo *type);
+extern DEFTYPE_API IValue * createQStringValue(unsigned len, const char * value, ITypeInfo *type);
+extern DEFTYPE_API IValue * createVarStringValue(unsigned len, const char * value, ITypeInfo *type);
+extern DEFTYPE_API IValue * createVarStringValue(const char *val, ITypeInfo *type, int srcLength, ICharsetInfo * srcCharset);
+extern DEFTYPE_API IValue * createPackedStringValue(const char *val, unsigned size);
+extern DEFTYPE_API IValue * createPackedStringValue(const char *val, ITypeInfo *type);
+extern DEFTYPE_API IValue * createCharValue(char value, bool caseSensitive);
+extern DEFTYPE_API IValue * createCharValue(char value, ITypeInfo *type);
+extern DEFTYPE_API IValue * createIntValue(__int64 value, unsigned size, bool isSigned);
+extern DEFTYPE_API IValue * createIntValue(__int64 value, ITypeInfo * type);
+extern DEFTYPE_API IValue * createTruncIntValue(__int64 value, unsigned size, bool isSigned);
+extern DEFTYPE_API IValue * createTruncIntValue(__int64 value, ITypeInfo * type);
+extern DEFTYPE_API IValue * createPackedIntValue(__int64 value, ITypeInfo * type);
+extern DEFTYPE_API IValue * createRealValue(double value, unsigned size);
+extern DEFTYPE_API IValue * createRealValue(double value, ITypeInfo * type);
+extern DEFTYPE_API IValue * createDecimalValue(void * value, ITypeInfo * type);
+extern DEFTYPE_API IValue * createDecimalValueFromStack(ITypeInfo * type);
+extern DEFTYPE_API IValue * createBoolValue(bool value);
+extern DEFTYPE_API IValue * createEnumValue(int value, ITypeInfo * type);
+extern DEFTYPE_API IValue * createBitfieldValue(__int64 value, ITypeInfo * type);
+
+extern DEFTYPE_API IValue * createMinIntValue(__int64 value);
+extern DEFTYPE_API IValue * createMinUIntValue(unsigned __int64 value);
+
+extern DEFTYPE_API IValue * addValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * subtractValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * multiplyValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * divideValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * modulusValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * powerValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * shiftLeftValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * shiftRightValues(IValue * left, IValue * right);
+
+extern DEFTYPE_API IValue * substringValue(IValue * v, IValue *lower, IValue *higher);
+extern DEFTYPE_API IValue * trimStringValue(IValue * v, char typecode);
+extern DEFTYPE_API IValue * concatValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * negateValue(IValue * v);
+extern DEFTYPE_API IValue * expValue(IValue * v);
+extern DEFTYPE_API IValue * truncateValue(IValue * v);
+extern DEFTYPE_API IValue * lnValue(IValue * v);
+extern DEFTYPE_API IValue * sinValue(IValue * v);
+extern DEFTYPE_API IValue * cosValue(IValue * v);
+extern DEFTYPE_API IValue * tanValue(IValue * v);
+extern DEFTYPE_API IValue * sinhValue(IValue * v);
+extern DEFTYPE_API IValue * coshValue(IValue * v);
+extern DEFTYPE_API IValue * tanhValue(IValue * v);
+extern DEFTYPE_API IValue * asinValue(IValue * v);
+extern DEFTYPE_API IValue * acosValue(IValue * v);
+extern DEFTYPE_API IValue * atanValue(IValue * v);
+extern DEFTYPE_API IValue * atan2Value(IValue * y, IValue* x);
+extern DEFTYPE_API IValue * log10Value(IValue * v);
+extern DEFTYPE_API IValue * sqrtValue(IValue * v);
+extern DEFTYPE_API IValue * absValue(IValue * v);
+extern DEFTYPE_API IValue * roundValue(IValue * v);
+extern DEFTYPE_API IValue * roundToValue(IValue * v, int places);
+extern DEFTYPE_API IValue * roundUpValue(IValue * v);
+extern DEFTYPE_API IValue * binaryAndValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * binaryOrValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * binaryXorValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * logicalAndValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * logicalOrValues(IValue * left, IValue * right);
+
+extern DEFTYPE_API int orderValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * equalValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * notEqualValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * lessValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * lessEqualValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * greaterValues(IValue * left, IValue * right);
+extern DEFTYPE_API IValue * greaterEqualValues(IValue * left, IValue * right);
+extern DEFTYPE_API void getStringFromIValue(unsigned & len, char* & str, IValue* val);
+extern DEFTYPE_API unsigned getMinimumIntegerSize(unsigned __int64 value, bool isSigned);
+#endif

common/deftype/defvalue.ipp

@@ -0,0 +1,469 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef _DEFVALUEI_INCL
+#define _DEFVALUEI_INCL
+
+#include "eclrtl.hpp"
+#include "defvalue.hpp"
+
+class DEFTYPE_API CValue : public CInterface, public IValue
+{
+protected:
+    ITypeInfo *type;
+
+public:
+    IMPLEMENT_IINTERFACE;
+
+    CValue(ITypeInfo *);
+    ~CValue();
+
+//interface  IValue (mostly implemented in derived classes)
+
+    virtual const void *queryValue() const { UNIMPLEMENTED; }
+    virtual int compare(const void * mem);
+    virtual ITypeInfo *getType();
+    virtual ITypeInfo *queryType() const;
+    virtual double getRealValue();  
+    virtual type_t getTypeCode();
+    virtual size32_t getSize();
+    virtual void * getUCharStringValue(unsigned len, void * out) { UNIMPLEMENTED; return out; }
+    virtual const char *getUTF8Value(StringBuffer & out) { return getStringValue(out); }
+    virtual const char *getCodepageValue(StringBuffer & out, char const * codepage) { UNIMPLEMENTED; }
+
+    // serializable
+    virtual void serialize(MemoryBuffer &tgt) { UNIMPLEMENTED; }
+    virtual void deserialize(MemoryBuffer &src) { UNIMPLEMENTED; }
+
+protected:
+    IValue * doCastTo(unsigned osize, const char * text, ITypeInfo *t);     // common code for string casting
+};
+
+class VarStringValue : public CValue
+{
+protected:
+    StringAttr val;
+
+public:
+    VarStringValue(unsigned len, const char *v, ITypeInfo *t);
+
+//interface  IValue
+    virtual const void *queryValue() const;
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void pushDecimalValue();
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer &tgt);
+    virtual void deserialize(MemoryBuffer &src);
+}; 
+
+class MemoryValue : public CValue
+{
+public:
+//interface  IValue
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue() { return 0; };
+    virtual void pushDecimalValue();    
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer &tgt);
+    virtual void deserialize(MemoryBuffer &src);
+
+protected:
+    MemoryValue(ITypeInfo *t);
+    MemoryValue(const void *v, ITypeInfo *t);
+
+protected:
+    MemoryAttr val;
+}; 
+
+class StringValue : public MemoryValue
+{
+public:
+    StringValue(const char *v, ITypeInfo *t);
+
+//interface  IValue
+    virtual const void *queryValue() const;
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();
+    virtual void pushDecimalValue();
+    virtual const char *getStringValue(StringBuffer &); 
+
+}; 
+
+class UnicodeValue : public MemoryValue
+{
+public:
+    UnicodeValue(UChar const * _value, ITypeInfo * _type);
+
+//interface  IValue
+    virtual const void *queryValue() const;
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();
+    virtual void pushDecimalValue();
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void * getUCharStringValue(unsigned len, void * out);
+    virtual const char *getUTF8Value(StringBuffer & out);
+    virtual const char *getCodepageValue(StringBuffer & out, char const * codepage);
+}; 
+
+IValue * createUnicodeValue(UChar const * text, size32_t len, ITypeInfo * t);
+
+class UnicodeAttr
+{
+public:
+    UnicodeAttr() : text(0) {}
+    ~UnicodeAttr() { free(text); }
+    
+    operator UChar const * () const { return text; }
+    UChar const * get(void) const { return text; }
+    size32_t length() const { return text ? rtlUnicodeStrlen(text) : 0; }
+    void set(UChar const * _text, unsigned _len);
+
+private:
+    UChar * text;
+
+private:
+    UnicodeAttr &operator = (const StringAttr & from);
+};
+
+class VarUnicodeValue : public CValue
+{
+protected:
+    UnicodeAttr val;
+
+public:
+    VarUnicodeValue(unsigned len, const UChar * v, ITypeInfo * _type);
+
+//interface  IValue
+    virtual const void *queryValue() const;
+    virtual const char *generateCPP(StringBuffer & out, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * to);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();
+    virtual __int64 getIntValue();
+    virtual const char *getStringValue(StringBuffer & out);
+    virtual void * getUCharStringValue(unsigned len, void * out);
+    virtual const char *getUTF8Value(StringBuffer & out);
+    virtual const char *getCodepageValue(StringBuffer & out, char const * codepage);
+    virtual void pushDecimalValue();
+    virtual void toMem(void * target);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer & tgt);
+    virtual void deserialize(MemoryBuffer & src);
+}; 
+
+IValue * createVarUnicodeValue(UChar const * text, size32_t len, ITypeInfo * t);
+
+class Utf8Value : public MemoryValue
+{
+public:
+    Utf8Value(char const * _value, ITypeInfo * _type);
+
+//interface  IValue
+    virtual size32_t getSize();
+    virtual const void *queryValue() const;
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();
+    virtual void pushDecimalValue();
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void * getUCharStringValue(unsigned len, void * out);
+    virtual const char *getUTF8Value(StringBuffer & out);
+    virtual const char *getCodepageValue(StringBuffer & out, char const * codepage);
+}; 
+
+class DataValue : public MemoryValue
+{
+public:
+    DataValue(const void *v, ITypeInfo *t);
+
+//interface  IValue
+    virtual const void *queryValue() const;
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+
+    virtual bool getBoolValue();    
+    virtual const char *getStringValue(StringBuffer &); 
+protected:
+    void generateHex(StringBuffer &out);
+
+}; 
+
+class QStringValue : public MemoryValue
+{
+public:
+    QStringValue(unsigned len, const void *v, ITypeInfo *t);
+    QStringValue(const char *v, ITypeInfo *_type);
+
+//interface  IValue
+    virtual const void *queryValue() const;
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();
+    virtual void pushDecimalValue();    
+    virtual const char *getStringValue(StringBuffer &); 
+protected:
+    void generateHex(StringBuffer &out);
+
+}; 
+
+class CharValue : public CValue
+{
+private:
+    char val;
+
+public:
+    CharValue(char v, ITypeInfo *t);
+
+//interface  IValue
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue() { return 0; };    
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void pushDecimalValue();
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer &tgt);
+    virtual void deserialize(MemoryBuffer &src);
+
+}; 
+
+class IntValue : public CValue
+{
+protected:
+    unsigned __int64 val;
+
+public:
+    IntValue(unsigned __int64 v, ITypeInfo *_type) : CValue(_type), val(v) {}
+
+// implementing IValue
+    virtual const void *queryValue() const;
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();  
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void pushDecimalValue();
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer &tgt);
+    virtual void deserialize(MemoryBuffer &src);
+
+protected:
+    IValue * castViaString(ITypeInfo * t);
+    byte * getAddressValue();
+}; 
+
+class PackedIntValue : public CValue
+{
+protected:
+    Owned<IValue> value;
+
+public:
+    PackedIntValue(IValue * v, ITypeInfo *_type) : CValue(_type), value(v) {}
+
+// implementing IValue
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler)     { return value->generateCPP(s, compiler); }
+    virtual const char *generateECL(StringBuffer &s)        { return value->generateECL(s); }
+    virtual IValue * castTo(ITypeInfo * type)               { return value->castTo(type); }
+    virtual int compare(IValue * other);
+
+    virtual bool getBoolValue()                             { return value->getBoolValue(); }
+    virtual __int64 getIntValue()                           { return value->getIntValue(); }
+    virtual const char *getStringValue(StringBuffer & s)    { return value->getStringValue(s); }
+    virtual void pushDecimalValue()                         { value->pushDecimalValue(); }
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval)              { return value->getHash(initval); }
+    virtual int rangeCompare(ITypeInfo * targetType)        { return value->rangeCompare(targetType); }
+// serializable
+    virtual void serialize(MemoryBuffer &tgt)               { value->serialize(tgt); }
+    virtual void deserialize(MemoryBuffer &src)             { value->deserialize(src); }
+}; 
+
+class BitfieldValue : public IntValue
+{
+public:
+    BitfieldValue(unsigned __int64 v, ITypeInfo *_type) : IntValue(v, _type) {}
+
+};
+
+
+class RealValue : public CValue
+{
+protected:
+    double val;
+
+public:
+    RealValue(double v, ITypeInfo *_type) : CValue(_type), val(v) {}
+    RealValue(double v, int size = sizeof(double)) : CValue(makeRealType(size)), val(v) {}
+
+// implementing IValue
+
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();  
+    virtual double getRealValue();  
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void pushDecimalValue();
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer &tgt);
+    virtual void deserialize(MemoryBuffer &src);
+
+}; 
+
+class DecimalValue : public CValue
+{
+protected:
+    void * val;
+
+public:
+    DecimalValue(const void * v, ITypeInfo * _type);
+    ~DecimalValue();
+
+// implementing IValue
+
+    virtual const void *queryValue() const;
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();  
+    virtual double getRealValue();  
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void pushDecimalValue();
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer &tgt);
+    virtual void deserialize(MemoryBuffer &src);
+
+};
+
+class EnumValue : public IntValue
+{
+public:
+    EnumValue(unsigned int v, ITypeInfo *_type) : IntValue(v, _type) {}
+
+// implementing IValue
+    virtual IValue * castTo(ITypeInfo * type);
+}; 
+
+class BoolValue : public CValue
+{
+private:
+    bool val;
+
+public:
+    BoolValue(bool v) : CValue(makeBoolType()), val(v)
+    {   
+    }
+
+    static BoolValue *trueconst;
+    static BoolValue *falseconst;
+
+//  implementing IValue
+
+    virtual const void *queryValue() const { return &val; }
+    virtual const char *generateCPP(StringBuffer &s, CompilerType compiler);
+    virtual const char *generateECL(StringBuffer &s);
+    virtual IValue * castTo(ITypeInfo * type);
+    virtual int compare(IValue * other);
+    virtual int compare(const void * mem);
+
+    virtual bool getBoolValue();    
+    virtual __int64 getIntValue();  
+    virtual const char *getStringValue(StringBuffer &); 
+    virtual void pushDecimalValue();
+    virtual void toMem(void * ptr);
+    virtual unsigned getHash(unsigned initval);
+    virtual int rangeCompare(ITypeInfo * targetType);
+// serializable
+    virtual void serialize(MemoryBuffer &tgt);
+    virtual void deserialize(MemoryBuffer &src);
+
+//  others
+
+    static BoolValue *getTrue();
+    static BoolValue *getFalse();
+}; 
+
+#endif

common/deftype/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/deftype</title>
+
+    <para>
+        The common/deftype directory contains the sources for the common/deftype library.
+    </para>
+</section>

common/dllserver/CMakeLists.txt

@@ -0,0 +1,67 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: dllserver 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for dllserver
+#####################################################
+
+project( dllserver ) 
+
+set (    SRCS 
+         dllserver.cpp 
+         thorplugin.cpp 
+
+         dllserver.hpp 
+         thorplugin.hpp 
+    )
+
+include_directories ( 
+         ./../../common/remote 
+         ./../../system/mp 
+         ./../../rtl/eclrtl 
+         ./../../system/include 
+         ./../../dali/base 
+         ./../../system/jlib 
+         ./../../common/environment 
+    )
+
+IF (NOT WIN32)
+  include_directories(${BINUTILS_INCLUDE_DIR})
+ENDIF()
+
+ADD_DEFINITIONS( -D_USRDLL -DDLLSERVER_EXPORTS )
+
+HPCC_ADD_LIBRARY( dllserver SHARED ${SRCS} )
+install ( TARGETS dllserver DESTINATION ${OSSDIR}/lib )
+target_link_libraries ( dllserver
+         ${ZLIB_LIBRARIES}
+         jlib 
+         mp 
+         hrpc 
+         eclrtl 
+         remote 
+         dalibase 
+         environment 
+    )
+if (NOT WIN32)
+  target_link_libraries ( dllserver ${BINUTILS_LIBRARIES})
+endif()
+

common/dllserver/dllserver.cpp

@@ -0,0 +1,826 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "dllserver.hpp"
+#include "jiter.ipp"
+#include "jexcept.hpp"
+
+#include "dllservererr.hpp"
+#include "daclient.hpp"
+#include "dasds.hpp"
+#include "rmtfile.hpp"
+#include "dalienv.hpp"
+#include "thorplugin.hpp"
+
+#ifdef _WIN32
+#define DEFAULT_SERVER_ROOTDIR          "c:\\dllserver"
+#else
+#define DEFAULT_SERVER_ROOTDIR          "/c$/dllserver"
+#endif
+
+static Owned<IConstDomainInfo> hostDomain;
+
+IConstDomainInfo * getDomainFromIp(const char * ip)
+{
+    Owned<IEnvironmentFactory> ef = getEnvironmentFactory();
+    Owned<IConstEnvironment> env = ef->openEnvironment();
+    if (!env)
+        return NULL;
+    Owned<IConstMachineInfo> curMachine = env->getMachineByAddress(ip);
+    if (!curMachine)
+        return NULL;
+    return curMachine->getDomain();
+}
+
+
+IConstDomainInfo * cachedHostDomain()
+{
+    if (!hostDomain)
+    {
+        StringBuffer ipText;
+        queryHostIP().getIpText(ipText);
+        hostDomain.setown(getDomainFromIp(ipText.str()));
+    }
+    return hostDomain;
+}
+
+void getPath(StringBuffer & path, const char * name)
+{
+    path.append("/GeneratedDlls/GeneratedDll[@uid=\"").append(name).append("\"]");
+}
+
+//---------------------------------------------------------------------------
+
+class TreeIteratorWrapper : public CInterface, implements IIterator
+{
+public:
+    TreeIteratorWrapper(IPropertyTreeIterator * _iter) { iter.setown(_iter); }
+    IMPLEMENT_IINTERFACE
+
+    virtual bool first()
+    {
+        cur.clear();
+        return iter->first();
+    }
+    virtual bool next()
+    {
+        cur.clear();
+        return iter->next();
+    }
+    virtual bool isValid()
+    {
+        return (cur != NULL);
+    }
+    virtual IInterface & get()
+    {
+        return *cur.getLink();
+    }
+    virtual IInterface & query()
+    {
+        return *cur;
+    }
+
+protected:
+    Owned<IInterface> cur;
+    Owned<IPropertyTreeIterator> iter;
+};
+
+
+//---------------------------------------------------------------------------
+
+
+static void deleteLocationFiles(IDllLocation & cur, bool removeDirectory)
+{
+    RemoteFilename filename;
+    cur.getDllFilename(filename);
+    Owned<IFile> file = createIFile(filename);
+    file->remove();
+
+    if (cur.getLibFilename(filename))
+    {
+        file.setown(createIFile(filename));
+        file->remove();
+    }
+
+    if (removeDirectory)
+    {
+        //Finally try and remove the directory - this will fail if it isn't empty
+        StringBuffer path, dir;
+
+        cur.getDllFilename(filename);
+        filename.getRemotePath(path);
+        splitDirTail(path.str(), dir);
+        filename.setRemotePath(dir.str());
+        file.setown(createIFile(filename));
+        file->remove();
+    }
+}
+
+class DllLocation : public CInterface, implements IDllLocation
+{
+    const char *cacheRoot;
+public:
+    DllLocation(IPropertyTree * _entryRoot, IPropertyTree * _locationRoot, const char *_cacheRoot) 
+        : cacheRoot(_cacheRoot)
+    { 
+        entryRoot.set(_entryRoot); 
+        locationRoot.set(_locationRoot); 
+    }
+    IMPLEMENT_IINTERFACE
+
+    virtual void getDllFilename(RemoteFilename & filename)
+    {
+        SocketEndpoint ep(locationRoot->queryProp("@ip"));
+        filename.setPath(ep, locationRoot->queryProp("@dll"));
+    }
+    virtual bool getLibFilename(RemoteFilename & filename)
+    {
+        const char * lib = locationRoot->queryProp("@lib");
+        if (!lib) return false;
+
+#ifndef _WIN32
+        int namelen = strlen(lib);
+        StringBuffer libnamebuf(lib);
+        if(!((namelen >= 3) && (strcmp(lib+namelen-3, ".so") == 0)))
+        {
+            libnamebuf.append(".so");
+            lib = libnamebuf.str();
+        }
+#endif
+        SocketEndpoint ep(locationRoot->queryProp("@ip"));
+        filename.setPath(ep, lib);
+        return true;
+    }
+    virtual void getIP(IpAddress & ip)
+    {
+        ip.ipset(locationRoot->queryProp("@ip"));
+    }
+    virtual DllLocationType queryLocation()
+    {
+        SocketEndpoint ep(locationRoot->queryProp("@ip"));
+        if (ep.isLocal())
+        {
+            RemoteFilename filename;
+            getDllFilename(filename);
+            StringBuffer lp;
+            filename.getLocalPath(lp);
+            if (strncmp(lp, cacheRoot, strlen(cacheRoot))==0)
+                return DllLocationDirectory;
+            return DllLocationLocal;
+        }
+#ifdef _WIN32
+        Owned<IConstDomainInfo> curDomain = getDomainFromIp(locationRoot->queryProp("@ip"));
+        IConstDomainInfo * hostDomain = cachedHostDomain();
+        if (curDomain && hostDomain && (curDomain == hostDomain))
+            return DllLocationDomain;
+#endif
+        return DllLocationAnywhere;
+    }
+    virtual void remove(bool removeFiles, bool removeDirectory);
+
+protected:
+    Owned<IPropertyTree> locationRoot;
+    Owned<IPropertyTree> entryRoot;
+};
+
+static bool propsMatch(IPropertyTree & left, IPropertyTree & right, const char * prop)
+{
+    const char * l = left.queryProp(prop);
+    const char * r = right.queryProp(prop);
+
+    if (!l || !r) return (l == r);
+    return strcmp(l, r) == 0;
+}
+
+void DllLocation::remove(bool removeFiles, bool removeDirectory)
+{
+    if (removeFiles)
+    {
+        try
+        {
+            deleteLocationFiles(*this, removeDirectory);
+        }
+        catch (IException * e)
+        {
+            EXCLOG(e, "Removing dll");
+            e->Release();
+        }
+    }
+
+    StringBuffer path;
+    getPath(path, entryRoot->queryProp("@name"));
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+    Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements("location");
+    ForEach(*iter)
+    {
+        IPropertyTree & cur = iter->query();
+        if (propsMatch(cur, *locationRoot, "@ip") && propsMatch(cur, *locationRoot, "@dll") && propsMatch(cur, *locationRoot, "@lib"))
+        {
+            conn->queryRoot()->removeTree(&cur);
+            break;
+        }
+    }
+}
+
+
+class LocationIterator : public TreeIteratorWrapper
+{
+    const char *cacheRoot;
+public:
+    LocationIterator(IPropertyTree * _dllEntry, IPropertyTreeIterator * _iter, const char *_cacheRoot) 
+        : TreeIteratorWrapper(_iter), cacheRoot(_cacheRoot)
+    {
+        dllEntry.set(_dllEntry); 
+    }
+
+    virtual bool first()
+    {
+        bool ok = TreeIteratorWrapper::first();
+        if (ok)
+            cur.setown(new DllLocation(dllEntry, &iter->query(), cacheRoot));
+        return ok;
+    }
+    virtual bool next()
+    {
+        bool ok = TreeIteratorWrapper::next();
+        if (ok)
+            cur.setown(new DllLocation(dllEntry, &iter->query(), cacheRoot));
+        return ok;
+    }
+
+protected:
+    Owned<IPropertyTree>    dllEntry;
+};
+
+
+//---------------------------------------------------------------------------
+
+class DllEntry : public CInterface, implements IDllEntry
+{
+public:
+    DllEntry(IPropertyTree * root, const char *cacheRoot);
+    IMPLEMENT_IINTERFACE
+
+    virtual IIterator * createLocationIterator();
+    virtual IDllLocation * getBestLocation();
+    virtual IDllLocation * getBestLocationCandidate();
+    virtual void getCreated(IJlibDateTime & dateTime)
+    {
+        dateTime.setString(root->queryProp("@created"));
+    }
+    virtual const char * queryKind()
+    {
+        return root->queryProp("@kind");
+    }
+    virtual const char * queryName()
+    {
+        return root->queryProp("@name");
+    }
+    virtual void remove(bool removeFiles, bool removeDirectory);
+
+protected:
+    Owned<IPropertyTree> root;
+    const char *cacheRoot;
+};
+
+DllEntry::DllEntry(IPropertyTree * _root, const char *_cacheRoot)
+: cacheRoot(_cacheRoot)
+{
+    root.set(_root);
+}
+
+IIterator * DllEntry::createLocationIterator()
+{
+    return new LocationIterator(root, root->getElements("location"), cacheRoot);
+}
+
+typedef IArrayOf<IDllLocation> LocationArray;
+
+int orderLocations(IInterface * * pLeft, IInterface * * pRight)
+{
+    IDllLocation * left = (IDllLocation *)*pLeft;
+    IDllLocation * right = (IDllLocation *)*pRight;
+    return right->queryLocation() - left->queryLocation();
+}
+
+IDllLocation * DllEntry::getBestLocation()
+{
+    LocationArray locations;
+    Owned<IIterator> iter = createLocationIterator();
+    ForEach(*iter)
+        locations.append((IDllLocation &)iter->get());
+    locations.sort(orderLocations);
+
+    ForEachItemIn(idx, locations)
+    {
+        IDllLocation & cur = locations.item(idx);
+        try
+        {
+            RemoteFilename filename;
+            cur.getDllFilename(filename);
+            Owned<IFile> file = createIFile(filename);
+            Owned<IFileIO> io = file->open(IFOread);
+            if (io)
+                return LINK(&cur);
+        }
+        catch (IException * e)
+        {
+            StringBuffer s;
+            EXCLOG(e, s.appendf("GetBestLocation - could not load entry for %s", root->queryProp("@name")).str());
+            e->Release();
+            //MORE: Should possibly remove the entry... but we don't know why we couldn't read it - could be a domain issue.
+        }
+    }
+    throwError1(DSVERR_CouldNotFindDll, root->queryProp("@name"));
+    return NULL;
+}
+
+IDllLocation * DllEntry::getBestLocationCandidate()
+{
+    Owned<IIterator> iter = createLocationIterator();
+    Owned<IDllLocation> best;
+    DllLocationType bestLocation = DllLocationNowhere;
+
+    ForEach(*iter)
+    {
+        IDllLocation & cur = (IDllLocation &)iter->query();
+        DllLocationType location = cur.queryLocation();
+        if (location == DllLocationDirectory)
+            return LINK(&cur);
+        if (location > bestLocation)
+        {
+            best.set(&cur);
+            bestLocation = location;
+        }
+    }
+
+    if (!best)
+        throwError1(DSVERR_CouldNotFindDll, root->queryProp("@name"));
+
+    return best.getClear();
+}
+
+
+
+void DllEntry::remove(bool removeFiles, bool removeDirectory)
+{
+    if (removeFiles)
+    {
+        Owned<IIterator> iter = createLocationIterator();
+        // a bit nasty, but don't remove the directory for the first location, since this is where things are created in the first place.
+        bool isFirst = true;                
+        ForEach(*iter)
+        {
+            try
+            {
+                IDllLocation & cur = (IDllLocation &)iter->query();
+                deleteLocationFiles(cur, removeDirectory && !isFirst);
+            }
+            catch (IException * e)
+            {
+                EXCLOG(e, "Removing dll");
+                e->Release();
+            }
+            isFirst = false;
+        }
+    }
+    StringBuffer path;
+    getPath(path, root->queryProp("@name"));
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+    conn->close(true);
+}
+
+
+
+class DllIterator : public TreeIteratorWrapper
+{
+public:
+    DllIterator(IPropertyTree * _root, IPropertyTreeIterator * _iter, const char *_cacheRoot) 
+        : TreeIteratorWrapper(_iter), cacheRoot(_cacheRoot)
+    { 
+        root.set(_root); 
+    }
+
+    virtual bool first()
+    {
+        bool ok = TreeIteratorWrapper::first();
+        if (ok)
+            cur.setown(new DllEntry(&iter->query(), cacheRoot));
+        return ok;
+    }
+    virtual bool next()
+    {
+        bool ok = TreeIteratorWrapper::next();
+        if (ok)
+            cur.setown(new DllEntry(&iter->query(), cacheRoot));
+        return ok;
+    }
+
+protected:
+    Owned<IPropertyTree> root;
+    const char *cacheRoot;
+};
+
+
+//---------------------------------------------------------------------------
+
+class DllServer : public CInterface, implements IDllServer
+{
+public:
+    DllServer(const char * _rootDir);
+    IMPLEMENT_IINTERFACE
+
+    virtual IIterator * createDllIterator();
+    virtual void ensureAvailable(const char * name, DllLocationType location);
+    virtual void getDll(const char * name, MemoryBuffer & dllText);
+    virtual IDllEntry * getEntry(const char * name);
+    virtual void getLibrary(const char * name, MemoryBuffer & dllText);
+    virtual void getLocalLibraryName(const char * name, StringBuffer & libraryName);
+    virtual DllLocationType isAvailable(const char * name);
+    virtual ILoadedDllEntry * loadDll(const char * name, DllLocationType location);
+    virtual void removeDll(const char * name, bool removeDlls, bool removeDirectory);
+    virtual void registerDll(const char * name, const char * kind, const char * dllPath);
+
+protected:
+    void copyFileLocally(RemoteFilename & targetName, RemoteFilename & sourceName);
+    DllEntry * doGetEntry(const char * name);
+    void doRegisterDll(const char * name, const char * kind, const char * dllPath, const char * libPath);
+    IDllLocation * getBestMatch(const char * name);
+    IDllLocation * getBestMatchEx(const char * name);
+
+protected:
+    StringAttr rootDir;
+};
+
+DllServer::DllServer(const char * _rootDir) 
+{
+    rootDir.set(_rootDir); 
+}
+
+void DllServer::copyFileLocally(RemoteFilename & targetName, RemoteFilename & sourceName)
+{
+    StringBuffer sourceLocalPath,targetLocalPath;
+
+    sourceName.getLocalPath(sourceLocalPath);
+
+    targetLocalPath.append(rootDir);
+    recursiveCreateDirectory(targetLocalPath.str());
+    targetLocalPath.append(PATHSEPCHAR);
+    splitFilename(sourceLocalPath.str(),NULL,NULL,&targetLocalPath,&targetLocalPath);
+
+    SocketEndpoint hostEndpoint;
+    hostEndpoint.setLocalHost((unsigned short)0);
+    targetName.setPath(hostEndpoint, targetLocalPath.str());
+
+    OwnedIFile source = createIFile(sourceName);
+    OwnedIFile target = createIFile(targetName);
+    copyFile(target, source);
+}
+
+IIterator * DllServer::createDllIterator()
+{
+    Owned<IRemoteConnection> conn = querySDS().connect("/GeneratedDlls", myProcessSession(), 0, 5000);
+    IPropertyTree * root = conn->queryRoot();
+    return conn ? (IIterator *)new DllIterator(root, root->getElements("GeneratedDll"), rootDir) : (IIterator *)new CNullIterator;
+}
+
+DllEntry * DllServer::doGetEntry(const char * name)
+{
+    StringBuffer path;
+    getPath(path, name);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), 0, 5000);
+    if (conn)
+        return new DllEntry(conn->queryRoot(), rootDir);
+    return NULL;
+}
+
+
+void DllServer::doRegisterDll(const char * name, const char * kind, const char * dllPath, const char * libPath)
+{
+    Owned<IRemoteConnection> lock = querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE, 5000);
+
+    RemoteFilename dllRemote;
+    StringBuffer ipText, dllText;
+    dllRemote.setRemotePath(dllPath);
+    dllRemote.queryIP().getIpText(ipText);
+    dllRemote.getLocalPath(dllText);
+
+    StringBuffer path;
+    getPath(path, name);
+    Owned<IRemoteConnection> conn = querySDS().connect(path.str(), myProcessSession(), RTM_LOCK_WRITE, 5000);
+    if (conn)
+    {
+        //check the entry doesn't exist already....
+        Owned<IPropertyTreeIterator> iter = conn->queryRoot()->getElements("location");
+        ForEach(*iter)
+        {
+            IPropertyTree & cur = iter->query();
+            if ((stricmp(cur.queryProp("@ip"),ipText.str()) == 0) && 
+                (stricmp(cur.queryProp("@dll"),dllText.str()) == 0))
+                return;
+        }
+    }
+    else
+    {
+        conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
+        if (!conn)
+        {
+            ::Release(querySDS().connect("/GeneratedDlls", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
+            conn.setown(querySDS().connect("/GeneratedDlls/GeneratedDll", myProcessSession(), RTM_LOCK_WRITE|RTM_CREATE_ADD, 5000));
+        }
+
+        IPropertyTree * entry = conn->queryRoot();
+        entry->setProp("@name", name);
+        entry->setProp("@kind", kind);
+
+        Owned<IJlibDateTime> now = createDateTimeNow();
+        StringBuffer nowText;
+        StringBufferAdaptor strval(nowText);
+        now->getString(strval);
+        entry->setProp("@created", nowText.str());
+
+        conn->queryRoot()->setProp("@uid", name);
+    }
+
+    IPropertyTree * locationTree = createPTree("location", false);
+    locationTree->setProp("@ip", ipText.str());
+    locationTree->setProp("@dll", dllText.str());
+
+    if (libPath && strlen(libPath))
+    {
+        RemoteFilename libRemote;
+        libRemote.setRemotePath(libPath);
+        if (!dllRemote.queryIP().ipequals(libRemote.queryIP()))
+            throwError(DSVERR_DllLibIpMismatch);
+
+        StringBuffer libText;
+        libRemote.getLocalPath(libText);
+        locationTree->setProp("@lib", libText.str());
+    }
+
+    conn->queryRoot()->addPropTree("location", locationTree);
+}
+
+void DllServer::ensureAvailable(const char * name, DllLocationType location)
+{
+    Owned<DllEntry> match = doGetEntry(name);
+    if (!match)
+        throwError1(DSVERR_CouldNotFindDll, name);
+
+    Owned<IDllLocation> bestLocation = match->getBestLocation();
+    if (bestLocation->queryLocation() < location)
+    {
+        StringBuffer remoteDllPath, remoteLibPath;
+        RemoteFilename sourceName, dllName, libName;
+
+        bestLocation->getDllFilename(sourceName);
+        copyFileLocally(dllName, sourceName);
+        dllName.getRemotePath(remoteDllPath);
+
+        if (bestLocation->getLibFilename(sourceName))
+        {
+            copyFileLocally(libName, sourceName);
+            libName.getRemotePath(remoteLibPath);
+        }
+
+        doRegisterDll(name, "**Error**", remoteDllPath.str(), remoteLibPath.str());
+        assertex(isAvailable(name) >= DllLocationLocal);
+    }
+}
+
+IDllLocation * DllServer::getBestMatch(const char * name)
+{
+    Owned<DllEntry> match = doGetEntry(name);
+    if (!match)
+        return NULL;
+
+    return match->getBestLocation();
+}
+
+
+IDllLocation * DllServer::getBestMatchEx(const char * name)
+{
+    IDllLocation * location = getBestMatch(name);
+    if (!location)
+        throwError1(DSVERR_CouldNotFindDll, name);
+    return location;
+}
+
+
+IDllEntry * DllServer::getEntry(const char * name)
+{
+    return doGetEntry(name);
+}
+
+
+void DllServer::getDll(const char * name, MemoryBuffer & dllText)
+{
+    RemoteFilename filename;
+    Owned<IDllLocation> match = getBestMatchEx(name);
+    match->getDllFilename(filename);
+
+    Owned<IFile> file = createIFile(filename);
+    OwnedIFileIO io = file->open(IFOread);
+    read(io, 0, (size32_t)-1, dllText);
+}
+
+void DllServer::getLibrary(const char * name, MemoryBuffer & libText)
+{
+    RemoteFilename filename;
+    Owned<IDllLocation> match = getBestMatchEx(name);
+    if (!match->getLibFilename(filename))
+        throwError1(DSVERR_NoAssociatedLib, name);
+
+    Owned<IFile> file = createIFile(filename);
+    OwnedIFileIO io = file->open(IFOread);
+    read(io, 0, (size32_t)-1, libText);
+}
+
+void DllServer::getLocalLibraryName(const char * name, StringBuffer & libraryName)
+{
+    RemoteFilename filename;
+    Owned<IDllLocation> match = getBestMatchEx(name);
+    if (match->queryLocation() < DllLocationLocal)
+        throwError1(DSVERR_LibNotLocal, name);
+
+    if (!match->getLibFilename(filename))
+        throwError1(DSVERR_NoAssociatedLib, name);
+
+    filename.getLocalPath(libraryName);
+}
+
+DllLocationType DllServer::isAvailable(const char * name)
+{
+    try
+    {
+        Owned<IDllLocation> match = getBestMatch(name);
+        if (match)
+            return match->queryLocation();
+    }
+    catch (IException *E)
+    {
+        E->Release();
+    }
+    return DllLocationNowhere;
+}
+
+ILoadedDllEntry * DllServer::loadDll(const char * name, DllLocationType type)
+{
+#ifdef _WIN32
+    if (type < DllLocationDomain)
+        type = DllLocationDomain;
+#else
+    if (type < DllLocationLocal)
+        type = DllLocationLocal;
+#endif
+    ensureAvailable(name, type);
+    Owned<IDllLocation> location = getBestMatchEx(name);
+    RemoteFilename rfile;
+    location->getDllFilename(rfile);
+    StringBuffer x;
+    rfile.getPath(x);
+    LOG(MCdebugInfo, unknownJob, "Loading dll (%s) from location %s", name, x.str());
+    return createDllEntry(x.str(), false, NULL);
+}
+
+void DllServer::removeDll(const char * name, bool removeDlls, bool removeDirectory)
+{
+    Owned<IDllEntry> entry = getEntry(name);
+    if (entry)
+        entry->remove(removeDlls, removeDirectory);
+}
+
+void DllServer::registerDll(const char * name, const char * kind, const char * dllLocation)
+{
+    doRegisterDll(name, kind, dllLocation, NULL);
+}
+
+//---------------------------------------------------------------------------
+
+static DllServer * dllServer;
+
+#if 0
+struct __init_dllserver
+{
+    ~__init_dllserver() { ::Release(dllServer); }
+} __do_init_dllserver;
+#endif
+
+IDllServer & queryDllServer()
+{
+    if (!dllServer)
+    {
+        const char* dllserver_root = getenv("DLLSERVER_ROOT");
+        StringBuffer dir;
+        if(dllserver_root == NULL)
+        {
+            if (envGetConfigurationDirectory("temp","dllserver","dllserver",dir)) // not sure if different instance might be better but never separated in past
+                dllserver_root = dir.str();
+            else
+                dllserver_root = DEFAULT_SERVER_ROOTDIR;
+        }
+        initDllServer(dllserver_root);
+    }
+
+    return *dllServer;
+}
+
+void closeDllServer()
+{
+    hostDomain.clear();
+    if (dllServer)
+    {
+        dllServer->Release();
+        dllServer = NULL;
+    }
+}
+
+
+void initDllServer(const char * localRoot)
+{
+    ::Release(dllServer);
+    dllServer = new DllServer(localRoot);
+}
+
+
+void cleanUpOldDlls()
+{
+    Owned<IJlibDateTime> cutoff = createDateTimeNow();
+    unsigned year;
+    unsigned month;
+    unsigned day;
+    cutoff->getGmtDate(year, month, day);
+    if (false && month-- == 1)
+    {
+        month = 12;
+        year--;
+    }
+    cutoff->setGmtDate(year, month, day);
+
+    //Remove all but one copy of all workunits more than a month old
+    Owned<IIterator> dllIter = queryDllServer().createDllIterator();
+    ForEach(*dllIter)
+    {
+        IDllEntry & entry = (IDllEntry &)dllIter->query();
+
+        if (strcmp(entry.queryKind(), "workunit")==0)
+        {
+            Owned<IJlibDateTime> created = createDateTime();
+            entry.getCreated(*created);
+            if (created->compare(*cutoff) < 0)
+            {
+                Owned<IIterator> locIter = entry.createLocationIterator();
+                bool first = true;
+
+                ForEach(*locIter)
+                {
+                    IDllLocation & location = (IDllLocation &)locIter->query();
+                    if (!first)
+                        location.remove(true, true);
+                    first = false;
+                }
+            }
+        }
+    }
+}
+
+
+void testDllServer()
+{
+    IDllServer & server = queryDllServer();
+    Owned<IDllEntry> oldentry2 = server.getEntry("WorkUnit1");
+    if (oldentry2)
+        oldentry2->remove(false, false);
+    server.registerDll("WorkUnit1","workunit","\\\\ghalliday\\c$\\edata\\ecl\\regress\\process0.dll");
+    assertex(server.isAvailable("WorkUnit1") == DllLocationLocal);
+    server.ensureAvailable("WorkUnit1",DllLocationLocal);
+    server.registerDll("WorkUnit1","workunit","\\\\1.1.1.1\\c$\\edata\\ecl\\regress\\process0.dll");
+
+    const char * abcFilename = "\\\\127.0.0.1\\c$\\temp\\dlltest\abc";
+    Owned<IFile> temp = createIFile(abcFilename);
+    recursiveCreateDirectoryForFile(abcFilename);
+    Owned<IFileIO> io = temp->open(IFOcreate);
+    io->write(0, 10, "abcdefghij");
+    io.clear();
+    server.registerDll("WorkUnitAbc","workunit",abcFilename);
+    server.removeDll("WorkUnitAbc", true, true);
+
+    cleanUpOldDlls();
+}
+

common/dllserver/dllserver.hpp

@@ -0,0 +1,94 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef DLLSERVER_INCL
+#define DLLSERVER_INCL
+
+#ifdef _WIN32
+    #ifdef DLLSERVER_EXPORTS
+        #define DLLSERVER_API __declspec(dllexport)
+    #else
+        #define DLLSERVER_API __declspec(dllimport)
+    #endif
+#else
+    #define DLLSERVER_API
+#endif
+
+#include "jiface.hpp"
+
+class IpAddress;
+class RemoteFilename;
+class MemoryBuffer;
+class StringBuffer;
+interface IJlibDateTime;
+
+enum DllLocationType
+{
+    DllLocationNowhere = 0,
+    DllLocationAnywhere = 1,
+    DllLocationDomain = 2,
+    DllLocationLocal = 3,
+    DllLocationDirectory = 4,
+    DllLocationSize = 5
+};
+
+interface IDllLocation : extends IInterface
+{
+    virtual void getDllFilename(RemoteFilename & filename) = 0;
+    virtual bool getLibFilename(RemoteFilename & filename) = 0;
+    virtual void getIP(IpAddress & ip) = 0;
+    virtual DllLocationType queryLocation() = 0;
+    virtual void remove(bool removeFiles, bool removeDirectory) = 0;
+};
+
+
+
+interface IDllEntry : extends IInterface
+{
+    virtual IIterator * createLocationIterator() = 0;
+    virtual IDllLocation * getBestLocation() = 0;
+    virtual IDllLocation * getBestLocationCandidate() = 0;
+    virtual void getCreated(IJlibDateTime & dateTime) = 0;
+    virtual const char * queryKind() = 0;
+    virtual const char * queryName() = 0;
+    virtual void remove(bool removeFiles, bool removeDirectory) = 0;
+};
+
+
+interface ILoadedDllEntry;
+
+interface IDllServer : extends IInterface
+{
+    virtual IIterator * createDllIterator() = 0;
+    virtual void ensureAvailable(const char * name, DllLocationType location) = 0;
+    virtual void getDll(const char * name, MemoryBuffer & dllText) = 0;
+    virtual IDllEntry * getEntry(const char * name) = 0;
+    virtual void getLibrary(const char * name, MemoryBuffer & dllText) = 0;
+    virtual void getLocalLibraryName(const char * name, StringBuffer & libraryName) = 0;
+    virtual DllLocationType isAvailable(const char * name) = 0;
+    virtual ILoadedDllEntry * loadDll(const char * name, DllLocationType location) = 0;
+    virtual void removeDll(const char * name, bool removeDlls, bool removeDirectory) = 0;
+    virtual void registerDll(const char * name, const char * kind, const char * dllPath) = 0;
+};
+
+extern "C" DLLSERVER_API IDllServer & queryDllServer();
+extern "C" DLLSERVER_API void closeDllServer();
+extern "C" DLLSERVER_API void initDllServer(const char * localRoot);
+extern DLLSERVER_API void testDllServer();
+
+#endif

common/dllserver/dllservererr.hpp

@@ -0,0 +1,37 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef DLLSERVEERR_HPP
+#define DLLSERVEERR_HPP
+
+#define ERR_DSV_FIRST                           9000
+#define ERR_DSV_LAST                            9009
+
+#define DSVERR_CouldNotFindDll                  9000
+#define DSVERR_DllLibIpMismatch                 9001
+#define DSVERR_NoAssociatedLib                  9002
+#define DSVERR_LibNotLocal                      9003
+
+//---- Text for all errors (make it easy to internationalise) ---------------------------
+
+#define DSVERR_CouldNotFindDll_Text             "Could not find location of %s"
+#define DSVERR_DllLibIpMismatch_Text            "The DLL and Library have a mismatched IP address"
+#define DSVERR_NoAssociatedLib_Text             "No Library associated with DLL %s"
+#define DSVERR_LibNotLocal_Text                 "Library for %s not copied locally"
+
+#endif

common/dllserver/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/dllserver</title>
+
+    <para>
+        The common/dllserver directory contains the sources for the common/dllserver library.
+    </para>
+</section>

common/dllserver/thorplugin.cpp

@@ -0,0 +1,516 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jexcept.hpp"
+#include "jmisc.hpp"
+#include "jthread.hpp"
+#include "jsocket.hpp"
+#include "jprop.hpp"
+#include "jdebug.hpp"
+#include "jlzw.hpp"
+#include "eclrtl.hpp"
+#ifndef _WIN32
+#include "bfd.h"
+#endif
+
+#include "thorplugin.hpp"
+
+void * SimplePluginCtx::ctxMalloc(size_t size)
+{ 
+    return rtlMalloc(size); 
+}
+
+void * SimplePluginCtx::ctxRealloc(void * _ptr, size_t size)
+{ 
+    return rtlRealloc(_ptr, size);
+}
+void SimplePluginCtx::ctxFree(void * _ptr)
+{ 
+    rtlFree(_ptr);
+}
+
+char * SimplePluginCtx::ctxStrdup(char * _ptr)
+{ 
+    return strdup(_ptr); 
+}
+
+int SimplePluginCtx::ctxGetPropInt(const char *propName, int defaultValue) const
+{
+    return defaultValue;
+}
+
+const char * SimplePluginCtx::ctxQueryProp(const char *propName) const
+{
+    return NULL;
+}
+
+//-------------------------------------------------------------------------------------------------------------------
+
+class HelperDll : public CInterface, implements ILoadedDllEntry
+{
+    SharedObject so;
+    StringBuffer version;
+    StringBuffer fileLocation;
+    StringAttr name;
+    Linked<const IFileIO> dllFile;
+    bool logLoad;
+public:
+    IMPLEMENT_IINTERFACE;
+    HelperDll(const char *_name, const IFileIO *dllFile);
+    ~HelperDll();
+
+//interface ILoadedDllEntry
+    virtual HINSTANCE getInstance() const;
+    virtual void * getEntry(const char * name) const;
+    virtual bool IsShared();
+    virtual const char * queryVersion() const;
+    virtual const char * queryName() const;
+    virtual const byte * getResource(unsigned id) const;
+    virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id) const;
+
+    bool load(bool isGlobal, bool raiseOnError);
+    bool loadCurrentExecutable();
+    virtual void logLoaded();
+    virtual bool checkVersion(const char *expected);
+};
+
+class PluginDll : public HelperDll
+{
+    ECLPluginDefinitionBlockEx pb;
+
+public:
+    PluginDll(const char *_name, const IFileIO *_dllFile) : HelperDll(_name, _dllFile) {}
+
+    bool init(IPluginContextEx * pluginCtx);
+
+    virtual bool checkVersion(const char *expected);
+    virtual void logLoaded();
+};
+
+HelperDll::HelperDll(const char *_name, const IFileIO *_dllFile) 
+: name(_name), dllFile(_dllFile)
+{
+    logLoad = false;
+}
+
+bool HelperDll::load(bool isGlobal, bool raiseOnError)
+{
+    if (!so.load(name, isGlobal, raiseOnError))
+        return false;
+    return true;
+}
+
+bool HelperDll::loadCurrentExecutable()
+{
+    if (!so.loadCurrentExecutable())
+        return false;
+    return true;
+}
+
+HelperDll::~HelperDll()
+{
+    if (logLoad)
+        DBGLOG("Unloading dll %s", name.get());
+}
+
+HINSTANCE HelperDll::getInstance() const
+{
+    return so.getInstanceHandle();
+}
+
+void * HelperDll::getEntry(const char * name) const
+{
+    return GetSharedProcedure(so.getInstanceHandle(), name);
+}
+
+bool HelperDll::IsShared()
+{
+    return CInterface::IsShared();
+}
+
+const char * HelperDll::queryVersion() const
+{
+    return version.str();
+}
+
+void HelperDll::logLoaded()
+{
+    logLoad = true;
+    DBGLOG("Loaded DLL %s", name.get());
+}
+
+bool HelperDll::checkVersion(const char *expected)
+{
+    return true;
+}
+
+const char * HelperDll::queryName() const
+{
+    return name.get();
+}
+
+const byte * HelperDll::getResource(unsigned id) const
+{
+#ifdef _WIN32
+    HINSTANCE dllHandle = so.getInstanceHandle();
+    HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), "BIGSTRING");
+    if (hrsrc)
+        return (const byte *) LoadResource(dllHandle, hrsrc);
+    return NULL;
+#else
+    StringBuffer resourceName;
+    resourceName.appendf("BIGSTRING_%d_txt_start", id);
+    return (const byte *) getEntry(resourceName.str());
+#endif
+}
+
+bool HelperDll::getResource(size32_t & len, const void * & data, const char * type, unsigned id) const
+{
+#ifdef _WIN32
+    HINSTANCE dllHandle = so.getInstanceHandle();
+    HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
+    if (!hrsrc)
+        return false;
+    len = SizeofResource(dllHandle, hrsrc);
+    data = (const byte *) LoadResource(dllHandle, hrsrc);
+    return true;
+#else
+    StringBuffer baseName;
+    baseName.append(type).append("_").append(id);
+    StringBuffer symName, sizeName;
+    symName.append(baseName).append("_txt_start");
+    sizeName.append(baseName).append("_txt_size");
+    data = (const void *) getEntry(symName.str());
+    if (!data)
+        return false;
+    len = (unsigned)(memsize_t)getEntry(sizeName.str());
+    return true;
+#endif
+}
+
+#ifndef _WIN32
+struct SecScanParam
+{
+    MemoryBuffer &result;
+    const char *sectionName;
+    SecScanParam(MemoryBuffer &_result, const char *_sectionName) 
+        : result(_result), sectionName(_sectionName)
+    {
+    }
+};
+
+static void secscan (bfd *file, sec_ptr sec, void *userParam)
+{
+    SecScanParam *param = (SecScanParam *) userParam;
+    if (strcmp(param->sectionName, bfd_section_name (file, sec))==0)
+    {
+        bfd_size_type size = bfd_section_size (file, sec);
+        void *data = (void *) param->result.reserve(size);
+        bfd_get_section_contents(file, sec, data, 0, size);
+    }
+}
+#endif
+
+extern bool getResourceFromFile(const char *filename, MemoryBuffer &data, const char * type, unsigned id)
+{
+#ifdef _WIN32
+    HINSTANCE dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE|LOAD_LIBRARY_AS_IMAGE_RESOURCE);
+    if (dllHandle == NULL)
+        dllHandle = LoadLibraryEx(filename, NULL, LOAD_LIBRARY_AS_DATAFILE); // the LOAD_LIBRARY_AS_IMAGE_RESOURCE flag is not supported on all versions of Windows
+    if (dllHandle == NULL)
+    {
+        DBGLOG("Failed to load library %s: %d", filename, GetLastError());
+        return false;
+    }
+    HRSRC hrsrc = FindResource(dllHandle, MAKEINTRESOURCE(id), type);
+    if (!hrsrc)
+        return false;
+    size32_t len = SizeofResource(dllHandle, hrsrc);
+    const void *rdata = (const void *) LoadResource(dllHandle, hrsrc);
+    data.append(len, rdata);
+    FreeLibrary(dllHandle);
+    return true;
+#else
+    bfd_init ();
+    bfd *file = bfd_openr(filename, NULL);
+    if (file)
+    {
+        StringBuffer sectionName;
+        sectionName.append(type).append("_").append(id).append(".data");
+        SecScanParam param(data, sectionName.str());
+        if (bfd_check_format (file, bfd_object))
+            bfd_map_over_sections (file, secscan, &param);
+        bfd_close (file);
+   }
+   return data.length() != 0;
+#endif
+}
+
+//-------------------------------------------------------------------------------------------------------------------
+
+bool PluginDll::init(IPluginContextEx * pluginCtx)
+{
+    HINSTANCE h = getInstance();
+    assertex(h != (HINSTANCE) -1);
+    EclPluginSetCtxEx pSetCtxEx = (EclPluginSetCtxEx) GetSharedProcedure(h,"setPluginContextEx");
+    if (pSetCtxEx)
+        pSetCtxEx(pluginCtx);
+    else
+    {
+        // Older plugins may only support setPluginContext - fall back to that
+        EclPluginSetCtx pSetCtx = (EclPluginSetCtx) GetSharedProcedure(h,"setPluginContext");
+        if (pSetCtx)
+            pSetCtx(pluginCtx);
+    }
+
+    EclPluginDefinition p= (EclPluginDefinition) GetSharedProcedure(h,"getECLPluginDefinition");
+    if (!p)
+        return false;
+
+    pb.size = sizeof(ECLPluginDefinitionBlockEx);
+    if (!p(&pb))
+    {
+        pb.compatibleVersions = NULL;
+        pb.size = sizeof(ECLPluginDefinitionBlock);
+        if (!p(&pb))
+            return false;
+    }
+    return true;
+}
+
+
+bool PluginDll::checkVersion(const char *expected)
+{
+    assertex(expected);
+    if (stricmp(pb.version, expected) == 0)
+        return true;
+
+    if (pb.compatibleVersions)
+    {
+        const char **finger = pb.compatibleVersions;
+        while (*finger)
+        {
+            if (stricmp(*finger, expected) == 0)
+                return true;
+            finger++;
+        }
+    }
+    return false;
+}
+
+
+void PluginDll::logLoaded()
+{
+    HelperDll::logLoaded();
+    DBGLOG("Current reported version is %s", pb.version);
+    if (pb.compatibleVersions)
+    {
+        const char **finger = pb.compatibleVersions;
+        while (*finger)
+        {
+            DBGLOG("Compatible version %s", *finger);
+            finger++;
+        }
+    }
+}
+
+extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *path, bool isGlobal, const IFileIO *dllFile)
+{
+    Owned<HelperDll> result = new HelperDll(path, dllFile);
+    if (!result->load(isGlobal, true))
+        throw MakeStringException(0, "Failed to create ILoadedDllEntry for dll %s", path);
+    return result.getClear();
+}
+
+extern DLLSERVER_API ILoadedDllEntry * createExeDllEntry(const char *path)
+{
+    Owned<HelperDll> result = new HelperDll(path, NULL);
+    if (!result->loadCurrentExecutable())
+        throw MakeStringException(0, "Failed to create ILoadedDllEntry for current executable");
+    return result.getClear();
+}
+
+extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result)
+{
+    bool hasVersion = len && (*(const byte *)data == 0x80);
+    MemoryBuffer src;
+    src.setBuffer(len, const_cast<void *>(data), false);
+    byte version = 1;
+    if (hasVersion)
+    {
+        src.skip(1);
+        src.read(version);
+    }
+
+    MemoryBuffer tgt;
+    switch (version)
+    {
+    case 1:
+        decompressToBuffer(tgt, src);
+        break;
+    default:
+        throwUnexpected();
+    }
+
+    tgt.append((char)0);
+    unsigned expandedLen = tgt.length();
+    result.setBuffer(expandedLen, reinterpret_cast<char *>(tgt.detach()), expandedLen-1);
+    return true;
+}
+
+extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data)
+{
+    byte version = 1;
+    compressed.append((byte)0x80).append(version);
+    compressToBuffer(compressed, len, data);
+}
+
+extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuffer &xml)
+{
+    size32_t len = 0;
+    const void * data = NULL;
+    if (!dll->getResource(len, data, "WORKUNIT", 1000))
+        return false;
+    return decompressResource(len, data, xml);
+}
+
+extern DLLSERVER_API bool getEmbeddedSoapInfoXML(ILoadedDllEntry *dll, StringBuffer &xml)
+{
+    size32_t len = 0;
+    const void * data = NULL;
+    if (!dll->getResource(len, data, "SOAPINFO", 1000))
+        return false;
+    return decompressResource(len, data, xml);
+}
+
+extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll)
+{
+    size32_t len = 0;
+    const void * data = NULL;
+    return dll->getResource(len, data, "WORKUNIT", 1000);
+}
+
+extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml)
+{
+    MemoryBuffer data;
+    if (!getResourceFromFile(filename, data, type, id))
+        return false;
+    return decompressResource(data.length(), data.toByteArray(), xml);
+}
+
+extern DLLSERVER_API bool getWorkunitXMLFromFile(const char *filename, StringBuffer &xml)
+{
+    return getResourceXMLFromFile(filename, "WORKUNIT", 1000, xml);
+}
+
+extern DLLSERVER_API bool getSoapInfoXMLFromFile(const char *filename, StringBuffer &xml)
+{
+    return getResourceXMLFromFile(filename, "SOAPINFO", 1000, xml);
+}
+
+
+//-------------------------------------------------------------------------------------------------------------------
+
+
+//-------------------------------------------------------------------------------------------------------------------
+
+bool SafePluginMap::addPlugin(const char *path, const char *dllname)
+{
+    if (!endsWithIgnoreCase(path, SharedObjectExtension))
+    {
+        if (trace)
+            DBGLOG("Ecl plugin %s ignored", path);
+        return false;
+    }
+
+    try
+    {
+        CriticalBlock b(crit);
+        ILoadedDllEntry *dll = map.getValue(dllname);
+        if (!dll)
+        {
+            Owned<PluginDll> n = new PluginDll(path, NULL);
+            if (!n->load(true, false) || !n->init(pluginCtx))
+                throw MakeStringException(0, "Failed to load plugin %s", path);
+            if (trace)
+                n->logLoaded();
+            map.setValue(dllname, n);  // note: setValue links arg
+            return true;
+        }
+        return false;
+    }
+    catch (IException * e) // MORE - not sure why we don't throw exceptions back here...
+    {
+        e->Release();
+        return false;
+    }
+}
+
+
+ILoadedDllEntry * SafePluginMap::getPluginDll(const char *id, const char *version, bool checkVersion)
+{
+    CriticalBlock b(crit);
+    Linked<PluginDll> ret = static_cast<PluginDll *>(map.getValue(id));
+    if (ret && checkVersion)
+    {
+        if (!ret->checkVersion(version))
+            return NULL;
+    }
+    return ret.getLink();
+}
+
+void SafePluginMap::loadFromList(const char * pluginsList)
+{
+    const char *pluginDir = pluginsList;
+    for (;*pluginDir;)
+    {
+        StringBuffer thisPlugin;
+        while (*pluginDir && *pluginDir != ENVSEPCHAR)
+            thisPlugin.append(*pluginDir++);
+        if(*pluginDir)
+            pluginDir++;
+
+        if(!thisPlugin.length())
+            continue;
+
+        Owned<IFile> file = createIFile(thisPlugin.str());
+        if (file->isDirectory() == foundYes)
+            loadFromDirectory(thisPlugin);
+        else
+        {
+            StringBuffer tail;
+            splitFilename(thisPlugin, NULL, NULL, &tail, &tail);
+            addPlugin(thisPlugin, tail.str());
+        }
+    }
+}
+
+void SafePluginMap::loadFromDirectory(const char * pluginDirectory)
+{
+    const char * mask = "*" SharedObjectExtension;
+    
+    Owned<IFile> pluginDir = createIFile(pluginDirectory);
+    Owned<IDirectoryIterator> pluginFiles = pluginDir->directoryFiles(mask,false,false);
+    ForEach(*pluginFiles)
+    {
+        const char *thisPlugin = pluginFiles->query().queryFilename();
+        StringBuffer tail;
+        splitFilename(thisPlugin, NULL, NULL, &tail, &tail);
+        addPlugin(thisPlugin, tail.str());
+    }
+}

common/dllserver/thorplugin.hpp

@@ -0,0 +1,79 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef THORPLUGIN_HPP
+#define THORPLUGIN_HPP
+
+#include "hqlplugins.hpp"
+#include "dllserver.hpp"
+
+// call release to unload it.
+interface ILoadedDllEntry : extends IInterface
+{
+    virtual void * getEntry(const char * name) const = 0;
+    virtual HINSTANCE getInstance() const = 0;
+    virtual bool IsShared() = 0;
+    virtual const char * queryVersion() const = 0;
+    virtual const char * queryName() const = 0;
+    virtual const byte * getResource(unsigned id) const = 0;
+    virtual bool getResource(size32_t & len, const void * & data, const char * type, unsigned id) const = 0;
+};
+
+extern DLLSERVER_API ILoadedDllEntry * createDllEntry(const char *name, bool isGlobal, const IFileIO *dllFile);
+extern DLLSERVER_API ILoadedDllEntry * createExeDllEntry(const char *name);
+extern DLLSERVER_API bool getEmbeddedWorkUnitXML(ILoadedDllEntry *dll, StringBuffer &xml);
+extern DLLSERVER_API bool getEmbeddedSoapInfoXML(ILoadedDllEntry *dll, StringBuffer &xml);
+extern DLLSERVER_API bool checkEmbeddedWorkUnitXML(ILoadedDllEntry *dll);
+extern DLLSERVER_API bool getResourceXMLFromFile(const char *filename, const char *type, unsigned id, StringBuffer &xml);
+extern DLLSERVER_API bool getWorkunitXMLFromFile(const char *filename, StringBuffer &xml);
+extern DLLSERVER_API bool getSoapInfoXMLFromFile(const char *filename, StringBuffer &xml);
+
+extern DLLSERVER_API bool decompressResource(size32_t len, const void *data, StringBuffer &result);
+extern DLLSERVER_API void compressResource(MemoryBuffer & compressed, size32_t len, const void *data);
+
+class DLLSERVER_API SimplePluginCtx : implements IPluginContextEx
+{
+public:
+    virtual void * ctxMalloc(size_t size);
+    virtual void * ctxRealloc(void * _ptr, size_t size);
+    virtual void   ctxFree(void * _ptr);
+    virtual char * ctxStrdup(char * _ptr);
+    virtual int ctxGetPropInt(const char *propName, int defaultValue) const;
+    virtual const char *ctxQueryProp(const char *propName) const;
+};
+
+class DLLSERVER_API SafePluginMap : public CInterface
+{
+    IPluginContextEx * pluginCtx;
+    MapStringToMyClass<ILoadedDllEntry> map;
+    CriticalSection crit;
+    bool trace;
+public:
+    SafePluginMap(IPluginContextEx * _pluginCtx, bool _trace) 
+    : pluginCtx(_pluginCtx), map(true), trace(_trace)
+    {
+        assertex(pluginCtx);
+    }
+
+    ILoadedDllEntry *getPluginDll(const char *id, const char *version, bool checkVersion);
+    bool addPlugin(const char *path, const char *dllname);
+    void loadFromList(const char * pluginsList);
+    void loadFromDirectory(const char * pluginDirectory);
+};
+
+#endif // THORPLUGIN_HPP

common/environment/CMakeLists.txt

@@ -0,0 +1,55 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: environment 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for environment
+#####################################################
+
+project( environment ) 
+
+set (    SRCS 
+         dalienv.cpp 
+         environment.cpp 
+    )
+
+set (    INCLUDES
+         dalienv.hpp 
+         environment.hpp 
+    )
+
+include_directories ( 
+         ./../../common/remote 
+         ./../../system/mp 
+         ./../../system/include 
+         ./../../dali/base 
+         ./../../system/jlib 
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DENVIRONMENT_EXPORTS )
+HPCC_ADD_LIBRARY( environment SHARED ${SRCS} ${INCLUDES} )
+install ( TARGETS environment DESTINATION ${OSSDIR}/lib )
+target_link_libraries ( environment 
+         jlib 
+         mp 
+         remote 
+         dalibase 
+    )
+

common/environment/dalienv.cpp

@@ -0,0 +1,455 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "platform.h"
+
+#include "jlib.hpp"
+#include "jio.hpp"
+
+#include "jmutex.hpp"
+#include "jfile.hpp"
+#include "jptree.hpp"
+
+#include "dasds.hpp"
+#include "daclient.hpp"
+
+#include "environment.hpp"
+#include "dalienv.hpp"
+#include "rmtfile.hpp"
+
+static CBuildVersion _bv("$HeadURL: https://svn.br.seisint.com/ecl/trunk/common/environment/dalienv.cpp $ $Id: dalienv.cpp 62376 2011-02-04 21:59:58Z sort $");
+
+
+
+struct CIpInstance
+{
+    unsigned hash;
+    IpAddress ip;
+    CIpInstance(const IpAddress &_ip)
+        : ip(_ip)
+    {
+        hash = ip.iphash();
+    }
+    static unsigned getHash(const char *key)
+    {
+        return ((const IpAddress *)key)->iphash();
+    }
+
+    bool eq(const char *key)
+    {
+        return ((const IpAddress *)key)->ipequals(ip);
+    }
+};
+
+struct CIpPasswordInstance: public CIpInstance
+{
+    StringAttr password;
+    StringAttr user;
+    bool matched;
+    CIpPasswordInstance(const IpAddress &_ip)
+        : CIpInstance(_ip)
+    {
+        matched = false;
+    }
+
+    static void destroy(CIpPasswordInstance *i) { delete i; }
+    static CIpPasswordInstance *create(const char *key) { return new CIpPasswordInstance(*(const IpAddress *)key); }
+};
+
+struct CIpPasswordHashTable: public CMinHashTable<CIpPasswordInstance>
+{
+};
+
+
+struct CIpOsInstance: public CIpInstance
+{
+    EnvMachineOS            os;
+    CIpOsInstance(const IpAddress &key)
+        : CIpInstance(key)
+    {
+        os=MachineOsUnknown;
+    }
+
+    static void destroy(CIpOsInstance *i) { delete i; }
+    static CIpOsInstance *create(const char *key) { return new CIpOsInstance(*(const IpAddress *)key); }
+};
+
+class CIpOsHashTable: public CMinHashTable<CIpOsInstance>
+{
+};
+
+
+
+class SDSPasswordProvider : public CInterface, implements IPasswordProvider
+{
+public:
+    SDSPasswordProvider();
+    ~SDSPasswordProvider() { delete map; }
+    IMPLEMENT_IINTERFACE
+
+    virtual bool getPassword(const IpAddress & ip, StringBuffer & username, StringBuffer & password);
+    void clearCache()
+    {
+        delete map;
+        map = new CIpPasswordHashTable();
+    }
+
+protected:
+    //MORE: This cache needs to be invalidated at some point...
+    CIpPasswordHashTable    *map;
+    Owned<IConstEnvironment> env;
+    Mutex mutex;
+};
+
+SDSPasswordProvider::SDSPasswordProvider() 
+{
+    Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
+    env.setown(factory->openEnvironment());
+    map = new CIpPasswordHashTable();
+}
+
+bool SDSPasswordProvider::getPassword(const IpAddress & ip, StringBuffer & username, StringBuffer & password)
+{
+    synchronized procedure(mutex);
+    if (!env)
+        return false;
+    CIpPasswordInstance *match = map->find((const char *)&ip,false);
+    if (!match)
+    {
+        match = new CIpPasswordInstance(ip);
+
+        StringBuffer ipText;
+        ip.getIpText(ipText);
+        Owned<IConstMachineInfo> machine = env->getMachineByAddress(ipText.str());
+        if (machine)
+        {
+            Owned<IConstDomainInfo> domain = machine->getDomain();
+            if (domain)
+            {
+                SCMStringBuffer username;
+
+                StringAttrAdaptor strval(match->password);
+                domain->getAccountInfo(username, strval);
+
+                SCMStringBuffer domainname;
+                domain->getName(domainname);
+                match->user.set(domainname.s.append('\\').append(username.str()).str());
+                match->matched = true;
+            }
+        }
+        map->add(match);
+    }
+    username.append(match->user);
+    password.append(match->password);
+    return match->matched;
+}
+
+static CriticalSection passwordProviderCrit;
+static SDSPasswordProvider * passwordProvider;
+MODULE_INIT(INIT_PRIORITY_ENV_ENVIRONMENT)
+{
+    return true;
+}
+
+MODULE_EXIT()
+{
+    clearPasswordsFromSDS();
+}
+
+void __stdcall setPasswordsFromSDS()
+{
+    CriticalBlock block(passwordProviderCrit);
+    passwordProvider = new SDSPasswordProvider();
+    setPasswordProvider(passwordProvider);
+}
+
+void __stdcall resetPasswordsFromSDS()
+{
+    CriticalBlock block(passwordProviderCrit);
+    if (passwordProvider)
+        passwordProvider->clearCache();
+}
+
+void __stdcall clearPasswordsFromSDS()
+{
+    CriticalBlock block(passwordProviderCrit);
+    if (passwordProvider) {
+        setPasswordProvider(NULL);
+        passwordProvider->Release();
+        passwordProvider = NULL;
+    }
+}
+
+
+
+//---------------------------------------------------------------------------
+
+static CriticalSection ipcachesect;
+static CIpOsHashTable *ipToOsCache = NULL;
+
+EnvMachineOS queryOS(const IpAddress & ip)
+{
+    if (ip.isLocal()) { // we know!
+#ifdef _WIN32
+        return MachineOsW2K;
+#else
+        return MachineOsLinux;
+#endif
+    }
+    CriticalBlock block(ipcachesect);
+    EnvMachineOS ret = MachineOsUnknown;
+    if (!ipToOsCache) 
+        ipToOsCache = new CIpOsHashTable;
+    CIpOsInstance * match = ipToOsCache->find((const char *)&ip,false);
+    if (match) 
+        ret = match->os;
+    else {
+        Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
+        if (factory) {
+            Owned<IConstEnvironment> env = factory->openEnvironment();
+            if (env) {
+                StringBuffer ipText;
+                ip.getIpText(ipText);
+                Owned<IConstMachineInfo> machine = env->getMachineByAddress(ipText.str());
+                if (machine) 
+                    ret = machine->getOS();
+            }
+        }
+        if (ret==MachineOsUnknown) { // lets try asking dafilesrv
+            SocketEndpoint ep(0,ip);
+            switch (getDaliServixOs(ep)) { 
+              case DAFS_OSwindows: ret = MachineOsW2K; break;
+              case DAFS_OSlinux:     ret = MachineOsLinux; break;
+              case DAFS_OSsolaris: ret = MachineOsSolaris; break;
+            }
+        }
+        match = new CIpOsInstance(ip);
+        match->os = ret;
+        ipToOsCache->add(match);
+    }
+    return ret;
+}
+
+
+
+bool canAccessFilesDirectly(const IpAddress & ip)
+{
+    if (ip.isLocal()||ip.isNull())  // the isNull check is probably an error but saves time
+        return true;                // I think usually already checked, but another can't harm
+#ifdef _WIN32
+    EnvMachineOS os = queryOS(ip);
+    if (os==MachineOsW2K)
+        return true;
+    if ((os==MachineOsUnknown)&&!testDaliServixPresent(ip))
+        return true;    // maybe lucky if windows
+#endif
+    return false; 
+}
+
+bool canSpawnChildProcess(const IpAddress & ip)
+{
+    //MORE: This isn't the correct implementation, but at least the calls now have the 
+    //correct name.
+    return canAccessFilesDirectly(ip);
+}
+
+
+bool canAccessFilesDirectly(const char * ipText)
+{
+    IpAddress ip(ipText);
+    return canAccessFilesDirectly(ip);
+}
+
+bool canAccessFilesDirectly(const RemoteFilename & file)
+{
+    if (file.queryEndpoint().port!=0)
+        return false;
+#ifdef _WIN32
+    if (!file.isUnixPath()) // assume any windows path can be accessed using windows share
+        return true;
+#endif
+    return canAccessFilesDirectly(file.queryIP());
+}
+
+
+void setCanAccessDirectly(RemoteFilename & file)
+{
+    setCanAccessDirectly(file,canAccessFilesDirectly(file));
+}
+
+class CDaliEnvIntercept: public CInterface, implements IFileCreateHook
+{
+    bool active;
+    CriticalSection crit;
+public:
+    IMPLEMENT_IINTERFACE;
+    CDaliEnvIntercept() { active = false; }
+    virtual IFile * createIFile(const RemoteFilename & filename)
+    {
+        CriticalBlock block(crit);
+        if (active||!daliClientActive())
+            return NULL;
+        active = true;
+        IFile * ret;
+        if (canAccessFilesDirectly(filename)) 
+            ret = NULL;
+        else 
+            ret = createDaliServixFile(filename);   
+        active = false;
+        return ret;
+    }   
+} *DaliEnvIntercept;
+
+
+
+MODULE_INIT(INIT_PRIORITY_ENV_DALIENV)
+{
+    DaliEnvIntercept = new CDaliEnvIntercept;
+    addIFileCreateHook(DaliEnvIntercept);
+    return true;
+}
+
+MODULE_EXIT()
+{
+    removeIFileCreateHook(DaliEnvIntercept);
+    ::Release(DaliEnvIntercept);
+    delete ipToOsCache;
+    ipToOsCache = NULL;
+}
+
+//---------------------------------------------------------------------------
+
+const char * querySlaveExecutable(const char * keyName, const char * exeName, const char * version, const IpAddress &ip, StringBuffer &progpath, StringBuffer &workdir)
+{
+    Owned<IEnvironmentFactory> factory = getEnvironmentFactory();
+    Owned<IConstEnvironment> env = factory->openEnvironment();
+    StringBuffer addr;
+    ip.getIpText(addr);
+
+    StringBufferAdaptor spp(progpath);
+    StringBufferAdaptor swd(workdir);
+    if (!env || !env->getRunInfo(spp, swd, keyName, version, addr.str(), exeName)) {
+#ifdef _DEBUG
+        //printf("slave path not found\n");
+        progpath.append(exeName);
+#ifdef _WIN32
+        progpath.append(".exe");
+#endif
+#else
+        throw MakeStringException(1, "Could not find the location of the slave program %s for machine %s", keyName, addr.str());
+#endif
+    }
+    // on linux check that file exists where it is supposed to be 
+#ifndef _WIN32
+    if (progpath.length()) {
+        RemoteFilename rfn;
+        SocketEndpoint ep;
+        ep.ipset(ip);
+        rfn.setPath(ep,progpath.str());
+        Owned<IFile> file = createIFile(rfn); 
+        if (!file->exists())  {
+            WARNLOG("Could not find the the slave program %s for machine %s at %s", keyName, addr.str(), progpath.str());
+            throw MakeStringException(1, "Could not find the slave program %s for machine %s at %s", keyName, addr.str(), progpath.str());
+        }
+    }
+#endif
+    return progpath.str();
+}
+
+
+bool getRemoteRunInfo(const char * keyName, const char * exeName, const char * version, const IpAddress &ip, StringBuffer &progpath, StringBuffer &workdir, INode *remotedali, unsigned timeout)
+{
+    // use dafilesrv to work out OS
+    StringBuffer dalis;
+    if (remotedali)
+        remotedali->endpoint().getUrlStr(dalis);
+    // first get machine by IP
+    StringBuffer ips;
+    ip.getIpText(ips);
+    StringBuffer xpath;
+    xpath.appendf("Environment/Hardware/Computer[@netAddress=\"%s\"]", ips.str());
+    Owned<IPropertyTreeIterator> iter = querySDS().getElementsRaw(xpath,remotedali,timeout);
+    if (!iter->first()) {
+        ERRLOG("Unable to find machine for %s on dali %s", ips.str(),dalis.str());
+        return false;
+    }
+    Owned<IPropertyTree> machine;
+    machine.setown(&iter->get());
+    const char *domainname = machine->queryProp("@domain");
+    if (!domainname||!*domainname) {
+        ERRLOG("Unable to find domain for %s on dali %s", ips.str(),dalis.str());
+        return false;
+    }
+    xpath.clear().appendf("Environment/Software/%s",keyName);
+    if (version)
+        xpath.appendf("[@version='%s']",version);
+    xpath.append("/Instance");
+    iter.clear();
+    iter.setown(querySDS().getElementsRaw(xpath,remotedali,timeout));
+    ForEach(*iter) {
+        IPropertyTree *inst = &iter->query();
+        const char * comp = inst->queryProp("@computer");
+        if (comp) {
+            xpath.clear().appendf("Environment/Hardware/Computer[@name=\"%s\"]", comp);
+            Owned<IPropertyTreeIterator> iter2 = querySDS().getElementsRaw(xpath,remotedali,timeout);
+            if (iter2->first()) {
+                Owned<IPropertyTree> machine2= &iter2->get();
+                const char *domainname2 = machine2->queryProp("@domain");
+                const char *ips2 = machine2->queryProp("@netAddress");
+                if (ips2&&*ips2&&domainname2&&*domainname2&&(strcmp(domainname,domainname2)==0)) {
+                    bool appendexe;
+                    char psep;
+                    SocketEndpoint ep(ips2);
+                    if (getDaliServixOs(ep)==DAFS_OSwindows) {
+                        psep = '\\';
+                        appendexe = true;
+                    }
+                    else {
+                        psep = '/';
+                        appendexe = false;
+                    }
+                    StringBuffer tmp;
+                    const char *program = inst->queryProp("@program"); // if program specified assume absolute 
+                    if (!program||!*program) {
+                        tmp.append(psep).append(psep).append(ips2).append(psep).append(inst->queryProp("@directory")).append(psep).append(exeName);
+                        size32_t l = strlen(exeName);
+                        if (appendexe&&((l<5)||(stricmp(exeName+l-4,".exe")!=0)))
+                            tmp.append(".exe");
+                        program = tmp.str();
+                    }
+                    progpath.set(program);
+                    const char *workd = inst->queryProp("@workdir"); // if program specified assume absolute 
+                    workdir.set(workd?workd:"");
+                    return true;
+                }
+            }
+        }
+    }
+    return false;
+}
+
+bool envGetConfigurationDirectory(const char *category, const char *component,const char *instance, StringBuffer &dirout)
+{
+    SessionId sessid = myProcessSession();
+    if (!sessid)
+        return false;
+    Owned<IRemoteConnection> conn = querySDS().connect("/Environment/Software/Directories",sessid, 0, 10000);
+    if (conn) 
+        return getConfigurationDirectory(conn->queryRoot(),category,component,instance,dirout);
+    return false;
+}

common/environment/dalienv.hpp

@@ -0,0 +1,55 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef DALIENV_HPP
+#define DALIENV_HPP
+
+#ifdef _WIN32
+#ifdef ENVIRONMENT_EXPORTS
+#define ENVIRONMENT_API __declspec(dllexport)
+#else
+#define ENVIRONMENT_API __declspec(dllimport)
+#endif
+#else
+#define ENVIRONMENT_API
+#endif
+
+#include "environment.hpp"
+
+interface IFile;
+class RemoteFilename;
+
+extern ENVIRONMENT_API EnvMachineOS queryOS(const IpAddress & ip);
+extern "C" ENVIRONMENT_API void __stdcall setPasswordsFromSDS();
+extern "C" ENVIRONMENT_API void __stdcall resetPasswordsFromSDS();
+extern "C" ENVIRONMENT_API void __stdcall clearPasswordsFromSDS();
+
+
+extern ENVIRONMENT_API bool canAccessFilesDirectly(const RemoteFilename & file);
+extern ENVIRONMENT_API bool canAccessFilesDirectly(const IpAddress & ip);
+extern ENVIRONMENT_API bool canAccessFilesDirectly(const char * ipText);
+extern ENVIRONMENT_API const char * querySlaveExecutable(const char * keyName, const char * exeName, const char * version, const IpAddress &ip, StringBuffer &progpath, StringBuffer &workdir);
+extern ENVIRONMENT_API void setCanAccessDirectly(RemoteFilename & file);
+
+extern ENVIRONMENT_API bool canSpawnChildProcess(const IpAddress & ip);
+
+extern ENVIRONMENT_API bool getRemoteRunInfo(const char * keyName, const char * exeName, const char * version, const IpAddress &ip, StringBuffer &progpath, StringBuffer &workdir,INode *remotedali, unsigned timeout);
+
+extern ENVIRONMENT_API bool envGetConfigurationDirectory(const char *category, const char *component,const char *instance, StringBuffer &dirout);
+
+#endif

common/environment/environment.hpp

@@ -0,0 +1,154 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+// *** Include file generated by HIDL Version 1.3 from environment.scm ***
+// *** Not to be hand edited (changes will be lost on re-generation) ***
+
+#ifndef environment_SCM_INCL
+#define environment_SCM_INCL
+
+#include "jiface.hpp"
+
+#include "dasubs.hpp"
+
+#ifdef WIN32
+    #ifdef ENVIRONMENT_EXPORTS
+        #define ENVIRONMENT_API __declspec(dllexport)
+    #else
+        #define ENVIRONMENT_API __declspec(dllimport)
+    #endif
+#else
+    #define ENVIRONMENT_API
+#endif
+
+interface IPropertyTree;   // Not yet SCM-compliant
+interface IEnvironment;    // Forward reference
+interface ISDSSubscription;// Forward reference
+
+interface IConstEnvBase : extends IInterface
+{
+    virtual IStringVal & getXML(IStringVal & str) const = 0;
+    virtual IStringVal & getName(IStringVal & str) const = 0;
+    virtual IPropertyTree & getPTree() const = 0;
+};
+
+
+interface IConstDomainInfo : extends IConstEnvBase
+{
+    virtual void getAccountInfo(IStringVal & name, IStringVal & pw) const = 0;
+    virtual void getSnmpSecurityString(IStringVal & securityString) const = 0;
+    virtual void getSSHAccountInfo(IStringVal & name, IStringVal & sshKeyFile, IStringVal& sshKeyPassphrase) const = 0;
+};
+
+
+enum EnvMachineState
+{
+    MachineStateAvailable = 0,
+    MachineStateUnavailable = 1,
+    MachineStateUnknown = 2
+};
+
+
+
+enum EnvMachineOS
+{
+    MachineOsW2K = 0,
+    MachineOsSolaris = 1,
+    MachineOsLinux = 2,
+    MachineOsUnknown = 3,
+    MachineOsSize = 4
+};
+
+
+
+interface IConstComputerTypeInfo : extends IConstEnvBase
+{
+    virtual EnvMachineOS getOS() const = 0;
+    virtual unsigned getNicSpeedMbitSec() const = 0;
+};
+
+
+
+interface IConstMachineInfo : extends IConstEnvBase
+{
+    virtual IConstDomainInfo * getDomain() const = 0;
+    virtual IStringVal & getNetAddress(IStringVal & str) const = 0;
+    virtual unsigned getNicSpeedMbitSec() const = 0;
+    virtual IStringVal & getDescription(IStringVal & str) const = 0;
+    virtual EnvMachineOS getOS() const = 0;
+    virtual EnvMachineState getState() const = 0;
+};
+
+
+interface IConstInstanceInfo : extends IConstEnvBase
+{
+    virtual IConstMachineInfo * getMachine() const = 0;
+    virtual IStringVal & getEndPoint(IStringVal & str) const = 0;
+    virtual unsigned getPort() const = 0;
+    virtual IStringVal & getExecutableDirectory(IStringVal & str) const = 0;
+    virtual bool getRunInfo(IStringVal & progpath, IStringVal & workdir, const char * defaultprogname) const = 0;
+};
+
+
+interface IConstEnvironment : extends IConstEnvBase
+{
+    virtual IConstDomainInfo * getDomain(const char * name) const = 0;
+    virtual IConstMachineInfo * getMachine(const char * name) const = 0;
+    virtual IConstMachineInfo * getMachineByAddress(const char * netaddress) const = 0;
+    virtual IConstInstanceInfo * getInstance(const char * type, const char * version, const char * domain) const = 0;
+    virtual IConstComputerTypeInfo * getComputerType(const char * name) const = 0;
+    virtual bool getRunInfo(IStringVal & path, IStringVal & dir, const char * type, const char * version, const char * machineaddr, const char * defaultprogname) const = 0;
+    virtual IEnvironment & lock() const = 0;
+    virtual bool isConstEnvironment() const = 0;
+    virtual void clearCache() = 0;
+};
+
+
+interface IEnvironment : extends IConstEnvironment
+{
+    virtual void commit() = 0;
+    virtual void rollback() = 0;
+    virtual void setXML(const char * (null)) = 0;
+    virtual void preload() = 0;
+};
+
+
+interface IEnvironmentFactory : extends IInterface
+{
+    virtual IConstEnvironment * openEnvironment() = 0;
+    virtual IConstEnvironment * createEnvironmentByFile(const char * environmentConfFile, const char * environmentXMLFile) = 0;
+    virtual IConstEnvironment * openEnvironmentByFile() = 0;
+    virtual const char * getEnvironmentConf() = 0;
+    virtual IEnvironment * updateEnvironment() = 0;
+    virtual IEnvironment * loadLocalEnvironmentFile(const char * filename) = 0;
+    virtual IEnvironment * loadLocalEnvironment(const char * xml) = 0;
+    virtual SubscriptionId subscribe(ISDSSubscription * pSubHandler) = 0;
+    virtual void unsubscribe(SubscriptionId id) = 0;
+    virtual void validateCache() = 0;
+};
+
+
+class StringBuffer;
+extern "C" ENVIRONMENT_API IEnvironmentFactory * getEnvironmentFactory();
+extern "C" ENVIRONMENT_API void closeEnvironment();
+
+
+
+
+#endif // _environment_SCM_INCL
+//end

common/environment/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/environment</title>
+
+    <para>
+        The common/environment directory contains the sources for the common/environment library.
+    </para>
+</section>

common/fileview2/CMakeLists.txt

@@ -0,0 +1,133 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# File      : CMakeLists.txt
+# Component: fileview2 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for fileview2 and fvserver
+#####################################################
+
+cmake_policy( SET CMP0011 NEW )
+
+project( fileview2 ) 
+
+set (    SRCS 
+         fvdisksource.cpp 
+         fvdsremote.cpp 
+         fvidxsource.cpp 
+         fvquerysource.cpp 
+         fvrelate.cpp 
+         fvresultset.cpp 
+         fvsource.cpp 
+         fvtransform.cpp 
+         fvwugen.cpp 
+         fvwusource.cpp 
+         
+         fvdisksource.ipp 
+         fvdsremote.ipp 
+         fvidxsource.ipp 
+         fvquerysource.ipp 
+         fvrelate.ipp 
+         fvresultset.ipp 
+         fvsource.ipp 
+         fvtransform.ipp
+         fvwugen.ipp
+         fvwusource.ipp 
+         
+                 fileview.hpp 
+         fvdatasource.hpp 
+         fverror.hpp 
+         fvrelate.hpp 
+         fvwugen.hpp 
+                 
+         sourcedoc.xml
+    )
+
+include_directories ( 
+         ./../../system/mp 
+         ./../../system/jhtree 
+         ./../dllserver 
+         ./../../rtl/eclrtl 
+         ./../../rtl/include 
+         ./../../rtl/nbcd 
+         ./../../system/include 
+         ./../../system/security/shared
+         ./../../dali/base 
+         ./../../system/jlib 
+         ./../deftype 
+         ./../../ecl/hql 
+         ./../remote 
+         ./../../common/workunit 
+         ./../../common/environment 
+    )
+
+HPCC_ADD_LIBRARY( fileview2 SHARED ${SRCS} )
+set_target_properties(fileview2 PROPERTIES 
+    COMPILE_FLAGS -D_USRDLL
+    DEFINE_SYMBOL FILEVIEW2_EXPORTS )
+install ( TARGETS fileview2 DESTINATION ${OSSDIR}/lib )
+target_link_libraries ( fileview2 
+         jlib 
+         mp 
+         hrpc 
+         remote 
+         nbcd 
+         eclrtl 
+         jhtree 
+         dalibase 
+         environment 
+         dllserver 
+         deftype 
+         workunit 
+         thorhelper
+         hql 
+    )
+
+set (    SRCS 
+         fvserver.cpp 
+    )
+
+ADD_DEFINITIONS ( -D_CONSOLE )
+
+add_executable ( fvserver ${SRCS} )
+set_target_properties(fvserver PROPERTIES 
+    COMPILE_FLAGS -D_CONSOLE
+    )
+#install ( TARGETS fvserver DESTINATION ${OSSDIR}/bin ) #not used at present
+
+target_link_libraries ( fvserver
+         jlib 
+         mp 
+         hrpc 
+         remote 
+         dalibase 
+         environment 
+         dllserver 
+         jhtree 
+         nbcd 
+         eclrtl 
+         deftype 
+         workunit 
+         hql 
+         fileview2 
+    )
+
+
+

common/fileview2/fileview.hpp

@@ -0,0 +1,238 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FILEVIEW_INCL
+#define FILEVIEW_INCL
+
+#ifdef _WIN32
+    #ifdef FILEVIEW2_EXPORTS
+        #define FILEVIEW_API __declspec(dllexport)
+    #else
+        #define FILEVIEW_API __declspec(dllimport)
+    #endif
+#else
+    #define FILEVIEW_API
+#endif
+
+#include "workunit.hpp"
+
+#define UNKNOWN_NUM_ROWS    (I64C(0x7FFFFFFFFFFFFFFF))
+
+enum DisplayType
+{
+    TypeBoolean = 0,
+    TypeInteger = 1,
+    TypeUnsignedInteger = 2,
+    TypeReal = 3,
+    TypeString = 4,
+    TypeData = 5,
+    TypeUnicode = 6,
+    TypeUnknown = 7,
+    TypeBeginIfBlock = 8,
+    TypeEndIfBlock = 9,
+    TypeBeginRecord = 10,
+    TypeEndRecord = 11,
+    TypeSet = 12,
+    TypeDataset = 13
+};
+
+
+
+enum ResultSetType
+{
+    TYPE_FORWARD_ONLY = 0,
+    TYPE_SCROLL_INSENSITIVE = 1,
+    TYPE_SCROLL_SENSITIVE = 2
+};
+
+
+
+
+interface IResultSetMetaData : extends IInterface
+{
+    virtual IResultSetMetaData * getChildMeta(int column) const = 0;
+    virtual int getColumnCount() const = 0;
+    virtual DisplayType getColumnDisplayType(int column) const = 0;
+    virtual IStringVal & getColumnLabel(IStringVal & s, int column) const = 0;
+    virtual IStringVal & getColumnEclType(IStringVal & s, int column) const = 0;
+    virtual IStringVal & getColumnXmlType(IStringVal & s, int column) const = 0;
+    virtual bool isSigned(int column) const = 0;
+    virtual bool isEBCDIC(int column) const = 0;
+    virtual bool isBigEndian(int column) const = 0;
+    virtual unsigned getColumnRawType(int column) const = 0;
+    virtual unsigned getColumnRawSize(int column) const = 0;
+    virtual IStringVal & getXmlSchema(IStringVal & s, bool addHeader) const = 0;
+    virtual unsigned getNumKeyedColumns() const = 0;
+    virtual IStringVal & getXmlXPathSchema(IStringVal & str, bool addHeader) const = 0;
+    virtual bool hasGetTranslation(int column) const = 0;
+    virtual bool hasSetTranslation(int column) const = 0;
+    virtual IStringVal & getNaturalColumnLabel(IStringVal & s, int column) const = 0;
+    virtual bool isVirtual(int column) const = 0;
+};
+
+
+typedef double xdouble;
+interface IResultSet : extends IInterface
+{
+    virtual bool absolute(__int64 row) = 0;
+    virtual void afterLast() = 0;
+    virtual void beforeFirst() = 0;
+    virtual int findColumn(const char * columnName) const = 0;
+    virtual bool first() = 0;
+    virtual bool getBoolean(int columnIndex) = 0;
+    virtual IDataVal & getBytes(IDataVal & d, int columnIndex) = 0;
+    virtual xdouble getDouble(int columnIndex) = 0;
+    virtual int getFetchSize() const = 0;
+    virtual __int64 getInt(int columnIndex) = 0;
+    virtual int getType() = 0;
+    virtual const IResultSetMetaData & getMetaData() const = 0;
+    virtual __int64 getNumRows() const = 0;
+    virtual IDataVal & getRaw(IDataVal & d, int columnIndex) = 0;
+    virtual IDataVal & getRawRow(IDataVal & d) = 0;
+    virtual IStringVal & getString(IStringVal & ret, int columnIndex) = 0;
+    virtual bool isAfterLast() const = 0;
+    virtual bool isBeforeFirst() const = 0;
+    virtual bool isFirst() const = 0;
+    virtual bool isLast() const = 0;
+    virtual bool isNull(int columnIndex) const = 0;
+    virtual bool last() = 0;
+    virtual bool next() = 0;
+    virtual bool previous() = 0;
+    virtual bool relative(__int64 rows) = 0;
+    virtual void setFetchSize(int rows) = 0;
+    virtual bool supportsRandomSeek() const = 0;
+    virtual IStringVal & getDisplayText(IStringVal & ret, int columnIndex) = 0;
+    virtual void beginAccess() = 0;
+    virtual void endAccess() = 0;
+    virtual IStringVal & getXml(IStringVal & ret, int columnIndex) = 0;
+};
+
+
+
+interface INewResultSet;
+interface IResultSetCursor : extends IInterface
+{
+    virtual bool absolute(__int64 row) = 0;
+    virtual void afterLast() = 0;
+    virtual void beforeFirst() = 0;
+    virtual bool fetch(__int64 fileoffset) = 0;
+    virtual bool first() = 0;
+    virtual bool getBoolean(int columnIndex) = 0;
+    virtual IDataVal & getBytes(IDataVal & d, int columnIndex) = 0;
+    virtual xdouble getDouble(int columnIndex) = 0;
+    virtual int getFetchSize() const = 0;
+    virtual IResultSetCursor * getChildren(int columnIndex) const = 0;
+    virtual bool getIsAll(int columnIndex) const = 0;
+    virtual __int64 getInt(int columnIndex) = 0;
+    virtual IDataVal & getRaw(IDataVal & d, int columnIndex) = 0;
+    virtual IDataVal & getRawRow(IDataVal & d) = 0;
+    virtual IStringVal & getString(IStringVal & ret, int columnIndex) = 0;
+    virtual bool isAfterLast() const = 0;
+    virtual bool isBeforeFirst() const = 0;
+    virtual bool isFirst() const = 0;
+    virtual bool isLast() const = 0;
+    virtual bool isNull(int columnIndex) const = 0;
+    virtual bool isValid() const = 0;
+    virtual bool last() = 0;
+    virtual bool next() = 0;
+    virtual bool previous() = 0;
+    virtual INewResultSet * queryResultSet() = 0;
+    virtual bool relative(__int64 rows) = 0;
+    virtual void serialize(IDataVal & d) = 0;
+    virtual IStringVal & getDisplayText(IStringVal & ret, int columnIndex) = 0;
+    virtual IStringVal & getXml(IStringVal & ret, int columnIndex) = 0;
+    virtual IStringVal & getXmlRow(IStringVal & ret) = 0;
+    virtual __int64 getNumRows() const = 0;
+};
+
+
+interface INewResultSet;
+
+interface IResultSetFilter : extends IInterface
+{
+    virtual void clearFilter(unsigned columnIndex) = 0;
+    virtual void addFilter(unsigned columnIndex, const char * value) = 0;
+    virtual void addFilter(unsigned columnIndex, unsigned length, const char * utf8Value) = 0;
+    virtual void addNaturalFilter(unsigned columnIndex, unsigned length, const char * utf8Value) = 0;
+    virtual void clearFilters() = 0;
+};
+
+
+interface IFilteredResultSet : extends IResultSetFilter
+{
+    virtual INewResultSet * create() = 0;
+};
+
+
+
+//Following interface is stateless, and can be shared...
+interface INewResultSet : extends IInterface
+{
+    virtual IResultSetCursor * createCursor() = 0;
+    virtual IResultSetCursor * createCursor(IDataVal & buffer) = 0;
+    virtual IFilteredResultSet * createFiltered() = 0;
+    virtual IResultSetCursor * createSortedCursor(unsigned column, bool descend) = 0;
+    virtual const IResultSetMetaData & getMetaData() const = 0;
+    virtual __int64 getNumRows() const = 0;
+    virtual bool supportsRandomSeek() const = 0;
+};
+
+
+
+interface IResultSetFactory : extends IInterface
+{
+    virtual IResultSet * createResultSet(IConstWUResult * wuResult, const char * wuid) = 0;
+    virtual IResultSet * createFileResultSet(const char * logicalFile, const char * cluster) = 0;
+    virtual INewResultSet * createNewResultSet(IConstWUResult * wuResult, const char * wuid) = 0;
+    virtual INewResultSet * createNewFileResultSet(const char * logicalFile, const char * cluster) = 0;
+    virtual INewResultSet * createNewResultSet(const char * wuid, unsigned sequence, const char * name) = 0;
+    virtual INewResultSet * createNewFileResultSet(const char * logicalFile) = 0;
+    virtual IResultSetMetaData * createResultSetMeta(IConstWUResult * wuResult) = 0;
+    virtual IResultSetMetaData * createResultSetMeta(const char * wuid, unsigned sequence, const char * name) = 0;
+};
+
+
+//provided to wrap the exceptions for clarion....
+extern "C" FILEVIEW_API IResultSet* createResultSet(IResultSetFactory & factory, IStringVal & error, IConstWUResult * wuResult, const char * wuid);
+extern "C" FILEVIEW_API IResultSet* createFileResultSet(IResultSetFactory & factory, IStringVal & error, const char * logicalFile, const char * queue = NULL, const char * cluster = NULL);
+extern "C" FILEVIEW_API INewResultSet* createNewResultSet(IResultSetFactory & factory, IStringVal & error, IConstWUResult * wuResult, const char * wuid);
+extern "C" FILEVIEW_API INewResultSet* createNewFileResultSet(IResultSetFactory & factory, IStringVal & error, const char * logicalFile, const char * queue, const char * cluster);
+extern "C" FILEVIEW_API INewResultSet* createNewResultSetSeqName(IResultSetFactory & factory, IStringVal & error, const char * wuid, unsigned sequence, const char * name);
+
+
+extern "C" FILEVIEW_API IResultSetFactory * getResultSetFactory(const char * username, const char * password);
+extern "C" FILEVIEW_API IResultSetFactory * getSecResultSetFactory(ISecManager &secmgr, ISecUser &secuser);
+
+extern "C" FILEVIEW_API IResultSetFactory * getRemoteResultSetFactory(const char * remoteServer, const char * username, const char * password);
+extern "C" FILEVIEW_API IResultSetFactory * getSecRemoteResultSetFactory(const char * remoteServer, ISecManager &secmgr, ISecUser &secuser);
+
+//Formatting applied remotely, so it can be accessed between different operating systems...
+extern "C" FILEVIEW_API IResultSetFactory * getRemoteResultSetFactory(const char * remoteServer, const char * username, const char * password);
+extern "C" FILEVIEW_API int findResultSetColumn(const INewResultSet * results, const char * columnName);
+
+extern "C" FILEVIEW_API unsigned getResultCursorXml(IStringVal & ret, IResultSetCursor * cursor, const char * name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
+extern "C" FILEVIEW_API unsigned getResultXml(IStringVal & ret, INewResultSet * cursor,  const char* name, unsigned start=0, unsigned count=0, const char * schemaName=NULL);
+
+extern "C" FILEVIEW_API unsigned getResultCursorBin(MemoryBuffer & ret, IResultSetCursor * cursor, unsigned start=0, unsigned count=0);
+extern "C" FILEVIEW_API unsigned getResultBin(MemoryBuffer & ret, INewResultSet * cursor, unsigned start=0, unsigned count=0);
+extern "C" FILEVIEW_API IStringVal& getFullWorkUnitResultsXML(const char *user, const char *pw, const IConstWorkUnit *wu, IStringVal &str, bool inclschema, WUExceptionSeverity minSeverity=ExceptionSeverityInformation);
+
+extern FILEVIEW_API void startRemoteDataSourceServer(const char * queue, const char * cluster);
+extern FILEVIEW_API void stopRemoteDataSourceServer();
+
+#endif

common/fileview2/fvdatasource.hpp

@@ -0,0 +1,95 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVDATASOURCE_HPP
+#define FVDATASOURCE_HPP
+
+#include "fileview.hpp"
+#include "deftype.hpp"
+
+enum
+{
+    FVFFnone        = 0,
+    FVFFbeginif     = 1,
+    FVFFendif       = 2,
+    FVFFbeginrecord = 3,
+    FVFFendrecord   = 4,
+    FVFFdataset     = 5,
+    FVFFset         = 6,
+    FVFFvirtual     = 7,
+};
+
+interface IFvDataSource;
+interface IFvDataSourceMetaData : extends IInterface
+{
+    virtual unsigned numColumns() const = 0;
+    virtual ITypeInfo * queryType(unsigned column) const = 0;
+    virtual const char * queryName(unsigned column) const = 0;
+    virtual const char * queryXPath(unsigned column) const = 0;
+    virtual void serialize(MemoryBuffer & buffer) const = 0;
+    virtual bool supportsRandomSeek() const = 0;
+    virtual unsigned queryFieldFlags(unsigned column) const = 0;
+    virtual IFvDataSourceMetaData * queryChildMeta(unsigned column) const = 0;
+    virtual IFvDataSource * createChildDataSource(unsigned column, unsigned len, const void * data) = 0;
+    virtual unsigned numKeyedColumns() const = 0;
+    
+    inline bool isVirtual(unsigned column) const { return queryFieldFlags(column) == FVFFvirtual; }
+};
+
+IFvDataSourceMetaData * deserializeDataSourceMeta(MemoryBuffer & in);
+
+#define BEFORE_FIRST_ROW    ((__int64)-1)
+#define AFTER_LAST_ROW      ((__int64)-2)
+
+interface IFvDataSource : extends IInterface
+{
+    virtual IFvDataSourceMetaData * queryMetaData() = 0;
+    virtual void applyFilter() { }
+    virtual IFvDataSource * cloneForFilter() { return NULL; }
+    virtual __int64 numRows(bool force = false) = 0;
+    virtual bool fetchRow(MemoryBuffer & out, __int64 offset) = 0;
+    virtual bool fetchRawRow(MemoryBuffer & out, __int64 offset) = 0;
+    virtual bool getRow(MemoryBuffer & out, __int64 row) = 0;
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row) = 0;
+    virtual bool addFilter(unsigned offset, unsigned matchLen, unsigned len, const void * data) { return false; }       // more: support ranges
+    virtual void onClose() = 0;
+    virtual void onOpen() = 0;
+    virtual bool isIndex() = 0;
+};
+
+//not public outside of the dll
+class ADataSource : public CInterface, implements IFvDataSource
+{
+public:
+    IMPLEMENT_IINTERFACE
+
+    virtual bool init() = 0;
+};
+
+IFvDataSource * createDataSource(IConstWUResult * wuResult, const char * wuid, const char * username, const char * password);
+IFvDataSource * createFileDataSource(IDistributedFile * df, const char * logicalName, const char * cluster, const char * username, const char * password);
+IFvDataSource * createFileDataSource(const char * logicalName, const char * cluster, const char * username, const char * password);
+IFvDataSource * createRemoteDataSource(const SocketEndpoint & ep, const char * username, const char * password, const char * wuid, unsigned sequence, const char * name);
+IFvDataSource * createRemoteFileDataSource(const SocketEndpoint & ep, const char * username, const char * password, const char * logicalName);
+IFvDataSourceMetaData * createMetaData(IConstWUResult * wuResult);
+
+
+IConstWUResult * resolveResult(const char * wuid, unsigned sequence, const char * name);
+IConstWUResult * secResolveResult(ISecManager &secmgr, ISecUser &secuser, const char * wuid, unsigned sequence, const char * name);
+
+#endif

common/fileview2/fvdisksource.cpp

@@ -0,0 +1,705 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "eclrtl.hpp"
+
+#include "hqlexpr.hpp"
+#include "hqlthql.hpp"
+#include "fvresultset.ipp"
+#include "fileview.hpp"
+#include "fvdisksource.ipp"
+#include "fvwugen.hpp"
+#include "fverror.hpp"
+#include "dasess.hpp"
+
+#define DEFAULT_MAX_CSV_SIZE    0x1100
+
+PhysicalFileInfo::PhysicalFileInfo()
+{
+    cachedPart = (unsigned)-1;
+    totalSize = 0;
+}
+
+
+offset_t getPartSize(IDistributedFilePart & part, unsigned copy)
+{
+    try
+    {
+        RemoteFilename rfn;
+        Owned<IFile> in = createIFile(part.getFilename(rfn,copy));
+        return in->size();
+    }
+    catch (IException * e)
+    {
+        e->Release();
+    }
+    return (offset_t) -1;
+}
+
+void PhysicalFileInfo::init(IDistributedFile * _df)
+{
+    df.set(_df);
+    totalSize = 0;
+    Owned<IDistributedFilePartIterator> iter = df->getIterator();
+    ForEach(*iter)
+    {
+        IDistributedFilePart & cur = iter->query();
+
+        offset_t partSize = cur.getFileSize(true, false);
+        if (partSize == -1)
+            partSize = getPartSize(cur, 0);
+        if (partSize == -1)
+            partSize = getPartSize(cur, 1);
+        if (partSize == -1)
+            partSize = 0x100000;        // force an error when the part is opened.
+        partSizes.append(partSize);
+        totalSize += partSize;
+    }
+}
+
+
+offset_t PhysicalFileInfo::getOptimizedOffset(offset_t offset, unsigned copyLength)
+{
+    offset_t newOffset = 0;
+    ForEachItemIn(idx, partSizes)
+    {
+        offset_t curSize = partSizes.item(idx);
+        if (offset < curSize)
+            return newOffset + ((offset) / copyLength) * copyLength;
+        newOffset += curSize;
+        offset -= curSize;
+    }
+    return newOffset;
+}
+
+bool PhysicalFileInfo::readData(MemoryBuffer & out, __int64 startOffset, size32_t copyLength)
+{
+    CriticalBlock procedure(cs);
+
+    offset_t chunkOffset = startOffset;
+    unsigned numParts = partSizes.ordinality();
+    unsigned part;
+    offset_t curPartLength;
+    if (isLocalFpos(startOffset))
+    {
+        part = getLocalFposPart(startOffset);
+        chunkOffset = getLocalFposOffset(startOffset);
+        if (part >= numParts)
+            return false;
+        curPartLength = partSizes.item(part);
+    }
+    else
+    {
+        for (part = 0; part < numParts; part++)
+        {
+            curPartLength = partSizes.item(part);
+            if (chunkOffset < curPartLength)
+                break;
+            chunkOffset -= curPartLength;
+        }
+    }
+
+    if (part == numParts)
+        return false;
+
+    bool isLast = false;
+    if (chunkOffset + copyLength >= curPartLength)
+    {
+        copyLength = (size32_t)(curPartLength - chunkOffset);
+        isLast = true;
+    }
+
+    if (part != cachedPart)
+    {
+        cachedPart = (unsigned)-1;
+        cachedFile.clear();
+        cachedIO.clear();
+
+        Owned<IDistributedFilePart> dfp = df->getPart(part);
+        try
+        { 
+            RemoteFilename rfn;
+            cachedFile.setown(createIFile(dfp->getFilename(rfn)));
+            cachedIO.setown(cachedFile->open(IFOread));
+        }
+        catch (IException * e)
+        {
+            e->Release();
+        }
+        if (!cachedIO)
+        {
+            RemoteFilename rfn;
+            cachedFile.setown(createIFile(dfp->getFilename(rfn,1)));
+            cachedIO.setown(cachedFile->open(IFOread));
+            if (!cachedIO)
+            {
+                StringBuffer str;
+                throwError1(FVERR_FailedOpenFile, dfp->getPartName(str).str());
+                return false;
+            }
+        }
+        if (df->isCompressed())
+        {
+            cachedIO.setown(createCompressedFileReader(cachedIO));
+            if (!cachedIO)
+            {
+                StringBuffer str;
+                throwError1(FVERR_FailedOpenCompressedFile, dfp->getPartName(str).str());
+                return false;
+            }
+        }
+
+        cachedPart = part;
+    }
+
+    char * data = (char *)out.clear().reserve(copyLength);
+    unsigned numGot = cachedIO->read(chunkOffset, copyLength, data);
+    out.setLength(numGot);
+    return isLast;
+}
+
+
+void PhysicalFileInfo::close()
+{
+    cachedPart = (unsigned)-1;
+    cachedFile.clear();
+    cachedIO.clear();
+}
+
+//---------------------------------------------------------------------------
+
+DiskDataSource::DiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char* _username, const char* _password)
+{
+    logicalName.set(_logicalName);
+    diskRecord.set(_diskRecord);
+
+    Owned<IUserDescriptor> udesc;
+    if(_username != NULL && *_username != '\0')
+    {
+        udesc.setown(createUserDescriptor());
+        udesc->set(_username, _password);
+    }
+
+    df.setown(queryDistributedFileDirectory().lookup(logicalName, udesc.get()));
+}
+
+
+//---------------------------------------------------------------------------
+
+DirectDiskDataSource::DirectDiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char* _username, const char* _password) : DiskDataSource(_logicalName, _diskRecord, _username, _password)
+{
+}
+
+
+bool DirectDiskDataSource::init()
+{
+    if (!df)
+        return false;
+
+    IPropertyTree & properties = df->queryProperties();
+    const char * kind = properties.queryProp("@kind");
+    bool isGrouped =properties.getPropBool("@grouped");
+    if (kind && (stricmp(kind, "key") == 0))
+        throwError1(FVERR_CannotViewKey, logicalName.get());
+
+    //Need to assign the transformed record to meta
+    diskMeta.setown(new DataSourceMetaData(diskRecord, 0, true, isGrouped, 0));
+    if (!returnedMeta)
+    {
+        returnedMeta.set(diskMeta);
+        returnedRecordSize.set(returnedMeta);
+    }
+    if (!transformedMeta)
+        transformedMeta.set(returnedMeta);
+
+    addFileposition();
+    physical.init(df);
+    if (diskMeta->isFixedSize())
+    {
+        if (diskMeta->fixedSize() == 0)
+            throwError1(FVERR_ZeroSizeRecord, logicalName.get());
+        totalRows = physical.totalSize / diskMeta->fixedSize();
+    }
+    else if (properties.hasProp("@recordCount"))
+        totalRows = properties.getPropInt64("@recordCount");
+    else
+        totalRows = UNKNOWN_NUM_ROWS;
+
+    readBlockSize = 4 * diskMeta->getRecordSize(NULL);
+    if (readBlockSize < DISK_BLOCK_SIZE) readBlockSize = DISK_BLOCK_SIZE;
+    return true;
+}
+
+
+bool DirectDiskDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
+{
+    physical.readData(out, offset, returnedMeta->getMaxRecordSize());
+    if (out.length() == 0)
+        return false;
+    out.setLength(returnedMeta->getRecordSize(out.toByteArray()));
+    return true;
+}
+
+size32_t DirectDiskDataSource::getCopyLength()
+{
+    size32_t copyLength = readBlockSize;
+    if (returnedMeta->isFixedSize())
+    {
+        unsigned fixedSize = returnedMeta->fixedSize();
+        copyLength = (copyLength / fixedSize) * fixedSize;
+    }
+    return copyLength;
+}
+
+
+void DirectDiskDataSource::improveLocation(__int64 row, RowLocation & location)
+{
+    if (!returnedMeta->isFixedSize())
+        return;
+
+    //Align the row so the chunks don't overlap....
+    unsigned fixedSize = returnedMeta->fixedSize();
+    size32_t copyLength = getCopyLength();
+    location.bestOffset = physical.getOptimizedOffset(row * fixedSize, copyLength);
+    location.bestRow = location.bestOffset / fixedSize;
+}
+
+
+bool DirectDiskDataSource::loadBlock(__int64 startRow, offset_t startOffset)
+{
+    size32_t copyLength = getCopyLength();
+    MemoryBuffer temp;
+    bool isLast = physical.readData(temp, startOffset, copyLength);
+    if (temp.length() == 0)
+        return false;
+
+    RowBlock * rows;
+    if (returnedMeta->isFixedSize())
+        rows = new FixedRowBlock(temp, startRow, startOffset, returnedMeta->fixedSize());
+    else
+        rows = new VariableRowBlock(temp, startRow, startOffset, returnedRecordSize, isLast);
+    cache.addRowsOwn(rows);
+    return true;
+}
+
+void DirectDiskDataSource::onClose()    
+{ 
+    DiskDataSource::onClose();
+    if (openCount == 0)
+        physical.close();
+}
+
+//---------------------------------------------------------------------------
+
+UtfReader::UtfFormat getFormat(const char * format)
+{
+    if (memicmp(format, "utf", 3) == 0)
+    {
+        const char * tail = format + 3;
+        if (*tail == '-')
+            tail++;
+        if (stricmp(tail, "8N")==0)
+            return UtfReader::Utf8;
+        else if (stricmp(tail, "16BE")==0)
+            return UtfReader::Utf16be;
+        else if (stricmp(tail, "16LE")==0)
+            return UtfReader::Utf16le;
+        else if (stricmp(tail, "32BE")==0)
+            return UtfReader::Utf32be;
+        else if (stricmp(tail, "32LE")==0)
+            return UtfReader::Utf32le;
+        else
+            throwError1(FVERR_UnknownUTFFormat, format);
+    }
+    return UtfReader::Utf8;
+}
+
+enum { NONE, TERMINATOR };
+
+void CsvRecordSize::init(IDistributedFile * df)
+{
+    IPropertyTree * props = &df->queryProperties();
+    UtfReader::UtfFormat utfType = getFormat(props->queryProp("@format"));
+    switch (utfType)
+    {
+    case UtfReader::Utf16be:
+    case UtfReader::Utf16le:
+        unitSize = 2;
+        break;
+    case UtfReader::Utf32be:
+    case UtfReader::Utf32le:
+        unitSize = 4;
+        break;
+    default:
+        unitSize = 1;
+    }
+    maxRecordSize = props->getPropInt("@maxRecordSize", DEFAULT_MAX_CSV_SIZE);
+    const char * terminate = props->queryProp("@csvTerminate");
+    addUtfActionList(matcher, terminate ? terminate : "\\n,\\r\\n", TERMINATOR, NULL, utfType);
+}
+
+size32_t CsvRecordSize::getRecordLength(size32_t maxLength, const void * start, bool includeTerminator)
+{
+    //If we need more complicated processing...
+    const byte * cur = (const byte *)start;
+    const byte * end = (const byte *)start + maxLength;
+
+    while (cur != end)
+    {
+        unsigned matchLen;
+        unsigned match = matcher.getMatch(end-cur, (const char *)cur, matchLen);
+        switch (match & 255)
+        {
+        case NONE:
+            cur += unitSize;            // matchLen == 0;
+            break;
+        case TERMINATOR:
+            if (includeTerminator)
+                return cur + matchLen - (const byte *)start;
+            return cur - (const byte *)start;
+        }
+        cur += matchLen;
+    }
+
+    return end - (const byte *)start;
+
+}
+
+size32_t CsvRecordSize::getRecordSize(const void * start)
+{
+    if (!start) return maxRecordSize;
+    return getRecordLength(maxRecordSize, start, true);
+
+}
+
+size32_t CsvRecordSize::getRecordSize(unsigned maxLength, const void * start)
+{
+    if (!start) return maxRecordSize;
+    return getRecordLength(maxLength, start, true);
+
+}
+
+size32_t CsvRecordSize::getFixedSize() const
+{
+    return 0; // is variable
+}
+
+
+DirectCsvDiskDataSource::DirectCsvDiskDataSource(IDistributedFile * _df, const char * _format)
+{
+    df.set(_df);
+    isUnicode = (memicmp(_format, "utf", 3) == 0);
+    utfFormat = getFormat(_format);
+    returnedMeta.setown(new DataSourceMetaData(isUnicode ? type_unicode : type_string));
+    returnedRecordSize.set(&recordSizer);
+    transformedMeta.set(returnedMeta);
+    addFileposition();
+    IPropertyTree & properties = df->queryProperties();
+    if (properties.hasProp("@recordCount"))
+        totalRows = properties.getPropInt64("@recordCount");
+}
+
+bool DirectCsvDiskDataSource::init()
+{
+    physical.init(df);
+    recordSizer.init(df);
+
+    readBlockSize = 4 * recordSizer.getRecordSize(NULL);
+    if (readBlockSize < DISK_BLOCK_SIZE) readBlockSize = DISK_BLOCK_SIZE;
+    return true;
+}
+
+
+void DirectCsvDiskDataSource::copyRow(MemoryBuffer & out, size32_t length, const void * data)
+{
+    if (isUnicode)
+    {
+        unsigned offsetOfLength = out.length();
+        out.append(length);
+        convertUtf(out, UtfReader::Utf16le, length, data, utfFormat);
+        unsigned savedLength = out.length();
+        out.setWritePos(offsetOfLength);
+        out.append((unsigned) (savedLength - offsetOfLength - sizeof(unsigned))/2);
+        out.setWritePos(savedLength);
+    }
+    else
+    {
+        out.append(length);
+        out.append(length, data);
+    }
+}
+
+bool DirectCsvDiskDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
+{
+    MemoryBuffer temp;
+    physical.readData(temp, offset, recordSizer.getRecordSize(NULL));
+    if (temp.length() == 0)
+        return false;
+    unsigned realLength = recordSizer.getRecordSize(temp.length(), temp.toByteArray());
+    copyRow(out, realLength, temp.toByteArray());
+    return true;
+}
+
+
+bool DirectCsvDiskDataSource::loadBlock(__int64 startRow, offset_t startOffset)
+{
+    size32_t copyLength = readBlockSize;
+    MemoryBuffer temp;
+    bool isLast = physical.readData(temp, startOffset, copyLength);
+    if (temp.length() == 0)
+        return false;
+
+    RowBlock * rows = new VariableRowBlock(temp, startRow, startOffset, &recordSizer, isLast);
+    cache.addRowsOwn(rows);
+    return true;
+}
+
+
+bool DirectCsvDiskDataSource::getRow(MemoryBuffer & out, __int64 row)
+{
+    size32_t length;
+    const void * data;
+    unsigned __int64 offset = 0;
+    if (getRowData(row, length, data, offset))
+    {
+        //strip the end of line terminator from the length...
+        length = recordSizer.getRecordLength(length, data, false);
+        copyRow(out, length, data);
+        out.append(offset);
+        return true;
+    }
+    return false;
+}
+
+
+//---------------------------------------------------------------------------
+
+WorkunitDiskDataSource::WorkunitDiskDataSource(const char * _logicalName, IConstWUResult * _wuResult, const char * _wuid, const char * _username, const char * _password) : DirectDiskDataSource(_logicalName, NULL, _username, _password)
+{
+    wuid.set(_wuid);
+    wuResult.set(_wuResult);
+}
+
+
+bool WorkunitDiskDataSource::init()
+{
+    if (!setReturnedInfoFromResult())
+        return false;
+    diskRecord.set(returnedRecord);
+    return DirectDiskDataSource::init();
+}
+
+
+//---------------------------------------------------------------------------
+
+TranslatedDiskDataSource::TranslatedDiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char * _cluster, const char * _username, const char * _password)
+{
+    logicalName.set(_logicalName);
+    diskRecord.set(_diskRecord);
+    cluster.set(_cluster);
+    username.set(_username);
+    password.set(_password);
+    openCount = 0;
+}
+
+TranslatedDiskDataSource::~TranslatedDiskDataSource()
+{
+    if (helperWuid)
+    {
+        directSource.clear();
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+        factory->deleteWorkUnit(helperWuid);
+    }
+}
+
+bool TranslatedDiskDataSource::createHelperWU()
+{
+    OwnedHqlExpr browseWUcode = buildDiskOutputEcl(logicalName, diskRecord);
+    if (!browseWUcode)
+        return false;
+
+    // MORE: Where should we get these parameters from ????
+    StringAttr application("fileViewer");
+    StringAttr customerid("viewer");
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IWorkUnit> workunit = factory->createWorkUnit(NULL, application, username);
+    workunit->setUser(username);
+    workunit->setClusterName(cluster);  
+    workunit->setCustomerId(customerid);
+    workunit->setAction(WUActionCompile);
+
+    StringBuffer jobName;
+    jobName.append("FileView_for_").append(logicalName);
+    workunit->setJobName(jobName.str());
+
+    StringBuffer eclText;
+    toECL(browseWUcode, eclText, true);
+    Owned<IWUQuery> query = workunit->updateQuery();
+    query->setQueryText(eclText.str());
+    query->setQueryName(jobName.str());
+
+    workunit->setCompareMode(CompareModeOff);
+    StringAttrAdaptor xxx(helperWuid); workunit->getWuid(xxx);
+    return true;
+}
+
+
+bool TranslatedDiskDataSource::init()
+{
+    if (!createHelperWU() || !compileHelperWU())
+        return false;
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnit> wu = factory->openWorkUnit(helperWuid, false);
+    Owned<IConstWUResult> dataResult = wu->getResultBySequence(0);
+    directSource.setown(new WorkunitDiskDataSource(logicalName, dataResult, helperWuid, username.get(), password.get()));
+    return directSource->init();
+}
+
+
+bool TranslatedDiskDataSource::compileHelperWU()
+{
+    submitWorkUnit(helperWuid, username, password);
+    return waitForWorkUnitToCompile(helperWuid);
+}
+
+//---------------------------------------------------------------------------
+
+IndirectDiskDataSource::IndirectDiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char * _cluster, const char * _username, const char * _password) : DiskDataSource(_logicalName, _diskRecord, _username, _password)
+{
+    cluster.set(_cluster);
+    username.set(_username);
+    password.set(_password);
+    extraFieldsSize = sizeof(offset_t) + sizeof(unsigned short);
+}
+
+IndirectDiskDataSource::~IndirectDiskDataSource()
+{
+    if (browseWuid)
+    {
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+        factory->deleteWorkUnit(browseWuid);
+    }
+}
+
+bool IndirectDiskDataSource::createBrowseWU()
+{
+    OwnedHqlExpr browseWUcode = buildDiskFileViewerEcl(logicalName, diskRecord);
+    if (!browseWUcode)
+        return false;
+    returnedRecord.set(browseWUcode->queryChild(0)->queryRecord());
+
+    // MORE: Where should we get these parameters from ????
+    StringAttr application("fileViewer");
+    StringAttr owner("fileViewer");
+    StringAttr customerid("viewer");
+
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IWorkUnit> workunit = factory->createWorkUnit(NULL, application, owner);
+    workunit->setUser(owner);
+    workunit->setClusterName(cluster);  
+    workunit->setCustomerId(customerid);
+
+    StringBuffer jobName;
+    jobName.append("FileView_for_").append(logicalName);
+    workunit->setJobName(jobName.str());
+
+    StringBuffer eclText;
+    toECL(browseWUcode, eclText, true);
+    Owned<IWUQuery> query = workunit->updateQuery();
+    query->setQueryText(eclText.str());
+    query->setQueryName(jobName.str());
+
+    workunit->setCompareMode(CompareModeOff);
+    StringAttrAdaptor xxx(browseWuid); workunit->getWuid(xxx);
+    return true;
+}
+
+
+bool IndirectDiskDataSource::init()
+{
+    if (!df)
+        return false;
+
+    if (!createBrowseWU())
+        return false;
+
+    //Need to assign the transformed record to meta
+    bool isGrouped = false;     // more not sure this is strictly true...
+    returnedMeta.setown(new DataSourceMetaData(returnedRecord, 2, true, isGrouped, 0));
+    transformedMeta.set(returnedMeta);
+    diskMeta.setown(new DataSourceMetaData(diskRecord, 0, true, isGrouped, 0));
+
+    totalSize = df->getFileSize(true,false);
+
+    if (diskMeta->isFixedSize())
+        totalRows = totalSize / diskMeta->fixedSize();
+    else
+        totalRows = UNKNOWN_NUM_ROWS;
+    return true;
+}
+
+
+bool IndirectDiskDataSource::loadBlock(__int64 startRow, offset_t startOffset)
+{
+    MemoryBuffer temp;
+
+    //enter scope....>
+    {
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+        Owned<IWorkUnit> wu = factory->updateWorkUnit(browseWuid);
+        Owned<IWUResult> lower = wu->updateVariableByName(LOWER_LIMIT_ID);
+        lower->setResultInt(startOffset);
+        lower->setResultStatus(ResultStatusSupplied);
+
+        Owned<IWUResult> dataResult = wu->updateResultBySequence(0);
+        dataResult->setResultRaw(0, NULL, ResultFormatRaw);
+        dataResult->setResultStatus(ResultStatusUndefined);
+        wu->clearExceptions();
+        if (wu->getState() != WUStateUnknown)
+            wu->setState(WUStateCompiled);
+
+        //Owned<IWUResult> count = wu->updateVariableByName(RECORD_LIMIT_ID);
+        //count->setResultInt64(fetchSize);
+    }
+
+    //Resubmit the query...
+    submitWorkUnit(browseWuid, username, password);
+    WUState finalState = waitForWorkUnitToComplete(browseWuid, -1, true);
+    if(!((finalState == WUStateCompleted) || (finalState == WUStateWait)))
+        return false;
+
+    //Now extract the results...
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnit> wu = factory->openWorkUnit(browseWuid, false);
+    Owned<IConstWUResult> dataResult = wu->getResultBySequence(0);
+    MemoryBuffer2IDataVal xxx(temp); dataResult->getResultRaw(xxx, NULL, NULL);
+
+    if (temp.length() == 0)
+        return false;
+
+    RowBlock * rows;
+    if (returnedMeta->isFixedSize())
+        rows = new FilePosFixedRowBlock(temp, startRow, startOffset, returnedMeta->fixedSize());
+    else
+        rows = new FilePosVariableRowBlock(temp, startRow, startOffset, returnedMeta, true);
+    cache.addRowsOwn(rows);
+    return true;
+}

common/fileview2/fvdisksource.ipp

@@ -0,0 +1,196 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVDISKSOURCE_IPP
+#define FVDISKSOURCE_IPP
+
+#include "junicode.hpp"
+#include "fvdatasource.hpp"
+#include "dllserver.hpp"
+#include "hqlexpr.hpp"
+#include "eclhelper.hpp"
+
+#include "fvsource.ipp"
+#include "dadfs.hpp"
+
+
+class PhysicalFileInfo
+{
+public:
+    PhysicalFileInfo();
+    
+    void close();
+    offset_t getOptimizedOffset(offset_t offset, unsigned copyLength);
+    void init(IDistributedFile * _df);
+    bool readData(MemoryBuffer & out, __int64 offset, size32_t length);
+
+public:
+    Owned<IDistributedFile> df;
+
+    CriticalSection cs;
+    unsigned __int64 totalSize;
+    unsigned cachedPart;
+    OwnedIFile cachedFile;
+    OwnedIFileIO cachedIO;
+    Int64Array partSizes;
+};
+
+class DiskDataSource : public PagedDataSource
+{
+public:
+    DiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char* _username, const char* _password);
+
+protected:
+    StringAttr logicalName;
+    Owned<DataSourceMetaData> diskMeta;
+    HqlExprAttr diskRecord;
+    Owned<IDistributedFile> df;
+};
+
+
+class DirectDiskDataSource : public DiskDataSource
+{
+public:
+    DirectDiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char* _username, const char* _password);
+
+    virtual bool init();
+    virtual void onClose();
+    virtual bool fetchRowData(MemoryBuffer & out, __int64 offset);
+    virtual bool isIndex() { return false; }
+
+protected:
+    size32_t getCopyLength();
+    virtual bool loadBlock(__int64 startRow, offset_t startOffset);
+    void improveLocation(__int64 row, RowLocation & location);
+
+protected:
+    PhysicalFileInfo physical;
+    size32_t readBlockSize;
+};
+
+
+class CsvRecordSize : public CInterface, implements IRecordSizeEx
+{
+public:
+    IMPLEMENT_IINTERFACE
+
+    void init(IDistributedFile * df);
+
+    virtual size32_t getRecordSize(const void *rec);
+    virtual size32_t getRecordSize(unsigned maxLength, const void *rec);
+    virtual size32_t getFixedSize() const;
+
+    size32_t getRecordLength(size32_t maxLength, const void * start, bool includeTerminator);
+
+
+protected:
+    StringMatcher matcher;
+    size32_t unitSize;
+    size32_t maxRecordSize;
+};
+    
+class DirectCsvDiskDataSource  : public PagedDataSource
+{
+public:
+    DirectCsvDiskDataSource(IDistributedFile * _df, const char * _format);
+
+    virtual bool init();
+    virtual bool isIndex() { return false; }
+    virtual bool fetchRowData(MemoryBuffer & out, __int64 offset);
+    virtual bool loadBlock(__int64 startRow, offset_t startOffset);
+
+    virtual bool getRow(MemoryBuffer & out, __int64 row);
+
+protected:
+    void copyRow(MemoryBuffer & out, size32_t length, const void * data);
+
+protected:
+    Owned<IDistributedFile> df;
+    bool isUnicode;
+    UtfReader::UtfFormat utfFormat;
+    PhysicalFileInfo physical;
+    CsvRecordSize recordSizer;
+    size32_t readBlockSize;
+};
+
+class WorkunitDiskDataSource : public DirectDiskDataSource
+{
+public:
+    WorkunitDiskDataSource(const char * _logicalName, IConstWUResult * _wuResult, const char * _wuid, const char * _username, const char * _password);
+
+    virtual bool init();
+};
+
+
+class TranslatedDiskDataSource : public ADataSource
+{
+public:
+    TranslatedDiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char * _cluster, const char * _username, const char * _password);
+    ~TranslatedDiskDataSource();
+
+    virtual bool init();
+    virtual IFvDataSourceMetaData * queryMetaData()         { return directSource->queryMetaData(); }
+    virtual __int64 numRows(bool force = false)             { return directSource->numRows(force); }
+    virtual bool fetchRow(MemoryBuffer & out, __int64 offset) { return directSource->fetchRow(out, offset); }
+    virtual bool fetchRawRow(MemoryBuffer & out, __int64 offset) { return directSource->fetchRawRow(out, offset); }
+    virtual bool getRow(MemoryBuffer & out, __int64 row)    { return directSource->getRow(out, row); }
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row) { return directSource->getRawRow(out, row); }
+    virtual bool isIndex() { return false; }
+    virtual void onClose()  { openCount--; }
+    virtual void onOpen()   { openCount++; }
+
+protected:
+    bool createHelperWU();
+    bool compileHelperWU();
+
+protected:
+    StringAttr helperWuid;
+    StringAttr logicalName;
+    StringAttr cluster;
+    StringAttr username;
+    StringAttr password;
+    HqlExprAttr diskRecord;
+    Owned<ADataSource> directSource;
+    unsigned openCount;
+};
+
+
+
+class IndirectDiskDataSource : public DiskDataSource
+{
+public:
+    IndirectDiskDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char * _cluster, const char * _username, const char * _password);
+    ~IndirectDiskDataSource();
+
+    virtual bool init();
+
+protected:
+    bool createBrowseWU();
+    virtual bool loadBlock(__int64 startRow, offset_t startOffset);
+
+protected:
+    StringAttr browseWuid;
+    StringAttr queue;
+    StringAttr cluster;
+    StringAttr username;
+    StringAttr password;
+    unsigned __int64 totalSize;
+};
+
+
+#endif

common/fileview2/fvdsremote.cpp

@@ -0,0 +1,602 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "eclrtl.hpp"
+
+#include "hqlexpr.hpp"
+#include "hqlthql.hpp"
+#include "fvresultset.ipp"
+#include "fileview.hpp"
+#include "fvdisksource.ipp"
+#include "fvwugen.hpp"
+#include "fvdsremote.ipp"
+#include "fverror.hpp"
+#include "mpcomm.hpp"
+
+
+#define TIMEOUT             60000
+#define REMOTE_DATA_SIZE    8000        // roughly how much is sent back for each request
+
+enum { FVCMDnone, FVCMDrow, FVCMDraw, FVCMDnumrows, FVCMDcreatewu, FVCMDcreatefile, FVCMDdestroy, FVCMDfetch, FVCMDfetchraw, FVCMDmax };
+
+//---------------------------------------------------------------------------
+
+static void sendReceive(INode * serverNode, CMessageBuffer & msg)
+{
+    if (!queryWorldCommunicator().sendRecv(msg, serverNode, MPTAG_FILEVIEW, TIMEOUT))
+        throwError(FVERR_TimeoutRemoteFileView);
+
+    msg.setEndian(__BIG_ENDIAN);
+    IException * error = deserializeException(msg);
+    if (error)
+        throw error;
+}
+
+
+RemoteDataSource::RemoteDataSource(const SocketEndpoint & _serverEP, unique_id_t _id, IFvDataSourceMetaData * _metaData, __int64 _cachedNumRows, bool _isIndex) : serverEP(_serverEP)
+{
+    id = _id;
+    metaData.set(_metaData);
+    serverNode.setown(createINode(serverEP));
+    cachedNumRows = _cachedNumRows;
+    index = _isIndex;
+    openCount = 0;
+}
+
+
+IFvDataSourceMetaData * RemoteDataSource::queryMetaData()
+{
+    return metaData;
+}
+
+void RemoteDataSource::beforeDispose()
+{
+    CMessageBuffer msg;
+    msg.setEndian(__BIG_ENDIAN);
+    msg.append((byte)FVCMDdestroy);
+    msg.append(id);
+
+    sendReceive(msg);
+}
+
+
+bool RemoteDataSource::getARow(MemoryBuffer & out, RowCache & cache, byte cmd, __int64 row)
+{
+    RowLocation location;
+
+    if (cache.getCacheRow(row, location))
+    {
+        out.append(location.matchLength, location.matchRow);
+        return true;
+    }
+
+    CMessageBuffer msg;
+    msg.setEndian(__BIG_ENDIAN);
+    msg.append(cmd);
+    msg.append(id);
+    msg.append(row);
+
+    sendReceive(msg);
+
+    bool ok;
+    msg.read(ok);
+    if (!ok) return false;
+
+    __int64 start;
+    msg.read(start);
+
+    VariableRowBlock * next = new VariableRowBlock(msg, start);
+    cache.addRowsOwn(next);
+
+    if (!cache.getCacheRow(row, location))
+        assertex(!"Internal Error!");
+    out.append(location.matchLength, location.matchRow);
+    return true;
+}
+
+
+bool RemoteDataSource::fetchRow(MemoryBuffer & out, __int64 offset)
+{
+    CMessageBuffer msg;
+    msg.setEndian(__BIG_ENDIAN);
+    msg.append(FVCMDfetch);
+    msg.append(id);
+    msg.append(offset);
+
+    sendReceive(msg);
+
+    bool ok;
+    msg.read(ok);
+    if (!ok) return false;
+    size32_t len;
+    msg.read(len);
+    out.append(len, msg.readDirect(len));
+    return true;
+}
+
+bool RemoteDataSource::fetchRawRow(MemoryBuffer & out, __int64 offset)
+{
+    CMessageBuffer msg;
+    msg.setEndian(__BIG_ENDIAN);
+    msg.append(FVCMDfetchraw);
+    msg.append(id);
+    msg.append(offset);
+
+    sendReceive(msg);
+
+    bool ok;
+    msg.read(ok);
+    if (!ok) return false;
+    size32_t len;
+    msg.read(len);
+    out.append(len, msg.readDirect(len));
+    return true;
+}
+
+bool RemoteDataSource::getRow(MemoryBuffer & out, __int64 row)
+{
+    return getARow(out, translatedRows, FVCMDrow, row);
+}
+
+bool RemoteDataSource::getRawRow(MemoryBuffer & out, __int64 row)
+{
+    return getARow(out, rawRows, FVCMDraw, row);
+}
+
+
+__int64 RemoteDataSource::numRows(bool force)
+{
+    if (!force)
+        return cachedNumRows;
+    CMessageBuffer msg;
+    msg.setEndian(__BIG_ENDIAN);
+    msg.append((byte)FVCMDnumrows);
+    msg.append(id);
+
+    sendReceive(msg);
+
+    __int64 result;
+    msg.read(result);
+    return result;
+}
+
+
+void RemoteDataSource::onClose()
+{ 
+    if (--openCount == 0)
+    {
+        //MORE: Should tell the server...
+    }
+}
+
+void RemoteDataSource::onOpen() 
+{ 
+    //MORE: critical section
+    if (openCount++ == 0)
+    {
+        //MORE - tell the server...
+    }
+}
+
+void RemoteDataSource::sendReceive(CMessageBuffer & msg)
+{
+    ::sendReceive(serverNode, msg);
+}
+
+
+IFvDataSource * createRemoteDataSource(const SocketEndpoint & server, const char * username, const char * password, const char * wuid, unsigned sequence, const char * name)
+{
+    Owned<INode> serverNode = createINode(server);
+
+    CMessageBuffer msg;
+    msg.setEndian(__BIG_ENDIAN);
+    msg.append((byte)FVCMDcreatewu);
+    msg.append(myProcessSession());
+    msg.append(username);
+    msg.append(password);
+    msg.append(wuid);
+    msg.append(sequence);
+    msg.append(name);
+
+    sendReceive(serverNode, msg);
+
+    unsigned short version;
+    unique_id_t id;
+    __int64 numRows;
+    bool isIndex;
+    msg.read(version);
+    msg.read(id);
+    msg.read(numRows);
+    Owned<IFvDataSourceMetaData> meta = deserializeDataSourceMeta(msg);
+    msg.read(isIndex);
+    if (id)
+        return new RemoteDataSource(server, id, meta, numRows, isIndex);
+    return 0;
+}
+
+IFvDataSource * createRemoteFileDataSource(const SocketEndpoint & server, const char * username, const char * password, const char * logicalName)
+{
+    Owned<INode> serverNode = createINode(server);
+
+    CMessageBuffer msg;
+    msg.setEndian(__BIG_ENDIAN);
+    msg.append((byte)FVCMDcreatefile);
+    msg.append(myProcessSession());
+    msg.append(username);
+    msg.append(password);
+    msg.append(logicalName);
+
+    sendReceive(serverNode, msg);
+
+    unsigned short version;
+    unique_id_t id;
+    __int64 numRows;
+    bool isIndex;
+    msg.read(version);
+    msg.read(id);
+    msg.read(numRows);
+    Owned<IFvDataSourceMetaData> meta = deserializeDataSourceMeta(msg);
+    msg.read(isIndex);
+
+    if (id)
+        return new RemoteDataSource(server, id, meta, numRows, isIndex);
+    return 0;
+}
+
+
+
+//---------------------------------------------------------------------------
+
+static RemoteDataSourceServer * server;
+
+RemoteDataEntry::~RemoteDataEntry()
+{
+    if (subscription)
+        querySessionManager().unsubscribeSession(subscription);
+}
+
+
+RemoteDataSourceServer::RemoteDataSourceServer(const char * _queue, const char * _cluster) : Thread("Remote File View Server")
+{
+    alive = true;
+    nextId = 0;
+    queue.set(_queue);
+    cluster.set(_cluster);
+}
+
+
+unique_id_t RemoteDataSourceServer::addDataSource(SessionId session, IFvDataSource * ds)
+{
+    RemoteDataEntry * newEntry = new RemoteDataEntry;
+    newEntry->id = ++nextId;
+    newEntry->session = session;
+    newEntry->ds.set(ds);
+    newEntry->subscription = querySessionManager().subscribeSession(session, this);
+
+    //MORE: Register the session so if it dies then we get notified.
+    CriticalBlock procedure(cs);
+    entries.append(*newEntry);
+    return newEntry->id;
+}
+
+void RemoteDataSourceServer::doCmdFetch(bool raw, MemoryBuffer & in, MemoryBuffer & out)
+{
+    Owned<IFvDataSource> ds = readDataSource(in);
+    if (!ds)
+    {
+        out.append(false);
+        return;
+    }
+
+    __int64 requestedOffset;
+    in.read(requestedOffset);
+
+    MemoryBuffer temp;
+    bool ok = ds->fetchRow(temp, requestedOffset);
+    out.append(ok);                     // ok
+    out.append(temp.length());
+    out.append(temp.length(), temp.toByteArray());
+}
+
+void RemoteDataSourceServer::doCmdFetchRaw(bool raw, MemoryBuffer & in, MemoryBuffer & out)
+{
+    Owned<IFvDataSource> ds = readDataSource(in);
+    if (!ds)
+    {
+        out.append(false);
+        return;
+    }
+
+    __int64 requestedOffset;
+    in.read(requestedOffset);
+
+    MemoryBuffer temp;
+    bool ok = ds->fetchRawRow(temp, requestedOffset);
+    out.append(ok);                     // ok
+    out.append(temp.length());
+    out.append(temp.length(), temp.toByteArray());
+}
+
+void RemoteDataSourceServer::doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out)
+{
+    Owned<IFvDataSource> ds = readDataSource(in);
+    if (!ds)
+    {
+        out.append(false);
+        return;
+    }
+
+    __int64 requestedRow;
+    in.read(requestedRow);
+
+    unsigned startPos = out.length();
+    unsigned numRows = 0;
+    out.append(true);                       // ok
+    out.append(requestedRow);       // start 
+
+    unsigned numRowsPos = out.length();
+    out.append(numRows);                // total number of rows;
+    loop
+    {
+        unsigned lengthPos = out.length();
+        out.append((unsigned)0);                // size of this row.
+        unsigned startRow = out.length();
+        if (raw)
+        {
+            if (!ds->getRawRow(out, requestedRow+numRows))
+                break;
+        }
+        else
+        {
+            if (!ds->getRow(out, requestedRow+numRows))
+                break;
+        }
+        if ((numRows != 0) && (out.length() > REMOTE_DATA_SIZE))
+            break;
+        unsigned endRow = out.length();
+        out.rewrite(lengthPos);
+        out.append(endRow-startRow);
+        out.rewrite(endRow);
+        numRows++;
+    }
+
+    if (numRows == 0)
+    {
+        out.rewrite(startPos);
+        out.append(false);
+        return;
+    }
+
+    unsigned totalLength = out.length();
+    out.rewrite(numRowsPos);
+    out.append(numRows);
+    out.rewrite(totalLength);
+}
+
+void RemoteDataSourceServer::doCmdNumRows(MemoryBuffer & in, MemoryBuffer & out)
+{
+    Owned<IFvDataSource> ds = readDataSource(in);
+    __int64 numRows = ds ? ds->numRows(true) : 0;
+    out.append(numRows);
+}
+
+void RemoteDataSourceServer::doCmdCreateWorkunit(MemoryBuffer & in, MemoryBuffer & out)
+{
+    SessionId session;
+    StringAttr wuid, username, password;
+    unsigned sequence;
+    StringAttr name;
+
+    in.read(session);
+    in.read(username).read(password);
+    in.read(wuid);
+    in.read(sequence);
+    in.read(name);
+
+    DBGLOG("RemoteFileView:CreateWorkunit('%s',%d,'%s') by[%s:%"I64F"d", wuid.get(), sequence, name ? name.get() : "", username.get(), session);
+    Owned<IConstWUResult> wuResult = resolveResult(wuid, sequence, name);
+    Owned<IFvDataSource> ds = createDataSource(wuResult, wuid, username, password);
+    unique_id_t id = addDataSource(session, ds);
+
+    out.append((unsigned short)CurRemoteVersion);
+    out.append(id);
+    out.append(ds->numRows(false));
+    ds->queryMetaData()->serialize(out);
+    out.append(ds->isIndex());
+
+    DBGLOG("RemoteFileView:CreateWorkunit returns %"I64F"d", id);
+}
+
+void RemoteDataSourceServer::doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out)
+{
+    SessionId session;
+    StringAttr username, password, logicalName;
+
+    in.read(session);
+    in.read(username).read(password);
+    in.read(logicalName);
+
+    DBGLOG("RemoteFileView:CreateFile('%s') by[%s:%"I64F"d", logicalName.get(), username.get(), session);
+    Owned<IFvDataSource> ds = createFileDataSource(logicalName, cluster, username, password);
+    unique_id_t id = addDataSource(session, ds);
+
+    out.append((unsigned short)CurRemoteVersion);
+    out.append(id);
+    out.append(ds->numRows(false));
+    ds->queryMetaData()->serialize(out);
+    out.append(ds->isIndex());
+
+    DBGLOG("RemoteFileView:CreateFile returns %"I64F"d", id);
+}
+
+void RemoteDataSourceServer::doCmdDestroy(MemoryBuffer & in, MemoryBuffer & out)
+{
+    unique_id_t id;
+    in.read(id);
+
+    DBGLOG("RemoteFileView:Destroy(%"I64F"d)", id);
+    CriticalBlock block(cs);
+    ForEachItemIn(idx, entries)
+    {
+        RemoteDataEntry & cur = entries.item(idx);
+        if (cur.id == id)
+        {
+            entries.remove(idx);
+            return;
+        }
+    }
+}
+
+
+IFvDataSource * RemoteDataSourceServer::getDataSource(unique_id_t id)
+{
+    CriticalBlock block(cs);
+    ForEachItemIn(idx, entries)
+    {
+        RemoteDataEntry & cur = entries.item(idx);
+        if (cur.id == id)
+            return LINK(cur.ds);
+    }
+    return NULL;
+}
+
+
+void RemoteDataSourceServer::closed(SessionId id)
+{
+    removeSession(id);
+}
+
+void RemoteDataSourceServer::aborted(SessionId id)
+{
+    removeSession(id);
+}
+
+IFvDataSource * RemoteDataSourceServer::readDataSource(MemoryBuffer & in)
+{
+    unique_id_t id;
+    in.read(id);
+    return getDataSource(id);
+}
+
+
+void RemoteDataSourceServer::removeSession(SessionId id)
+{
+    DBGLOG("RemoteFileView:Session Died");
+    CriticalBlock block(cs);
+    ForEachItemInRev(idx, entries)
+    {
+        RemoteDataEntry & cur = entries.item(idx);
+        if (cur.session == id)
+        {
+            DBGLOG("RemoteFileView:Instance Died %"I64F"d", cur.id);
+            entries.remove(idx);
+        }
+    }
+}   
+
+
+//MORE: If this is ever actually used then it should probably have several threads
+//      processing the commands, especially if the commands can involve lots of processing.
+int RemoteDataSourceServer::run()
+{
+    CMessageBuffer msg;
+    MemoryBuffer result;
+    INode * sender;
+    while (alive)
+    {
+        msg.clear();
+        if (queryWorldCommunicator().recv(msg, 0, MPTAG_FILEVIEW, &sender))
+        {
+            msg.setEndian(__BIG_ENDIAN);
+            result.setEndian(__BIG_ENDIAN);
+
+
+            try
+            {
+                serializeException(NULL, result.clear());
+                byte cmd;
+                msg.read(cmd);
+                switch (cmd)
+                {
+                case FVCMDrow:          doCmdRow(false, msg, result); break;
+                case FVCMDraw:          doCmdRow(true, msg, result); break;
+                case FVCMDnumrows:      doCmdNumRows(msg, result); break;
+                case FVCMDcreatewu:     doCmdCreateWorkunit(msg, result); break;
+                case FVCMDcreatefile:   doCmdCreateFile(msg, result); break;
+                case FVCMDdestroy:      doCmdDestroy(msg, result); break;
+                case FVCMDfetch:        doCmdFetch(false, msg, result); break;
+                case FVCMDfetchraw:     doCmdFetchRaw(false, msg, result); break;
+                default:
+                    throwError(FVERR_UnknownRemoteCommand);
+                }
+                msg.clear().append(result);
+            }
+            catch (IException * e)
+            {
+                serializeException(e, msg.clear());
+                e->Release();
+            }
+
+            queryWorldCommunicator().reply(msg, MP_ASYNC_SEND);
+            ::Release(sender);
+        }
+    }
+    server = NULL;
+    return 0;
+}
+
+
+
+void RemoteDataSourceServer::stop()
+{
+    alive = false;
+    queryWorldCommunicator().cancel(0, MPTAG_FILEVIEW);
+    join();
+}
+
+extern FILEVIEW_API void startRemoteDataSourceServer(const char * queue, const char * cluster)
+{
+    //This isn't properly thread safe - it also isn't ever used in practice, so not a problem.
+    if (!server)
+    {
+        server = new RemoteDataSourceServer(queue, cluster);
+        server->start();
+    }
+}
+
+extern FILEVIEW_API void stopRemoteDataSourceServer()
+{
+    if (server)
+        server->stop();
+}
+
+
+IConstWUResult * resolveResult(const char * wuid, unsigned sequence, const char * name)
+{
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnit> wu = factory->openWorkUnit(wuid, false);
+    return getWorkUnitResult(wu, name, sequence);
+}
+
+IConstWUResult * secResolveResult(ISecManager &secmgr, ISecUser &secuser, const char * wuid, unsigned sequence, const char * name)
+{
+    Owned<IWorkUnitFactory> factory = getSecWorkUnitFactory(secmgr, secuser);
+    Owned<IConstWorkUnit> wu = factory->openWorkUnit(wuid, false);
+    return (wu) ? getWorkUnitResult(wu, name, sequence) : NULL;
+}

common/fileview2/fvdsremote.ipp

@@ -0,0 +1,118 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVDSREMOTE_IPP
+#define FVDSREMOTE_IPP
+
+#include "fvdatasource.hpp"
+#include "fvsource.ipp"
+#include "dadfs.hpp"
+#include "dasess.hpp"
+#include "mpbuff.hpp"
+
+enum { 
+    FVRemoteVersion1 = 1,
+    CurRemoteVersion = FVRemoteVersion1
+};
+
+class RemoteDataSource : public ADataSource
+{
+public:
+    RemoteDataSource(const SocketEndpoint & _serverEP, unique_id_t _id, IFvDataSourceMetaData * _metaData, __int64 _cachedNumRows, bool _isIndex);
+
+    virtual bool init() { return true; }
+    virtual IFvDataSourceMetaData * queryMetaData();
+    virtual __int64 numRows(bool force = false);
+    virtual bool fetchRow(MemoryBuffer & out, __int64 offset);
+    virtual bool fetchRawRow(MemoryBuffer & out, __int64 offset);
+    virtual bool getRow(MemoryBuffer & out, __int64 row);
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row);
+    virtual bool isIndex() { return index; }
+    virtual void onClose();
+    virtual void onOpen();
+
+    virtual void beforeDispose();
+    virtual bool optimizeFilter(unsigned offset, unsigned len, const void * data) { return false; } // MORE: Needs implementing if this is ever used.
+
+protected:
+    bool getARow(MemoryBuffer & out, RowCache & cache, byte cmd, __int64 row);
+    void sendReceive(CMessageBuffer & msg);
+
+protected:
+    const SocketEndpoint & serverEP;
+    unique_id_t id;
+    Owned<IFvDataSourceMetaData> metaData;
+    Owned<INode> serverNode;
+    RowCache rawRows;
+    RowCache translatedRows;
+    __int64 cachedNumRows;
+    unsigned openCount;
+    bool index;
+};
+
+
+class RemoteDataEntry : public CInterface
+{
+public:
+    ~RemoteDataEntry();
+    
+public:
+    unique_id_t id;
+    SessionId session;
+    SubscriptionId subscription;
+    Owned<IFvDataSource> ds;
+};
+
+class RemoteDataSourceServer : public Thread, public ISessionNotify
+{
+public:
+    RemoteDataSourceServer(const char * _queue, const char * _cluster);
+    IMPLEMENT_IINTERFACE
+
+//Thread
+    virtual int run();
+
+//ISessionNotify
+    virtual void closed(SessionId id);
+    virtual void aborted(SessionId id);
+    void stop();
+
+protected:
+    unique_id_t addDataSource(SessionId session, IFvDataSource * ds);
+    void doCmdFetch(bool raw, MemoryBuffer & in, MemoryBuffer & out);
+    void doCmdFetchRaw(bool raw, MemoryBuffer & in, MemoryBuffer & out);
+    void doCmdRow(bool raw, MemoryBuffer & in, MemoryBuffer & out);
+    void doCmdRaw(MemoryBuffer & in, MemoryBuffer & out);
+    void doCmdNumRows(MemoryBuffer & in, MemoryBuffer & out);
+    void doCmdCreateWorkunit(MemoryBuffer & in, MemoryBuffer & out);
+    void doCmdCreateFile(MemoryBuffer & in, MemoryBuffer & out);
+    void doCmdDestroy(MemoryBuffer & in, MemoryBuffer & out);
+    IFvDataSource * getDataSource(unique_id_t id);
+    IFvDataSource * readDataSource(MemoryBuffer & in);
+    void removeSession(SessionId id);
+
+protected:
+    bool alive;
+    unique_id_t nextId;
+    CriticalSection cs;
+    StringAttr queue;
+    StringAttr cluster;
+    CIArrayOf<RemoteDataEntry> entries;
+};
+
+#endif

common/fileview2/fverror.hpp

@@ -0,0 +1,86 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVERROR_HPP
+#define FVERROR_HPP
+
+#include "jerrorrange.hpp"
+
+#define ERR_FILEVIEW_FIRST  2000
+#define ERR_FILEVIEW_LAST   2049
+
+#define FVERR_CouldNotResolveX                  2000
+#define FVERR_NoRecordDescription               2001
+#define FVERR_BadRecordDesc                     2002
+#define FVERR_NeedClusterToBrowseX              2003
+#define FVERR_TimeoutRemoteFileView             2004
+#define FVERR_UnknownRemoteCommand              2005
+#define FVERR_UnknownUTFFormat                  2006
+#define FVERR_FailedOpenFile                    2007
+#define FVERR_CompressedFile                    2008
+#define FVERR_CannotViewKey                     2009
+#define FVERR_ViewComplexKey                    2010
+#define FVERR_FilterTooRestrictive              2011
+#define FVERR_ZeroSizeRecord                    2012
+#define FVERR_FailedOpenCompressedFile          2013
+#define FVERR_UnrecognisedJoinFieldSyntax       2014
+#define FVERR_UnrecognisedJoinFieldSyntaxXX     2015
+#define FVERR_UnrecognisedMappingFunctionX      2016
+#define FVERR_UnrecognisedFieldX                2017
+#define FVERR_ExpectedFieldSelectedFromDatasetXX 2018
+#define FVERR_CannotSelectFromDatasetX          2019
+#define FVERR_CannotSelectManyFromDatasetX      2020
+#define FVERR_ExpectedFieldSelectedFromRecordXX 2021
+#define FVERR_NumJoinFieldsMismatchXY           2022
+#define FVERR_ExpectedX                         2023
+#define FVERR_FailTransformation                2024
+#define FVERR_UnrecognisedMappingFunctionXY     2025
+#define FVERR_BadStringTermination              2026
+#define FVERR_CannotBrowseFile                  2027
+#define FVERR_PluginMismatch                    2028
+
+#define FVERR_CouldNotResolveX_Text             "Could not resolve file '%s' in DFS"
+#define FVERR_NoRecordDescription_Text          "DFS did not contain record description for '%s'"
+#define FVERR_BadRecordDesc_Text                "Could not process record description for '%s'"
+#define FVERR_NeedClusterToBrowseX_Text         "Need queue/cluster to browse file '%s'"
+#define FVERR_TimeoutRemoteFileView_Text        "Connection to file view server timed out"
+#define FVERR_UnknownRemoteCommand_Text         "Unknown remote command"
+#define FVERR_UnknownUTFFormat_Text             "Unknown utf format: %s"
+#define FVERR_FailedOpenFile_Text               "Failed to open file %s for browsing"
+#define FVERR_CompressedFile_Text               "Cannot view compressed file '%s'"
+#define FVERR_CannotViewKey_Text                "Cannot view key %s"
+#define FVERR_ViewComplexKey_Text               "Cannot view complex key '%s'"
+#define FVERR_FilterTooRestrictive_Text         "Filter too restrictive - no records matched within %d seconds"
+#define FVERR_ZeroSizeRecord_Text               "File %s appears to have a zero length row"
+#define FVERR_FailedOpenCompressedFile_Text     "Failed to open file %s as a compressed file"
+#define FVERR_UnrecognisedJoinFieldSyntax_Text  "Unrecognised field mapping syntax"
+#define FVERR_UnrecognisedJoinFieldSyntaxXX_Text "Unrecognised field mapping syntax %.*s"
+#define FVERR_UnrecognisedMappingFunctionX_Text "Unrecognised field mapping function %s"
+#define FVERR_UnrecognisedFieldX_Text           "Unrecognised field %s"
+#define FVERR_ExpectedFieldSelectedFromDatasetXX_Text "Expected a field selected from dataset %.*s"
+#define FVERR_CannotSelectFromDatasetX_Text     "Selection from dataset %s not supported in this context"
+#define FVERR_CannotSelectManyFromDatasetX_Text "Cannot select multiple fields from dataset %s in this context"
+#define FVERR_ExpectedFieldSelectedFromRecordXX_Text "Expected a field selected from dataset %.*s"
+#define FVERR_NumJoinFieldsMismatchXY_Text       "Number of join fields do not match (%d, %d) relation(%s,%s)"
+#define FVERR_ExpectedX_Text                     "Expected %s at [%.*s]"
+#define FVERR_FailTransformation_Text            "FAIL transform called unexpectedly"
+#define FVERR_UnrecognisedMappingFunctionXY_Text    "Unrecognised field mapping function %s.%s"
+#define FVERR_BadStringTermination_Text          "String not terminated correctly %.*s"
+#define FVERR_CannotBrowseFile_Text              "Cannot browse file '%s'"
+
+#endif

common/fileview2/fvidxsource.cpp

@@ -0,0 +1,460 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "eclrtl_imp.hpp"
+
+#include "hqlexpr.hpp"
+#include "fileview.hpp"
+#include "fvresultset.ipp"
+#include "fvidxsource.ipp"
+#include "fverror.hpp"
+#include "dasess.hpp"
+
+//cloned from hthor - a candidate for commoning up.
+static IKeyIndex *openKeyFile(IDistributedFilePart *keyFile)
+{
+    unsigned numCopies = keyFile->numCopies();
+    assertex(numCopies);
+    for (unsigned copy=0; copy < numCopies; copy++)
+    {
+        RemoteFilename rfn;
+        try
+        {
+            OwnedIFile ifile = createIFile(keyFile->getFilename(rfn,copy));
+            unsigned __int64 thissize = ifile->size();
+            if (thissize != -1)
+            {
+                StringBuffer remotePath;
+                rfn.getRemotePath(remotePath);
+                unsigned crc;
+                keyFile->getCrc(crc);
+                return createKeyIndex(remotePath.str(), crc, false, false);
+            }
+        }
+        catch (IException *E)
+        {
+            EXCLOG(E, "While opening index file");
+            E->Release();
+        }
+    }
+    RemoteFilename rfn;
+    StringBuffer url;
+    keyFile->getFilename(rfn).getRemotePath(url);
+    throw MakeStringException(1001, "Could not open key file at %s%s", url.str(), (numCopies > 1) ? " or any alternate location." : ".");
+}
+
+//---------------------------------------------------------------------------
+
+#define MIN_CACHED_ROWS         150
+#define MAX_CACHED_ROWS         200
+
+IndexPageCache::IndexPageCache()
+{
+    firstRow = 0;
+    offsetDelta = 0;
+    offsets.append(0);
+    saved.ensureCapacity(0x10000);
+}
+
+void IndexPageCache::addRow(__int64 row, size32_t len, const void * data)
+{
+    if (row != firstRow + numRowsCached())
+    {
+        firstRow = row;
+        offsetDelta = 0;
+        offsets.kill();
+        offsets.append(0);
+        saved.setWritePos(0);
+    }
+    else if (numRowsCached() >= MAX_CACHED_ROWS)
+    {
+        unsigned numToRemove = numRowsCached() - MIN_CACHED_ROWS;
+        __int64 newDelta = offsets.item(numToRemove);
+        size32_t sizeLost = (size32_t)(newDelta-offsetDelta);
+        //copy the cached rows
+        byte * base = (byte *)saved.bufferBase();
+        memmove(base, base+sizeLost, saved.length()-sizeLost);
+        saved.setWritePos(saved.length()-sizeLost);
+        offsets.removen(0, numToRemove);
+        firstRow += numToRemove;
+        offsetDelta = newDelta;
+    }
+
+    assertex(row == firstRow + numRowsCached());
+    assertex(offsets.tos() == saved.length() + offsetDelta);
+    saved.append(len, data);
+    offsets.append(saved.length() + offsetDelta);
+}
+
+bool IndexPageCache::getRow(__int64 row, size32_t & len, const void * & data)
+{
+    if (row < firstRow || row >= firstRow + numRowsCached())
+        return false;
+
+    unsigned __int64 startOffset = offsets.item((unsigned)(row-firstRow));
+    unsigned __int64 endOffset = offsets.item((unsigned)(row-firstRow+1));
+    len = (size32_t)(endOffset - startOffset);
+    data = saved.toByteArray() + (unsigned)(startOffset - offsetDelta);
+    return true;
+}
+
+
+//---------------------------------------------------------------------------
+
+//Could probably cope with unknown record, just displaying as binary.
+
+IndexDataSource::IndexDataSource(const char * _logicalName, IHqlExpression * _diskRecord, const char* _username, const char* _password)
+{
+    logicalName.set(_logicalName);
+    diskRecord.set(_diskRecord);
+
+    Owned<IUserDescriptor> udesc;
+    if(_username != NULL && *_username != '\0')
+    {
+        udesc.setown(createUserDescriptor());
+        udesc->set(_username, _password);
+    }
+
+    df.setown(queryDistributedFileDirectory().lookup(logicalName, udesc.get()));
+    filtered = false;
+}
+
+
+IndexDataSource::IndexDataSource(IndexDataSource * _other)
+{
+    logicalName.set(_other->logicalName);
+    diskRecord.set(_other->diskRecord);
+    df.set(_other->df);
+    original.set(_other);       // stop any work units etc. being unloaded.
+    diskMeta.set(_other->diskMeta);     // optimization - would be handled by init anyway
+    filtered = false;
+    //MORE: What else needs cloning/initializing?
+}
+
+
+IFvDataSource * IndexDataSource::cloneForFilter()
+{
+    Owned<IndexDataSource> ret = new IndexDataSource(this);
+    if (ret->init())
+        return ret.getClear();
+    return NULL;
+}
+
+bool IndexDataSource::init()
+{
+    if (!df)
+        return false;
+
+    numParts = df->numParts();
+    singlePart = (numParts == 1);
+    ignoreSkippedRows = true;               // better if skipping to a particular point
+    StringBuffer partName;
+    Owned<IDistributedFilePart> kf = df->getPart(numParts-1);
+    tlk.setown(openKeyFile(kf));
+    if (!tlk)
+        return false;
+
+    IPropertyTree & properties = df->queryProperties();
+    //Need to assign the transformed record to meta
+    if (!diskMeta)
+        diskMeta.setown(new DataSourceMetaData(diskRecord, 0, true, false, tlk->keyedSize()));
+
+    if (!returnedMeta)
+    {
+        returnedMeta.set(diskMeta);
+        returnedRecordSize.set(returnedMeta);
+    }
+    if (!transformedMeta)
+        transformedMeta.set(returnedMeta);
+
+    if (properties.hasProp("@recordCount"))
+        totalRows = properties.getPropInt64("@recordCount");
+    else
+        totalRows = UNKNOWN_NUM_ROWS;       // more: could probably count them
+
+    isLocal = properties.hasProp("@local");
+
+    diskMeta->extractKeyedInfo(keyedOffsets, keyedTypes);
+    ForEachItemIn(i, keyedTypes)
+    {
+        IStringSet * set = createRtlStringSet(fieldSize(i));
+        set->addAll();
+        values.append(*set);
+    }
+
+    fileposFieldType = diskMeta->queryType(diskMeta->numColumns()-1);
+    assertex(fileposFieldType && fileposFieldType->isInteger());
+
+    //Now gather all the 
+
+    //Default cursor if no filter is applied
+    applyFilter();
+    return true;
+}
+
+__int64 IndexDataSource::numRows(bool force)
+{
+    if (!filtered)
+        return totalRows;
+
+    //If leading component isn't filtered, then this can take a very long time...
+    if (!force && values.item(0).isFullSet())
+        return UNKNOWN_NUM_ROWS;
+
+    __int64 total = 0;
+    ForEachItemIn(i, matchingParts)
+    {
+        manager->setKey(NULL);
+        curPart.clear();
+        if (singlePart)
+            curPart.set(tlk);
+        else
+        {
+            Owned<IDistributedFilePart> kf = df->getPart(matchingParts.item(i));
+            curPart.setown(openKeyFile(kf));
+            if (!curPart)
+            {
+                total = UNKNOWN_NUM_ROWS;
+                break;
+            }
+        }
+
+        manager->setKey(curPart);
+        manager->reset();
+        total += manager->getCount();
+    }
+
+    manager->setKey(NULL);
+    curPart.clear();
+    resetCursor();
+    return total;
+}
+
+
+bool IndexDataSource::getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset)
+{
+    if (cache.getRow(row, length, data))
+        return true;
+
+    if (row < 0)
+        return false;
+
+    if ((unsigned __int64)row < nextRowToRead)
+        resetCursor();
+
+    MemoryBuffer temp;
+    while ((unsigned __int64)row >= nextRowToRead)
+    {
+        bool saveRow = !ignoreSkippedRows || (row == nextRowToRead);
+        if (!getNextRow(temp.clear(), saveRow))
+            return false;
+        if (saveRow)
+            cache.addRow(nextRowToRead, temp.length(), temp.toByteArray());
+        nextRowToRead++;
+    }
+    return cache.getRow(row, length, data);
+}
+
+
+bool IndexDataSource::getNextRow(MemoryBuffer & out, bool extractRow)
+{
+    bool nextPart = !matchingParts.isItem((unsigned)curPartIndex);
+    loop
+    {
+        if (nextPart)
+        {
+            if ((curPartIndex != -1) && ((unsigned)curPartIndex >= matchingParts.ordinality()))
+                return false;
+            manager->setKey(NULL);
+            curPart.clear();
+            if ((unsigned)++curPartIndex >= matchingParts.ordinality())
+                return false;
+            if (singlePart)
+                curPart.set(tlk);
+            else
+            {
+                Owned<IDistributedFilePart> kf = df->getPart(matchingParts.item(curPartIndex));
+                curPart.setown(openKeyFile(kf));
+                if (!curPart)
+                    return false;
+            }
+            manager->setKey(curPart);
+            manager->reset();
+        }
+        
+        if (manager->lookup(true))
+        {
+            if (extractRow)
+            {
+                if (false)
+                {
+                    //MORE: Allow a transformer to cope with blobs etc.
+                }
+                else
+                {
+                    unsigned fileposSize = fileposFieldType->getSize();
+    //              unsigned thisSize = manager->queryRecordSize();             // Should be possible - needs a new function to call cursor->getSize()
+                    offset_t filepos;
+                    const byte * thisRow = manager->queryKeyBuffer(filepos);
+                    unsigned thisSize = diskMeta->getRecordSize(thisRow) - fileposSize;
+                    void * temp = out.reserve(thisSize);
+                    memcpy(temp, thisRow, thisSize);
+
+                    //Append the fileposition, in the correct size/endianness
+                    assertex(sizeof(filepos) >= 8);
+
+#if __BYTE_ORDER == __LITTLE_ENDIAN
+                    void * data = &filepos;
+#else
+                    void * data = (byte *)&filepos + sizeof(filepos) - fileposSize;
+#endif
+                    if (fileposFieldType->isSwappedEndian())
+                        out.appendSwap(fileposSize, data);
+                    else
+                        out.append(fileposSize, data);
+                }
+            }
+            return true;
+        }
+
+        nextPart = true;
+    }
+}
+
+void IndexDataSource::resetCursor()
+{
+    curPartIndex = -1;
+    nextRowToRead = 0;
+}
+
+bool IndexDataSource::addFilter(unsigned column, unsigned matchLen, unsigned sizeData, const void * data)
+{
+    if (!values.isItem(column))
+        return false;
+    unsigned curSize = fieldSize(column);
+    IStringSet & set = values.item(column);
+    if (set.isFullSet())
+        set.reset();
+
+    ITypeInfo & cur = keyedTypes.item(column);
+    rtlDataAttr tempLow, tempHigh;
+    unsigned keyedSize = cur.getSize();
+    byte * temp = (byte *)alloca(keyedSize);
+    const void * low = data;
+    const void * high = data;
+    type_t tc = cur.getTypeCode();
+    switch (tc)
+    {
+    case type_int:
+    case type_swapint:
+        {
+            assertex(sizeData == curSize);
+            // values are already converted to bigendian and correctly biased
+            break;
+        }
+    case type_varstring:
+        //should cast from string to varstring
+        break;
+    case type_string:
+    case type_data:
+        {
+            const char * inbuff = (const char *)data;
+            if (matchLen != FullStringMatch)
+            {
+                unsigned lenLow, lenHigh;
+                rtlCreateRangeLow(lenLow, tempLow.refstr(), curSize, matchLen, sizeData, inbuff);
+                rtlCreateRangeHigh(lenHigh, tempHigh.refstr(), curSize, matchLen, sizeData, inbuff);
+                low = tempLow.getdata();
+                high = tempHigh.getdata();
+            }
+            else
+            {
+                if (tc == type_string)
+                {
+                    //may need to cast from ascii to ebcidic
+                    rtlStrToStr(curSize, temp, sizeData, data);
+                }
+                else
+                    rtlDataToData(curSize, temp, sizeData, data);
+                low = high = temp;
+            }
+            break;
+        }
+    case type_qstring:
+        {
+            const char * inbuff = (const char *)data;
+            unsigned lenData = rtlQStrLength(sizeData);
+            unsigned lenField = cur.getStringLen();
+            if (matchLen != FullStringMatch)
+            {
+                unsigned lenLow, lenHigh;
+                rtlCreateQStrRangeLow(lenLow, tempLow.refstr(), lenField, matchLen, lenData, inbuff);
+                rtlCreateQStrRangeHigh(lenHigh, tempHigh.refstr(), lenField, matchLen, lenData, inbuff);
+                low = tempLow.getdata();
+                high = tempHigh.getdata();
+            }
+            else
+            {
+                rtlQStrToQStr(lenField, (char *)temp, lenData, (const char *)data);
+                low = high = temp;
+            }
+            break;
+        }
+    default:
+        assertex(sizeData == curSize);
+        break;
+    }
+    set.addRange(low, high);
+    filtered = true;
+    return true;
+}
+
+void IndexDataSource::applyFilter()
+{
+    manager.setown(createKeyManager(tlk, tlk->keySize(), NULL));
+    ForEachItemIn(i, values)
+    {
+        IStringSet & cur = values.item(i);
+        bool extend = true; // almost certainly better
+        manager->append(createKeySegmentMonitor(extend, LINK(&cur), keyedOffsets.item(i), fieldSize(i)));
+    }
+    manager->finishSegmentMonitors();
+
+    //Now work out which parts are affected.
+    matchingParts.kill();
+    if (singlePart)
+        matchingParts.append(0);
+    else if (isLocal)
+    {
+        for (unsigned i = 0; i < numParts; i++)
+            matchingParts.append(i);
+    }
+    else
+    {
+        manager->reset();
+        while (manager->lookup(false))
+        {
+            offset_t node = manager->queryFpos();
+            if (node)
+                matchingParts.append((unsigned)(node-1));
+        }
+    }
+    resetCursor();
+}
+

common/fileview2/fvidxsource.ipp

@@ -0,0 +1,104 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVIDXSOURCE_IPP
+#define FVIDXSOURCE_IPP
+
+#include "junicode.hpp"
+#include "fvdatasource.hpp"
+#include "dllserver.hpp"
+#include "hqlexpr.hpp"
+#include "eclhelper.hpp"
+
+#include "fvsource.ipp"
+#include "dadfs.hpp"
+#include "rtlkey.hpp"
+#include "jhtree.hpp"
+
+//---------------------------------------------------------------------------
+
+
+//An initial simple implementation
+class IndexPageCache
+{
+public:
+    IndexPageCache();
+
+    void addRow(__int64 row, size32_t len, const void * data);
+    bool getRow(__int64 row, size32_t & len, const void * & data);
+
+    inline unsigned numRowsCached()               { return offsets.ordinality()-1; }
+
+protected:
+    MemoryBuffer saved;
+    UInt64Array offsets;
+    __int64 firstRow;
+    unsigned __int64 offsetDelta;
+};
+
+class IndexDataSource : public FVDataSource
+{
+public:
+    IndexDataSource(const char * _logicalName, IHqlExpression * _indexRecord, const char* _username, const char* _password);
+    IndexDataSource(IndexDataSource * _other);
+
+    virtual void applyFilter();
+    virtual IFvDataSource * cloneForFilter();
+    virtual __int64 numRows(bool force = false);
+    virtual bool addFilter(unsigned offset, unsigned matchLen, unsigned len, const void * data);
+
+    virtual bool init();
+    virtual bool fetchRowData(MemoryBuffer & out, __int64 offset)                   { return false; }
+    virtual bool getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset);
+    virtual bool isIndex() { return true; }
+
+protected:
+    inline unsigned fieldSize(unsigned column)      { return keyedOffsets.item(column+1) - keyedOffsets.item(column); }
+    bool getNextRow(MemoryBuffer & out, bool extractRow);
+    void resetCursor();
+    void translateKeyedValues(byte * row);
+
+protected:
+    StringAttr logicalName;
+    IArrayOf<IKeySegmentMonitor> segMonitors;
+    IArrayOf<IStringSet> values;
+    Owned<DataSourceMetaData> diskMeta;
+    HqlExprAttr diskRecord;
+    Owned<IDistributedFile> df;
+    Linked<FVDataSource> original;
+    ITypeInfo * fileposFieldType;
+    unsigned __int64 totalRows;
+    unsigned __int64 nextRowToRead;
+    unsigned keyedSize;
+    UnsignedArray keyedOffsets;
+    TypeInfoArray keyedTypes;
+    Owned<IKeyIndex> tlk;
+    Owned<IKeyManager> manager;
+    Owned<IKeyIndex> curPart;
+    int curPartIndex;
+    UnsignedArray matchingParts;
+    unsigned numParts;
+    bool singlePart;
+    bool filtered;
+    bool isLocal;
+    bool ignoreSkippedRows;
+    IndexPageCache cache;
+};
+
+
+#endif

common/fileview2/fvquerysource.cpp

@@ -0,0 +1,178 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "eclrtl.hpp"
+
+#include "hqlexpr.hpp"
+#include "hqlthql.hpp"
+#include "fvresultset.ipp"
+#include "fileview.hpp"
+#include "fvquerysource.ipp"
+
+#include "fvwugen.hpp"
+
+
+QueryDataSource::QueryDataSource(IConstWUResult * _wuResult, const char * _wuid, const char * _username, const char * _password)
+{
+    wuResult.set(_wuResult);
+    wuid.set(_wuid);
+    username.set(_username);
+    password.set(_password);
+}
+
+
+QueryDataSource::~QueryDataSource()
+{
+    if (browseWuid)
+    {
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+        factory->deleteWorkUnit(browseWuid);
+    }
+}
+
+bool QueryDataSource::createBrowseWU()
+{
+    StringAttr dataset, datasetDefs;
+    StringAttrAdaptor a1(dataset), a2(datasetDefs);
+    wuResult->getResultDataset(a1, a2);
+
+    if (!dataset || !datasetDefs)
+        return false;
+    StringBuffer fullText;
+    fullText.append(datasetDefs).append(dataset);
+    OwnedHqlExpr parsed = parseQuery(fullText.str());
+    if (!parsed)
+        return false;
+
+    HqlExprAttr selectFields = parsed.getLink();
+    if (selectFields->getOperator() == no_output)
+        selectFields.set(selectFields->queryChild(0));
+
+    OwnedHqlExpr browseWUcode = buildQueryViewerEcl(selectFields);
+    if (!browseWUcode)
+        return false;
+    returnedRecord.set(browseWUcode->queryChild(0)->queryRecord());
+
+    StringAttr tempAttr;
+    StringAttrAdaptor temp(tempAttr);
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnit> parent = factory->openWorkUnit(wuid, false);
+
+    SCMStringBuffer user;
+    StringAttrAdaptor acluster(cluster);
+    parent->getClusterName(acluster);
+    parent->getUser(user);
+
+    Owned<IWorkUnit> workunit = factory->createWorkUnit(NULL, "fileViewer", user.str());
+    workunit->setUser(user.str());
+    workunit->setClusterName(cluster);
+    workunit->setCustomerId(parent->getCustomerId(temp).str());
+    workunit->setCompareMode(CompareModeOff);   // ? parent->getCompareMode()
+    StringAttrAdaptor bwa(browseWuid); workunit->getWuid(bwa);
+
+    workunit->setDebugValueInt("importImplicitModules", false, true);
+    workunit->setDebugValueInt("importAllModules", false, true);
+    workunit->setDebugValueInt("forceFakeThor", 1, true);
+
+    StringBuffer jobName;
+    jobName.append("FileView for ").append(wuid).append(":").append("x");
+    workunit->setJobName(jobName.str());
+
+    StringBuffer eclText;
+    toECL(browseWUcode, eclText, true);
+    Owned<IWUQuery> query = workunit->updateQuery();
+    query->setQueryText(eclText.str());
+    query->setQueryName(jobName.str());
+
+    return true;
+}
+
+
+bool QueryDataSource::init()
+{
+    return createBrowseWU();
+}
+
+
+
+void QueryDataSource::improveLocation(__int64 row, RowLocation & location)
+{
+#if 0
+    if (!diskMeta->isFixedSize())
+        return;
+
+    if (location.bestRow <= row && location.bestRow + DISKREAD_PAGE_SIZE > row)
+        return;
+
+    assertex(row >= 0);
+    //Align the row so the chunks don't overlap....
+    location.bestRow = (row / DISKREAD_PAGE_SIZE) * DISKREAD_PAGE_SIZE;
+    location.bestOffset = location.bestRow * diskMeta->fixedSize();
+#endif
+}
+
+
+
+
+bool QueryDataSource::loadBlock(__int64 startRow, offset_t startOffset)
+{
+    MemoryBuffer temp;
+
+    //enter scope....>
+    {
+        Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+        Owned<IWorkUnit> wu = factory->updateWorkUnit(browseWuid);
+        Owned<IWUResult> lower = wu->updateVariableByName(LOWER_LIMIT_ID);
+        lower->setResultInt(startOffset);
+        lower->setResultStatus(ResultStatusSupplied);
+
+        Owned<IWUResult> dataResult = wu->updateResultBySequence(0);
+        dataResult->setResultRaw(0, NULL, ResultFormatRaw);
+        dataResult->setResultStatus(ResultStatusUndefined);
+        wu->clearExceptions();
+        if (wu->getState() != WUStateUnknown)
+            wu->setState(WUStateCompiled);
+
+        //Owned<IWUResult> count = wu->updateVariableByName(RECORD_LIMIT_ID);
+        //count->setResultInt64(fetchSize);
+    }
+
+    //Resubmit the query...
+    submitWorkUnit(browseWuid, username, password);
+    WUState finalState = waitForWorkUnitToComplete(browseWuid, -1, true);
+    if(!((finalState == WUStateCompleted) || (finalState == WUStateWait)))
+        return false;
+
+    //Now extract the results...
+    Owned<IWorkUnitFactory> factory = getWorkUnitFactory();
+    Owned<IConstWorkUnit> wu = factory->openWorkUnit(browseWuid, false);
+    Owned<IConstWUResult> dataResult = wu->getResultBySequence(0);
+    MemoryBuffer2IDataVal xxx(temp); dataResult->getResultRaw(xxx, NULL, NULL);
+
+    if (temp.length() == 0)
+        return false;
+
+    RowBlock * rows;
+    if (returnedMeta->isFixedSize())
+        rows = new FilePosFixedRowBlock(temp, startRow, startOffset, returnedMeta->fixedSize());
+    else
+        rows = new FilePosVariableRowBlock(temp, startRow, startOffset, returnedMeta, true);
+    cache.addRowsOwn(rows);
+    return true;
+}

common/fileview2/fvquerysource.ipp

@@ -0,0 +1,51 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVQUERYSOURCE_IPP
+#define FVQUERYSOURCE_IPP
+
+#include "fvdatasource.hpp"
+#include "dllserver.hpp"
+#include "hqlexpr.hpp"
+#include "eclhelper.hpp"
+
+#include "fvsource.ipp"
+#include "dadfs.hpp"
+
+class QueryDataSource : public PagedDataSource
+{
+public:
+    QueryDataSource(IConstWUResult * _wuResult, const char * _wuid, const char * _username, const char * _password);
+    ~QueryDataSource();
+
+    virtual bool init();
+    virtual bool loadBlock(__int64 startRow, offset_t startOffset);
+
+protected:
+    bool createBrowseWU();
+    virtual void improveLocation(__int64 row, RowLocation & location);
+
+protected:
+    StringAttr browseWuid;
+    StringAttr cluster;
+    StringAttr username;
+    StringAttr password;
+};
+
+
+#endif

common/fileview2/fvrelate.cpp

@@ -0,0 +1,857 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "eclrtl.hpp"
+
+#include "fileview.hpp"
+#include "fverror.hpp"
+#include "fvrelate.ipp"
+#include "deftype.hpp"
+#include "fvresultset.ipp"
+
+//---------------------------------------------------------------------------
+
+void ViewFile::addRelation(ViewRelation & _relation) 
+{ 
+    relations.append(_relation); 
+}
+
+void ViewFile::addSuperFile(ViewFile & _superFile) 
+{ 
+    if (!superFiles.contains(_superFile))
+        superFiles.append(_superFile); 
+}
+
+void ViewFile::addSubFile(ViewFile & _subFile) 
+{
+    throwUnexpected();
+}
+
+const char * ViewFile::queryFilename() const
+{
+    return definition->queryLogicalName();
+}
+
+
+INewResultSet * ViewFile::queryResultSet()
+{
+    if (!cachedResultSet)
+    {
+        //This used to protect against exceptions - but complicated indexes now complain when the cursor is created, so no need to protect this.
+        cachedResultSet.setown(options.resultSetFactory->createNewFileResultSet(queryFilename(), options.cluster));
+    }
+
+    return cachedResultSet;
+}
+
+
+void ViewSuperFile::addSubFile(ViewFile & _subFile) 
+{ 
+    if (!subFiles.contains(_subFile)) 
+        subFiles.append(_subFile); 
+}
+
+//---------------------------------------------------------------------------
+
+void ViewJoinColumnMapping::addFilterToPrimary(IFilteredResultSet * primaryResultSet, IResultSetCursor * secondaryCursor)
+{
+    primary->clearFilter(primaryResultSet);
+    if (secondaryIteratorColumn != NotFound)
+    {
+        Owned<IResultSetCursor> childCursor = secondaryCursor->getChildren(secondaryIteratorColumn);
+        if (childCursor->first())
+        {
+            do
+            {
+                doAddFilterToPrimary(primaryResultSet, childCursor);
+            } while (childCursor->next());
+        }
+    }
+    else
+        doAddFilterToPrimary(primaryResultSet, secondaryCursor);
+}
+
+void ViewJoinColumnMapping::addFilterToSecondary(IFilteredResultSet * secondaryResultSet, IResultSetCursor * primaryCursor)
+{
+    assertex(secondaryIteratorColumn == NotFound);
+    MemoryAttr value;
+    primary->getValue(value, primaryCursor);
+    secondary->clearFilter(secondaryResultSet);
+    secondary->addFilter(secondaryResultSet, value);
+}
+
+void ViewJoinColumnMapping::doAddFilterToPrimary(IFilteredResultSet * primaryResultSet, IResultSetCursor * secondaryCursor)
+{
+    MemoryAttr value;
+    secondary->getValue(value, secondaryCursor);
+    primary->addFilter(primaryResultSet, value);
+}
+
+bool ViewJoinColumnMapping::canMapPrimaryToSecondary() const
+{
+    return (secondaryIteratorColumn == NotFound) && primary->canGet() && secondary->canSet();
+}
+
+bool ViewJoinColumnMapping::canMapSecondaryToPrimary() const
+{
+    return primary->canSet() && secondary->canGet();
+}
+
+//---------------------------------------------------------------------------
+
+inline bool isSameString(const char * left, const char * right)
+{
+    if (left == right)
+        return true;
+    if (left && right)
+        return strcmp(left, right) == 0;
+    return false;
+}
+
+void ViewRelation::addFilterToPrimary(IFilteredResultSet * primaryResultSet, IResultSetCursor * secondaryCursor)
+{
+    ForEachItemIn(i, columnMappings)
+        columnMappings.item(i).addFilterToPrimary(primaryResultSet, secondaryCursor);
+}
+
+void ViewRelation::addFilterToSecondary(IFilteredResultSet * secondaryResultSet, IResultSetCursor * primaryCursor)
+{
+    ForEachItemIn(i, columnMappings)
+        columnMappings.item(i).addFilterToSecondary(secondaryResultSet, primaryCursor);
+}
+
+unsigned ViewRelation::queryMappingField(unsigned whichMapping, bool needPrimary) const
+{
+    if (whichMapping >= columnMappings.ordinality())
+        return NotFound;
+    
+    ViewJoinColumnMapping & mapping = columnMappings.item(whichMapping);
+    if (needPrimary)
+        return mapping.primary->queryBaseColumn();
+    return mapping.secondary->queryBaseColumn();
+}
+
+//Not in the constructor so exceptions can be handled cleanly.
+void ViewRelation::init()
+{
+    INewResultSet * primaryResultSet = primaryFile->queryResultSet();
+    INewResultSet * secondaryResultSet = secondaryFile->queryResultSet();
+    if (primaryResultSet && secondaryResultSet)
+    {
+        FieldTransformInfoArray primaryFields;
+        FieldTransformInfoArray secondaryFields;
+        
+        parseColumnMappingList(primaryFields, primaryResultSet->getMetaData(), false, definition->queryPrimaryFields());
+        parseColumnMappingList(secondaryFields, secondaryResultSet->getMetaData(), true, definition->querySecondaryFields());
+
+        //MORE: Check that if a secondary dataset column is specified that there is only one of them.
+        if (primaryFields.ordinality() == secondaryFields.ordinality())
+        {
+            ForEachItemIn(i, primaryFields)
+            {
+                FieldTransformInfo & curPrimary = primaryFields.item(i);
+                FieldTransformInfo & curSecondary = secondaryFields.item(i);
+
+                Owned<ViewJoinColumn> primaryColumn = new ViewJoinColumn(curPrimary.column, curPrimary.getTransforms, curPrimary.setTransforms);
+                Owned<ViewJoinColumn> secondaryColumn = new ViewJoinColumn(curSecondary.column, curSecondary.getTransforms, curSecondary.setTransforms);
+                columnMappings.append(* new ViewJoinColumnMapping(primaryColumn, secondaryColumn, curSecondary.datasetColumn));
+            }
+        }
+        else
+            throwError4(FVERR_NumJoinFieldsMismatchXY, primaryFields.ordinality(), secondaryFields.ordinality(), primaryFile->queryFilename(), secondaryFile->queryFilename());
+    }
+}
+
+
+bool ViewRelation::matches(IFileRelationship * searchDefinition, ViewFile * searchPrimaryFile, ViewFile * searchSecondaryFile) const
+{
+    if ((searchPrimaryFile != primaryFile) || (searchSecondaryFile != secondaryFile))
+        return false;
+
+    if (!isSameString(definition->queryPrimaryFields(), searchDefinition->queryPrimaryFields()))
+        return false;
+
+    if (!isSameString(definition->querySecondaryFields(), searchDefinition->querySecondaryFields()))
+        return false;
+
+    if (!isSameString(definition->queryKind(), searchDefinition->queryKind()))
+        return false;
+
+    if (definition->isPayload() != searchDefinition->isPayload())
+        return false;
+    
+    return true;
+}
+
+bool ViewRelation::canMapPrimaryToSecondary(bool isEfficient) const
+{
+    //MORE: Implement isEfficient!
+    ForEachItemIn(i, columnMappings)
+        if (!columnMappings.item(i).canMapPrimaryToSecondary())
+            return false;
+
+    return true;
+}
+
+bool ViewRelation::canMapSecondaryToPrimary(bool isEfficient) const
+{
+    ForEachItemIn(i, columnMappings)
+        if (!columnMappings.item(i).canMapSecondaryToPrimary())
+            return false;
+
+    return true;
+}
+
+
+void ViewRelation::queryPrimaryColumns(UnsignedArray & columns)
+{
+    ForEachItemIn(i, columnMappings)
+    {
+        ViewJoinColumnMapping & mapping = columnMappings.item(i);
+        columns.append(mapping.primary->queryBaseColumn());
+    }
+}
+
+void ViewRelation::querySecondaryColumns(UnsignedArray & columns)
+{
+    ForEachItemIn(i, columnMappings)
+    {
+        ViewJoinColumnMapping & mapping = columnMappings.item(i);
+        if (mapping.secondaryIteratorColumn != NotFound)
+            columns.append(mapping.secondaryIteratorColumn);
+        else
+            columns.append(mapping.secondary->queryBaseColumn());
+    }
+}
+
+//---------------------------------------------------------------------------
+
+unsigned BrowseCreateInfo::getNestingCount(const ViewFile * search)
+{
+    unsigned count = 0;
+    ForEachItemIn(i, nestedFiles)
+    {
+        if (&nestedFiles.item(i) == search)
+            count++;
+    }
+    return count;
+}
+
+//---------------------------------------------------------------------------
+
+ViewFileWeb::ViewFileWeb(IResultSetFactory & resultSetFactory, const char * cluster) 
+: viewOptions(resultSetFactory, cluster), directory(queryDistributedFileDirectory())
+{
+    nextId = 0;
+}
+
+
+ViewFile * ViewFileWeb::addFile(IDistributedFile * definition)
+{
+    ViewFile * file;
+    if (definition->querySuperFile())
+        file = new ViewSuperFile(++nextId, viewOptions, definition);
+    else
+        file = new ViewFile(++nextId, viewOptions, definition);
+    files.append(*file);
+    return file;
+}
+
+
+ViewRelation * ViewFileWeb::addRelation(IFileRelationship * definition, ViewFile * primaryFile, ViewFile * secondaryFile)
+{
+    ForEachItemIn(i, relations)
+    {
+        ViewRelation & cur = relations.item(i);
+        if (cur.matches(definition, primaryFile, secondaryFile))
+            return &cur;
+    }
+
+    Owned<ViewRelation> relation = new ViewRelation(++nextId, definition, primaryFile, secondaryFile);
+    relation->init();
+    relations.append(*LINK(relation));
+    primaryFile->addRelation(*relation);
+    secondaryFile->addRelation(*relation);
+    return relation;
+}
+
+
+IFileTreeBrowser * ViewFileWeb::createBrowseTree(const char * browseRootFilename, bool isEfficient, unsigned maxRecursion)
+{
+    ViewFile * root = findFile(browseRootFilename);
+    if (!root)
+        throwError1(FVERR_CouldNotResolveX, browseRootFilename);
+
+    CFileTreeBrowser * browser = new CFileTreeBrowser(this);
+    BrowseCreateInfo info(browser, isEfficient, maxRecursion);
+    gatherBrowseFiles(info, root, NULL, NULL);
+    return browser;
+}
+
+ViewFile * ViewFileWeb::findFile(const char * filename)
+{
+    ForEachItemIn(i, files)
+    {
+        ViewFile & cur = files.item(i);
+        if (cur.matches(filename))
+            return &cur;
+    }
+    return NULL;
+}
+
+void ViewFileWeb::gatherBrowseFiles(BrowseCreateInfo & info, ViewFile * file, CRelatedBrowseFile * parentFile, ViewRelation * parentRelation)
+{
+    if (info.getNestingCount(file) >= info.maxRecursion)
+        return;
+
+    CRelatedBrowseFile * browseFile = info.browser->addFile(file, parentFile, parentRelation);
+    info.nestedFiles.append(*file);
+    ForEachItemIn(i2, file->relations)
+    {
+        ViewRelation & cur = file->relations.item(i2);
+        if (&cur != parentRelation)
+        {
+            if (stricmp(cur.queryDefinition()->queryKind(), S_LINK_RELATIONSHIP_KIND) != 0)
+                continue;
+
+            //Check for secondary first - so that recursive definitions default to secondary->primary
+            if (file == cur.querySecondary())
+            {
+                if (cur.canMapSecondaryToPrimary(info.isEfficient))
+                    gatherBrowseFiles(info, cur.queryPrimary(), browseFile, &cur);
+            }
+            else if (file == cur.queryPrimary())
+            {
+                if (cur.canMapPrimaryToSecondary(info.isEfficient))
+                    gatherBrowseFiles(info, cur.querySecondary(), browseFile, &cur);
+            }
+        }
+    }
+    info.nestedFiles.pop();
+}
+
+
+void ViewFileWeb::gatherWeb(const char * rootFilename, const ViewGatherOptions & options)
+{
+    ViewWalkOptions localOptions(options);
+    if (!localOptions.kind)
+        localOptions.kind = S_LINK_RELATIONSHIP_KIND;
+    localOptions.isExplicitFile = true;
+
+    if (!walkFile(rootFilename, NULL, localOptions))
+        throwError1(FVERR_CouldNotResolveX, rootFilename);
+
+    //MORE: Should possibly percolate relations between superfiles down to files they contain.
+}
+
+
+void ViewFileWeb::gatherWebFromPattern(const char * filenamePattern, const ViewGatherOptions & options)
+{
+    ViewWalkOptions localOptions(options);
+    if (!localOptions.kind)
+        localOptions.kind = S_LINK_RELATIONSHIP_KIND;
+
+    Owned<IDistributedFileIterator> iter = queryDistributedFileDirectory().getIterator(filenamePattern, false, NULL);
+    if (iter->first())
+    {
+        do
+        {
+            IDistributedFile & cur = iter->query();
+            localOptions.isExplicitFile = true;
+            walkFile(cur.queryLogicalName(), &cur, localOptions);
+        } while (iter->next());
+    }
+    else
+        throwError1(FVERR_CouldNotResolveX, filenamePattern);
+
+    //MORE: Should possibly percolate relations between superfiles down to files they contain.
+}
+
+
+IViewRelatedFileIterator * ViewFileWeb::getFileIterator()
+{
+    return new CViewRelatedFileIterator(files, 0, this);
+}
+
+
+IViewRelatedFile * ViewFileWeb::queryFile(unsigned i)
+{
+    if (files.isItem(i))
+        return &files.item(i);
+    return NULL;
+}
+
+void ViewFileWeb::walk(IViewFileWebVisitor & visitor)
+{
+    ForEachItemIn(i1, files)
+        visitor.visit(&files.item(i1));
+
+    ForEachItemIn(i2, relations)
+        visitor.visit(&relations.item(i2));
+}
+
+
+ViewFile * ViewFileWeb::walkFile(const char * filename, IDistributedFile * alreadyResolved, ViewWalkOptions & options)
+{
+    ViewFile * thisFile = findFile(filename);
+    if (thisFile)
+        return thisFile;
+
+    if (options.explicitFilesOnly)
+    {
+        if (!options.isExplicitFile)
+            return NULL;
+        options.isExplicitFile = false;
+    }
+
+    Owned<IDistributedFile> resolved = alreadyResolved ? LINK(alreadyResolved) : directory.lookup(filename);
+    if (!resolved)
+        return NULL;
+
+    thisFile = addFile(resolved);
+    if (options.maximumDepth > 0)
+    {
+        options.maximumDepth--;
+
+        if (options.primaryDepth > 0)
+        {
+            options.primaryDepth--;
+            Owned<IFileRelationshipIterator> iter = directory.lookupFileRelationships(NULL, filename, NULL, NULL, options.kind, NULL, options.payload);
+            ForEach(*iter)
+            {
+                IFileRelationship & cur = iter->query();
+                ViewFile * otherFile = walkFile(cur.queryPrimaryFilename(), NULL, options);
+                if (otherFile)
+                {
+                    addRelation(&cur, otherFile, thisFile);
+                }
+            }
+            options.primaryDepth++;
+        }
+
+        if (options.secondaryDepth > 0)
+        {
+            options.secondaryDepth--;
+            Owned<IFileRelationshipIterator> iter = directory.lookupFileRelationships(filename, NULL, NULL, NULL, options.kind, NULL, options.payload);
+            ForEach(*iter)
+            {
+                IFileRelationship & cur = iter->query();
+                ViewFile * otherFile = walkFile(cur.querySecondaryFilename(), NULL, options);
+                if (otherFile)
+                {
+                    addRelation(&cur, thisFile, otherFile);
+                }
+            }
+            options.primaryDepth++;
+        }
+
+        if ((options.superDepth > 0) && resolved->isSubFile())
+        {
+            options.superDepth--;
+            Owned<IDistributedSuperFileIterator> iter = resolved->getOwningSuperFiles();
+            ForEach(*iter)
+            {
+                IDistributedSuperFile & cur = iter->query();
+                //ViewFile * otherFile = walkFile(cur.queryLogicalName(), &cur, options):       // could pass in to save lookup
+                ViewFile * otherFile = walkFile(cur.queryLogicalName(), &cur, options);
+                if (otherFile)
+                {
+                    thisFile->addSuperFile(*otherFile);
+                    otherFile->addSubFile(*thisFile);
+                }
+            }
+            options.superDepth++;
+        }
+
+        IDistributedSuperFile * super = resolved->querySuperFile();
+        if ((options.subDepth > 0) && super)
+        {
+            options.subDepth--;
+            Owned<IDistributedFileIterator> iter = super->getSubFileIterator(false);
+            ForEach(*iter)
+            {
+                IDistributedFile & cur = iter->query();
+                ViewFile * otherFile = walkFile(cur.queryLogicalName(), &cur, options);
+                if (otherFile)
+                {
+                    thisFile->addSubFile(*otherFile);
+                    otherFile->addSuperFile(*thisFile);
+                }
+            }
+            options.subDepth++;
+        }
+        options.maximumDepth--;
+    }
+    return thisFile;
+}
+
+//---------------------------------------------------------------------------
+
+FilteredSecondaryResultSetCursor::FilteredSecondaryResultSetCursor(INewResultSet * _resultSet, IResultSetCursor * _primary, ViewRelation * _relation)
+: DelayedFilteredResultSetCursor(_resultSet)
+{
+    primary = _primary;
+    relation = _relation;
+}
+
+void FilteredSecondaryResultSetCursor::noteRelatedFileChanged()
+{
+    DelayedFilteredResultSetCursor::noteRelatedFileChanged();
+    clearFilters();
+}
+
+void FilteredSecondaryResultSetCursor::ensureFiltered()
+{
+    relation->addFilterToSecondary(filtered, primary);
+}
+
+
+//---------------------------------------------------------------------------
+
+FilteredPrimaryResultSetCursor::FilteredPrimaryResultSetCursor(INewResultSet * _resultSet, IResultSetCursor * _secondary, ViewRelation * _relation)
+: DelayedFilteredResultSetCursor(_resultSet)
+{
+    secondary = _secondary;
+    relation = _relation;
+}
+
+void FilteredPrimaryResultSetCursor::noteRelatedFileChanged()
+{
+    DelayedFilteredResultSetCursor::noteRelatedFileChanged();
+
+    clearFilters();
+}
+
+void FilteredPrimaryResultSetCursor::ensureFiltered()
+{
+    relation->addFilterToPrimary(filtered, secondary);
+}
+
+
+
+//---------------------------------------------------------------------------
+
+CRelatedBrowseFile::CRelatedBrowseFile(ViewFile * _file, CRelatedBrowseFile * _parent, ViewRelation * _relation)
+{
+    file = _file;
+    parent = _parent;
+    relation = _relation;
+
+    INewResultSet * resultSet = file->queryResultSet();
+    if (!resultSet)
+        throwError1(FVERR_CannotBrowseFile, file->queryFilename());
+
+    if (!parent)
+    {
+        filtered.setown(new DelayedFilteredResultSetCursor(resultSet));
+    }
+    else
+    {
+        if (file == relation->querySecondary())
+        {
+            filtered.setown(new FilteredSecondaryResultSetCursor(resultSet, parent->queryCursor(), relation));
+        }
+        else
+        {
+            filtered.setown(new FilteredPrimaryResultSetCursor(resultSet, parent->queryCursor(), relation));
+        }
+    }
+    notifier.setown(new NotifyingResultSetCursor(filtered));
+}
+
+void CRelatedBrowseFile::addChild(CRelatedBrowseFile * child)
+{
+    children.append(*child);
+    notifier->addDependent(*child->notifier);
+}
+
+
+IViewRelatedFile * CRelatedBrowseFile::queryDefinition() const
+{
+    return file;
+}
+
+IRelatedBrowseFile * CRelatedBrowseFile::queryParent() const
+{
+    return parent;
+}
+
+IViewRelation * CRelatedBrowseFile::queryParentRelation() const
+{
+    return relation;
+}
+
+IRelatedBrowseFile * CRelatedBrowseFile::queryChild(unsigned i) const
+{
+    if (children.isItem(i))
+        return &children.item(i);
+    return NULL;
+}
+
+IResultSetCursor * CRelatedBrowseFile::queryCursor()
+{
+    return notifier;
+}
+
+//---------------------------------------------------------------------------
+
+CFileTreeBrowser::CFileTreeBrowser(ViewFileWeb * _web) : web(_web)
+{
+}
+
+CRelatedBrowseFile * CFileTreeBrowser::addFile(ViewFile * file, CRelatedBrowseFile * parent, ViewRelation * relation)
+{
+    CRelatedBrowseFile * next = new CRelatedBrowseFile(file, parent, relation);
+    files.append(*next);
+    if (parent)
+        parent->addChild(next);
+
+    return next;
+}
+
+IRelatedBrowseFile * CFileTreeBrowser::queryRootFile()
+{
+    return &files.item(0);
+}
+
+IResultSetFilter * CFileTreeBrowser::queryRootFilter()
+{
+    return &files.item(0);
+}
+
+//---------------------------------------------------------------------------
+
+void ViewERdiagramVisitor::visit(ViewFile * file)
+{
+    activeFileId.clear().append(file->queryId());
+    activeFieldCount = 0;
+
+    IDistributedFile * definition = file->queryDefinition();
+    builder.beginFile(activeFileId, file->queryFilename(), definition);
+
+    activeFieldPrefix.clear().append(activeFileId).append("_");
+
+    INewResultSet * resultSet = file->queryResultSet();
+    if (resultSet)
+    {
+        const IResultSetMetaData & imeta = resultSet->getMetaData();
+        //Cheat and use an existing internal function
+        const CResultSetMetaData & meta = static_cast<const CResultSetMetaData &>(imeta);
+        meta.getXmlSchema(*this, false);
+    }
+    else
+    {
+        StringBuffer dummyFieldId;
+        dummyFieldId.append(activeFieldPrefix).append(0);
+        builder.noteField(dummyFieldId, "**field-information-unavailable**", "");
+    }
+
+    builder.endFile();
+}
+
+void ViewERdiagramVisitor::visit(ViewRelation * relation)
+{
+    UnsignedArray primaryColumns;
+    UnsignedArray secondaryColumns;
+    relation->queryPrimaryColumns(primaryColumns);
+    relation->querySecondaryColumns(secondaryColumns);
+
+    unsigned sourceFieldId;
+    unsigned targetFieldId;
+    if (primaryColumns.ordinality())
+    {
+        sourceFieldId = primaryColumns.item(0);
+        targetFieldId = secondaryColumns.item(0);
+    }
+    else
+    {
+        sourceFieldId = relation->queryPrimary()->queryFirstFieldId();
+        targetFieldId = relation->querySecondary()->queryFirstFieldId();
+    }
+
+    StringBuffer id, sourceId, targetId;
+    id.append(relation->queryId());
+    sourceId.append(relation->queryPrimary()->queryId()).append("_").append(sourceFieldId);
+    targetId.append(relation->querySecondary()->queryId()).append("_").append(targetFieldId);
+    builder.noteRelation(id, sourceId, targetId, relation->queryDefinition());
+}
+
+
+void ViewERdiagramVisitor::addField(const char * name, ITypeInfo & type)
+{
+    StringBuffer eclTypeName;
+    noteNextField();
+    type.getECLType(eclTypeName);
+    if (name && (stricmp(name, "__fileposition__") == 0))
+        builder.noteField(activeFieldId.str(), "{virtual(fileposition)}", eclTypeName);
+    else
+        builder.noteField(activeFieldId.str(), name, eclTypeName);
+}
+
+void ViewERdiagramVisitor::addSetField(const char * name, const char * itemname, ITypeInfo & type)
+{
+    addField(name, type);
+}
+
+void ViewERdiagramVisitor::beginIfBlock()
+{
+    activeFieldCount++;
+}
+
+bool ViewERdiagramVisitor::beginDataset(const char * name, const char * childname)
+{
+    noteNextField();
+    builder.noteField(activeFieldId.str(), name, "dataset");
+    savedFieldCounts.append(activeFieldCount);
+    savedFieldCounts.append(activeFieldPrefix.length());
+    activeFieldPrefix.append(activeFieldCount).append("_");
+    activeFieldCount = 0;
+    return false;       // don't expand contents
+}
+
+void ViewERdiagramVisitor::beginRecord(const char * name)
+{
+    noteNextField();
+    builder.noteField(activeFieldId.str(), name, "record");
+    builder.beginCompound();
+}
+
+void ViewERdiagramVisitor::endIfBlock()
+{
+    activeFieldCount++;
+}
+
+void ViewERdiagramVisitor::endDataset(const char * name, const char * childname)
+{
+    activeFieldPrefix.setLength(savedFieldCounts.pop());
+    activeFieldCount = savedFieldCounts.pop();
+}
+
+void ViewERdiagramVisitor::endRecord(const char * name)
+{
+    builder.endCompound();
+    activeFieldCount++;
+}
+
+void ViewERdiagramVisitor::noteNextField()
+{
+    activeFieldId.clear().append(activeFieldPrefix).append(activeFieldCount++);
+}
+
+//---------------------------------------------------------------------------
+
+ViewXgmmlERdiagramVisitor & ViewXgmmlERdiagramVisitor::addAttr(const char * name, const char * value)
+{
+    xgmml.append(" ").append(name).append("=\"");
+    encodeXML(value, xgmml, ENCODE_NEWLINES, (unsigned)-1, false);
+    xgmml.append("\"");
+    return *this;
+}
+
+ViewXgmmlERdiagramVisitor & ViewXgmmlERdiagramVisitor::addAttrTag(const char * name, const char * value)
+{
+    if (value && *value)
+    {
+        xgmml.append("<attr");
+        addAttr("name", name).addAttr("value", value);
+        xgmml.append("/>").newline();
+    }
+    return *this;
+}
+
+void ViewXgmmlERdiagramVisitor::beginFile(const char * id, const char * name, IDistributedFile * definition)
+{
+    xgmml.append(" <node");
+    addAttr("id", id).addAttr("label", name);
+    xgmml.append("><att>").newline();
+    xgmml.append(" <graph>").newline();
+}
+
+void ViewXgmmlERdiagramVisitor::noteField(const char * id, const char * name, const char * type)
+{
+    xgmml.append(" <node id=\"").append(id).append("\" label=\"").append(name).append("\">").newline();
+    addAttrTag("type", type);
+    xgmml.append(" </node>").newline();
+}
+
+void ViewXgmmlERdiagramVisitor::endFile()
+{
+    xgmml.append(" </graph>").newline();
+    xgmml.append(" </att></node>").newline();
+}
+
+void ViewXgmmlERdiagramVisitor::noteRelation(const char * id, const char * primaryId, const char * secondaryId, IFileRelationship * definition)
+{
+    StringBuffer temp;
+
+    //Note we want the direction from the secondary to the primary, so need to reverse some things
+    xgmml.append("  <edge");
+    addAttr("id", id).addAttr("source", secondaryId).addAttr("target", primaryId);
+
+    const char * kind = definition->queryKind();
+    if (kind && (stricmp(kind, S_VIEW_RELATIONSHIP_KIND) == 0))
+        addAttr("label", "view of");
+    xgmml.append(">").newline();
+
+    getInvertedCardinality(temp.clear(), definition->queryCardinality());
+    addAttrTag("cardinality", temp.str());
+    xgmml.append("  </edge>").newline();
+}
+
+
+void ViewXgmmlERdiagramVisitor::beginDiagram()
+{
+    xgmml.append("<xgmml>").newline();
+    xgmml.append("<graph>").newline();
+}
+
+void ViewXgmmlERdiagramVisitor::endDiagram()
+{
+    xgmml.append("</graph>").newline();
+    xgmml.append("</xgmml>").newline();
+}
+
+
+void ViewXgmmlERdiagramVisitor::beginCompound()
+{
+    depth++;
+}
+
+void ViewXgmmlERdiagramVisitor::endCompound()
+{
+    depth--;
+}
+
+
+IViewFileWeb * createViewFileWeb(IResultSetFactory & resultSetFactory, const char * cluster)
+{
+    return new ViewFileWeb(resultSetFactory, cluster);
+}
+
+void createERdiagram(StringBuffer & xgmml, IViewFileWeb & _web)
+{
+    ViewFileWeb & web = static_cast<ViewFileWeb &>(_web);
+
+    ViewXgmmlERdiagramVisitor builder(xgmml);
+    ViewERdiagramVisitor visitor(builder);
+    builder.beginDiagram();
+    web.walk(visitor);
+    builder.endDiagram();
+}
+

common/fileview2/fvrelate.hpp

@@ -0,0 +1,133 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVRELATE_HPP
+#define FVRELATE_HPP
+
+#include "fileview.hpp"
+
+#define CHEAP_UCHAR_DEF
+#ifdef _WIN32
+typedef wchar_t UChar;
+#else //__WIN32
+typedef unsigned short UChar;
+#endif //__WIN32
+
+typedef void (*stringFieldTransformerFunction)(unsigned & tgtLen, char * & tgt, unsigned srcLen, const char * src);
+typedef void (*utf8FieldTransformerFunction)(unsigned & tgtLen, char * & tgt, unsigned srcLen, const char * src);
+typedef void (*unicodeFieldTransformerFunction)(unsigned & tgtLen, UChar * & tgt, unsigned srcLen, const UChar * src);
+
+//Registry which can be used for registering transformations that can be included in the field lists.
+//It is also used for plugins required by dlls loaded to translate alien data types
+interface IViewTransformerRegistry
+{
+public:
+    virtual void addFieldStringTransformer(const char * name, stringFieldTransformerFunction func) = 0;
+    virtual void addFieldUtf8Transformer(const char * name, utf8FieldTransformerFunction func) = 0;
+    virtual void addFieldUnicodeTransformer(const char * name, unicodeFieldTransformerFunction func) = 0;
+    virtual void addPlugins(const char * pluginname) = 0;
+};
+extern FILEVIEW_API IViewTransformerRegistry & queryTransformerRegistry();
+
+//---------------------------------------------------------------------------
+
+struct ViewGatherOptions
+{
+    inline ViewGatherOptions()
+    {
+        primaryDepth = 100;
+        secondaryDepth = 100;
+        maximumDepth = 200;
+        kind = NULL;            // defaults to link
+        superDepth = 0;
+        subDepth = 0;
+        payload = NULL;
+        requirePayload = false;
+        explicitFilesOnly = false;
+    }
+
+    inline void setPayloadFilter(bool _value)
+    {
+        requirePayload = _value;
+        payload = &requirePayload;
+    }
+
+    unsigned primaryDepth;              // How many secondary->primary links to follow
+    unsigned secondaryDepth;            // How many primary->secondary links to follow
+    unsigned maximumDepth;              // maximum number of links
+    const char * kind;
+    unsigned superDepth;                // How many super files to walk up
+    unsigned subDepth;
+    bool * payload;
+    bool requirePayload;
+    bool explicitFilesOnly;             // If true, only files supplied to gatherFile/gatherFilePattern will be included
+};
+
+//---------------------------------------------------------------------------
+
+interface IFileTreeBrowser;
+interface IFileRelationship;
+
+interface IViewRelatedFile : public IInterface
+{
+    virtual IDistributedFile * queryDistributedFile() const = 0;
+    virtual INewResultSet * queryResultSet() = 0;
+};
+
+interface IViewRelatedFileIterator: public IIteratorOf<IViewRelatedFile> {};
+
+interface IViewRelation : public IInterface
+{
+    virtual IFileRelationship * queryFileRelationship() const = 0;
+    virtual unsigned numMappingFields() const = 0;
+    virtual unsigned queryMappingField(unsigned whichMapping, bool needPrimary) const = 0;
+};
+
+interface IViewFileWeb : public IInterface
+{
+public:
+    virtual void gatherWeb(const char * rootFilename, const ViewGatherOptions & options) = 0;
+    virtual void gatherWebFromPattern(const char * pattern, const ViewGatherOptions & options) = 0;
+    virtual IViewRelatedFile * queryFile(unsigned i) = 0;
+    virtual IViewRelatedFileIterator * getFileIterator() = 0;
+
+    //Create a tree of related file cursors starting from a particular file.
+    //Only files that can be reached will be added to the view tree.
+    virtual IFileTreeBrowser * createBrowseTree(const char * browseRootFilename, bool isEfficient=true, unsigned maxRecursion=3) = 0;
+};
+
+interface IRelatedBrowseFile
+{
+public:
+    virtual IViewRelatedFile * queryDefinition() const = 0;
+    virtual IViewRelation * queryParentRelation() const = 0;
+    virtual IRelatedBrowseFile * queryParent() const = 0;
+    virtual IRelatedBrowseFile * queryChild(unsigned i) const = 0;
+    virtual IResultSetCursor * queryCursor() = 0;
+};
+
+interface IFileTreeBrowser : public IInterface
+{
+    virtual IRelatedBrowseFile * queryRootFile() = 0;
+    virtual IResultSetFilter * queryRootFilter() = 0;
+};
+
+extern FILEVIEW_API IViewFileWeb * createViewFileWeb(IResultSetFactory & resultSetFactory, const char * cluster);
+extern FILEVIEW_API void createERdiagram(StringBuffer & xgmml, IViewFileWeb & web);
+
+#endif

common/fileview2/fvrelate.ipp

@@ -0,0 +1,388 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVRELATE_IPP
+#define FVRELATE_IPP
+
+#include "fvrelate.hpp"
+#include "dadfs.hpp"
+#include "deftype.hpp"
+#include "jiter.ipp"
+#include "fvresultset.ipp"
+
+class ViewFile;
+class ViewRelation;
+typedef IArrayOf<ViewFile> ViewFileArray;
+typedef CIArrayOf<ViewRelation> ViewRelationArray;
+typedef CopyCIArrayOf<ViewFile> ViewFileCopyArray;
+typedef CopyCIArrayOf<ViewRelation> ViewRelationCopyArray;
+typedef ICopyArrayOf<IAtom> AtomArray;
+
+//---------------------------------------------------------------------------
+
+//Common options which are required by the ViewFile class - e.g., top create a INewResultSet
+struct ViewProcessOptions
+{
+public:
+    ViewProcessOptions(IResultSetFactory & _resultSetFactory, const char * _cluster)
+        : resultSetFactory(&_resultSetFactory), cluster(_cluster)
+    {
+    }
+
+public:
+    Linked<IResultSetFactory> resultSetFactory;
+    StringAttr cluster;
+};
+
+
+//---------------------------------------------------------------------------
+
+
+//Note: All files and relationships are owned by the ViewFileWeb class - otherwise there would
+//be circular link counted references, so it would never be released.
+class FILEVIEW_API ViewFile : public CInterface, implements IViewRelatedFile
+{
+    friend class ViewFileWeb;
+public:
+    ViewFile(unsigned _uid, ViewProcessOptions & _options, IDistributedFile * _definition) : uid(_uid), options(_options), definition(_definition) {}
+    IMPLEMENT_IINTERFACE
+
+    virtual IDistributedFile * queryDistributedFile() const { return definition; }
+    virtual INewResultSet * queryResultSet();
+
+    void addRelation(ViewRelation & _relation);
+    void addSuperFile(ViewFile & _superFile);
+    virtual void addSubFile(ViewFile & _subFile);
+    const char * queryFilename() const;
+
+    inline IDistributedFile * queryDefinition() { return definition; }
+    inline unsigned queryFirstFieldId() { return 0; }           // MORE: Look at the meta and deduce - not correct if it starts with a record
+    inline unsigned queryId() const { return uid; }
+    inline bool matches(const char * search) const { return stricmp(search, queryFilename()) == 0; }
+
+protected:
+    const ViewProcessOptions & options;
+    Linked<IDistributedFile> definition;
+    Linked<INewResultSet> cachedResultSet;
+    unsigned uid;
+    ViewRelationCopyArray relations;
+    ViewFileCopyArray superFiles;
+};
+
+class FILEVIEW_API ViewSuperFile : public ViewFile
+{
+public:
+    ViewSuperFile(unsigned _uid, ViewProcessOptions & _options, IDistributedFile * _definition) : ViewFile(_uid, _options, _definition) {}
+
+    virtual void addSubFile(ViewFile & _subFile);
+
+protected:
+    ViewFileCopyArray subFiles;
+};
+
+
+class FILEVIEW_API ViewJoinColumnMapping : public CInterface
+{
+    friend class ViewRelation;
+public:
+    ViewJoinColumnMapping(ViewJoinColumn * _primary, ViewJoinColumn * _secondary, unsigned _secondaryIteratorColumn)
+        : primary(_primary), secondary(_secondary), secondaryIteratorColumn(_secondaryIteratorColumn)
+    {}
+
+    void addFilterToPrimary(IFilteredResultSet * primaryResultSet, IResultSetCursor * secondaryCursor);
+    void addFilterToSecondary(IFilteredResultSet * secondaryResultSet, IResultSetCursor * primaryCursor);
+
+    bool canMapPrimaryToSecondary() const;
+    bool canMapSecondaryToPrimary() const;
+
+protected:
+    void doAddFilterToPrimary(IFilteredResultSet * primaryResultSet, IResultSetCursor * secondaryCursor);
+
+protected:
+    Linked<ViewJoinColumn> primary;
+    Linked<ViewJoinColumn> secondary;
+    unsigned secondaryIteratorColumn;
+};
+
+
+class FILEVIEW_API ViewRelation : public CInterface, implements IViewRelation
+{
+public:
+    ViewRelation(unsigned _uid, IFileRelationship * _definition, ViewFile * _primaryFile, ViewFile * _secondaryFile)
+        : uid(_uid), definition(_definition), primaryFile(_primaryFile), secondaryFile(_secondaryFile)
+    {
+    }
+    IMPLEMENT_IINTERFACE
+
+    virtual IFileRelationship * queryFileRelationship() const { return definition; }
+    virtual unsigned numMappingFields() const { return columnMappings.ordinality(); }
+    virtual unsigned queryMappingField(unsigned whichMapping, bool needPrimary) const;
+
+    void init();
+    bool matches(IFileRelationship * _definition, ViewFile * _primaryFile, ViewFile * _secondaryFile) const;
+
+    void queryPrimaryColumns(UnsignedArray & columns);
+    void querySecondaryColumns(UnsignedArray & columns);
+
+    void addFilterToPrimary(IFilteredResultSet * primaryResultSet, IResultSetCursor * secondaryCursor);
+    void addFilterToSecondary(IFilteredResultSet * secondaryResultSet, IResultSetCursor * primaryCursor);
+
+    bool canMapPrimaryToSecondary(bool isEfficient) const;
+    bool canMapSecondaryToPrimary(bool isEfficient) const;
+
+    inline IFileRelationship * queryDefinition() const { return definition; }
+    inline unsigned queryId() const { return uid; }
+    inline ViewFile * queryPrimary() const { return primaryFile; }
+    inline ViewFile * querySecondary() const { return secondaryFile; }
+
+protected:
+    unsigned uid;
+    Linked<IFileRelationship> definition;
+    CIArrayOf<ViewJoinColumnMapping> columnMappings;
+    ViewFile * primaryFile;
+    ViewFile * secondaryFile;
+};
+
+interface IViewFileWebVisitor
+{
+public:
+    virtual void visit(ViewFile * file) = 0;
+    virtual void visit(ViewRelation * relation) = 0;
+};
+
+typedef CArrayIteratorOf<IViewRelatedFile, IViewRelatedFileIterator> CViewRelatedFileIterator;
+
+class CFileTreeBrowser;
+struct BrowseCreateInfo
+{
+public:
+    inline BrowseCreateInfo(CFileTreeBrowser * _browser, bool _isEfficient, unsigned _maxRecursion) 
+        : browser(_browser), isEfficient(_isEfficient), maxRecursion(_maxRecursion) {}
+
+    unsigned getNestingCount(const ViewFile * search);
+    
+public:
+    CFileTreeBrowser * browser;
+    unsigned maxRecursion;
+    bool isEfficient;
+    CopyCIArrayOf<ViewFile> nestedFiles;
+};
+
+struct ViewWalkOptions : public ViewGatherOptions
+{
+    ViewWalkOptions(const ViewGatherOptions & _options) : ViewGatherOptions(_options) { isExplicitFile = false; }
+
+    bool isExplicitFile;
+};
+
+class CRelatedBrowseFile;
+class FILEVIEW_API ViewFileWeb : public CInterface, implements IViewFileWeb
+{
+public:
+    ViewFileWeb(IResultSetFactory & resultSetFactory, const char * cluster);
+    IMPLEMENT_IINTERFACE
+
+    virtual void gatherWeb(const char * rootFilename, const ViewGatherOptions & options);
+    virtual void gatherWebFromPattern(const char * filenamePattern, const ViewGatherOptions & options);
+    virtual IViewRelatedFileIterator * getFileIterator();
+    virtual IViewRelatedFile * queryFile(unsigned i);
+    virtual IFileTreeBrowser * createBrowseTree(const char * browseRootFilename, bool isEfficient, unsigned maxRecursion);
+
+    ViewFile * findFile(const char * filename);
+    void walk(IViewFileWebVisitor & visitor);
+
+protected:
+    ViewFile * addFile(IDistributedFile * definition);
+    ViewRelation * addRelation(IFileRelationship * _definition, ViewFile * _primaryFile, ViewFile * _secondaryFile);
+
+    void gatherBrowseFiles(BrowseCreateInfo & info, ViewFile * file, CRelatedBrowseFile * parentFile, ViewRelation * parentRelation);
+    ViewFile * walkFile(const char * filename, IDistributedFile * alreadyResolved, ViewWalkOptions & options);
+
+protected:
+    unsigned nextId;
+    ViewProcessOptions viewOptions;
+    IDistributedFileDirectory & directory;
+    ViewFileArray files;
+    ViewRelationArray relations;
+};
+
+//---------------------------------------------------------------------------
+
+class FilteredSecondaryResultSetCursor : public DelayedFilteredResultSetCursor
+{
+public:
+    FilteredSecondaryResultSetCursor(INewResultSet * _resultSet, IResultSetCursor * _primary, ViewRelation * _relation);
+
+    virtual void noteRelatedFileChanged();
+    virtual void ensureFiltered();
+
+protected:
+    IResultSetCursor * primary;
+    ViewRelation * relation;
+};
+
+
+class FilteredPrimaryResultSetCursor : public DelayedFilteredResultSetCursor
+{
+public:
+    FilteredPrimaryResultSetCursor(INewResultSet * _resultSet, IResultSetCursor * _secondary, ViewRelation * _relation);
+
+    virtual void noteRelatedFileChanged();
+    virtual void ensureFiltered();
+
+protected:
+    IResultSetCursor * secondary;
+    ViewRelation * relation;
+};
+
+
+
+class CRelatedBrowseFile : public CInterface, implements IRelatedBrowseFile, implements IResultSetFilter
+{
+public:
+    CRelatedBrowseFile(ViewFile * _file, CRelatedBrowseFile * _parent, ViewRelation * _relation);
+    IMPLEMENT_IINTERFACE
+
+    void addChild(CRelatedBrowseFile * child);
+
+    virtual IViewRelatedFile * queryDefinition() const;
+    virtual IViewRelation * queryParentRelation() const;
+    virtual IRelatedBrowseFile * queryParent() const;
+    virtual IRelatedBrowseFile * queryChild(unsigned i) const;
+    virtual IResultSetCursor * queryCursor();
+
+    virtual void clearFilter(unsigned columnIndex)
+    {
+        filtered->clearFilter(columnIndex);
+    }
+    virtual void addFilter(unsigned columnIndex, const char * value)
+    {
+        filtered->addFilter(columnIndex, strlen(value), value);
+    }
+    virtual void clearFilters()
+    {
+        filtered->clearFilters();
+    }
+    virtual void addFilter(unsigned columnIndex, unsigned length, const char * utf8Value)
+    {
+        filtered->addFilter(columnIndex, length, utf8Value);
+    }
+    virtual void addNaturalFilter(unsigned columnIndex, unsigned length, const char * utf8Value)
+    {
+        filtered->addNaturalFilter(columnIndex, length, utf8Value);
+    }
+
+protected:
+    ViewFile * file;
+    CRelatedBrowseFile * parent;
+    ViewRelation * relation;
+    Owned<DelayedFilteredResultSetCursor> filtered;
+    Owned<NotifyingResultSetCursor> notifier;
+    CopyCIArrayOf<CRelatedBrowseFile> children;
+};
+
+
+class CFileTreeBrowser : public CInterface, implements IFileTreeBrowser
+{
+public:
+    CFileTreeBrowser(ViewFileWeb * _web);
+    IMPLEMENT_IINTERFACE
+
+    CRelatedBrowseFile * addFile(ViewFile * file, CRelatedBrowseFile * parent, ViewRelation * parentRelation);
+
+    virtual IRelatedBrowseFile * queryRootFile();
+    virtual IResultSetFilter * queryRootFilter();
+
+protected:
+    Linked<ViewFileWeb> web;
+    CIArrayOf<CRelatedBrowseFile> files;
+};
+
+//---------------------------------------------------------------------------
+
+interface IErDiagramBuilder
+{
+    virtual void beginFile(const char * id, const char * name, IDistributedFile * definition) = 0;
+    virtual void noteField(const char * id, const char * name, const char * type) = 0;
+    virtual void endFile() = 0;
+
+    virtual void beginCompound() = 0;
+    virtual void endCompound() = 0;
+
+    virtual void noteRelation(const char * id, const char * sourceId, const char * targetId, IFileRelationship * definition) = 0;
+};
+
+class FILEVIEW_API ViewERdiagramVisitor : public IViewFileWebVisitor, public ISchemaBuilder
+{
+public:
+    ViewERdiagramVisitor(IErDiagramBuilder & _builder) : builder(_builder) {}
+
+// IViewFileWebVisitor
+    virtual void visit(ViewFile * file);
+    virtual void visit(ViewRelation * relation);
+
+// ISchemaBuilder
+    virtual void addField(const char * name, ITypeInfo & type);
+    virtual void addSetField(const char * name, const char * itemname, ITypeInfo & type);
+    virtual void beginIfBlock();
+    virtual bool beginDataset(const char * name, const char * childname);
+    virtual void beginRecord(const char * name);
+    virtual void endIfBlock();
+    virtual void endDataset(const char * name, const char * childname);
+    virtual void endRecord(const char * name);
+    virtual bool addSingleFieldDataset(const char * name, const char * childname, ITypeInfo & type) { return false; }
+
+protected:
+    virtual void noteNextField();
+
+protected:
+    IErDiagramBuilder & builder;
+    StringBuffer activeFileId;
+    StringBuffer activeFieldPrefix;
+    StringBuffer activeFieldId;
+    unsigned activeFieldCount;
+    UnsignedArray savedFieldCounts; 
+};
+
+class FILEVIEW_API ViewXgmmlERdiagramVisitor : public IErDiagramBuilder
+{
+public:
+    ViewXgmmlERdiagramVisitor(StringBuffer & _xgmml) : xgmml(_xgmml) { depth = 0; }
+
+    virtual void beginFile(const char * id, const char * name, IDistributedFile * definition);
+    virtual void noteField(const char * id, const char * name, const char * type);
+    virtual void endFile();
+
+    virtual void beginCompound();
+    virtual void endCompound();
+
+    virtual void noteRelation(const char * id, const char * sourceId, const char * targetId, IFileRelationship * definition);
+
+    void beginDiagram();
+    void endDiagram();
+
+protected:
+    ViewXgmmlERdiagramVisitor & addAttr(const char * name, const char * value);
+    ViewXgmmlERdiagramVisitor & addAttrTag(const char * name, const char * value);
+
+protected:
+    StringBuffer & xgmml;
+    unsigned depth;
+};
+
+#endif

common/fileview2/fvresultset.ipp

@@ -0,0 +1,627 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVRESULTSET_IPP
+#define FVRESULTSET_IPP
+
+#include "fileview.hpp"
+#include "fvdatasource.hpp"
+#include "fvtransform.ipp"
+
+//---------------------------------------------------------------------------
+
+//Internal interface used to help implement the related dataset browsing
+interface IExtendedResultSetCursor : extends IResultSetCursor
+{
+public:
+    virtual void noteRelatedFileChanged() = 0;
+};
+
+
+//---------------------------------------------------------------------------
+
+struct MemoryAttrItem : public CInterface, public MemoryAttr
+{
+    MemoryAttrItem() : MemoryAttr() {}
+    MemoryAttrItem(unsigned len, const void * ptr) : MemoryAttr(len, ptr) {}
+
+};
+typedef CIArrayOf<MemoryAttrItem> MemoryAttrArray;
+
+class CStrColumnFilter : public CInterface
+{
+public:
+    void addValue(unsigned len, const char * value)     { values.append(* new MemoryAttrItem(len, value)); }
+    void clear()                            { values.kill(); }
+
+public:
+    MemoryAttrArray values;
+};
+typedef CIArrayOf<CStrColumnFilter> StrColumnFilterArray;
+
+//---------------------------------------------------------------------------
+
+class CResultSet;
+class CResultSetCursor;
+class ViewJoinColumn;
+
+class CResultSetColumnInfo : public CInterface
+{
+public:
+    ITypeInfo * type;
+    Owned<IResultSetMetaData> childMeta;
+    ViewFieldTransformerArray getTransforms;
+    ViewFieldTransformerArray setTransforms;
+    StringAttr naturalName;
+    byte flag;
+};
+
+class CResultSetMetaData : public CInterface, implements IResultSetMetaData
+{
+    friend class CResultSet;
+    friend class CResultSetCursor;
+public:
+    CResultSetMetaData(IFvDataSourceMetaData * _meta, bool _useXPath);
+    CResultSetMetaData(const CResultSetMetaData & _meta);
+    IMPLEMENT_IINTERFACE
+
+    virtual IResultSetMetaData * getChildMeta(int column) const;
+    virtual int getColumnCount() const;
+    virtual DisplayType getColumnDisplayType(int column) const;
+    virtual IStringVal & getColumnLabel(IStringVal & s, int column) const;
+    virtual IStringVal & getColumnEclType(IStringVal & s, int column) const;
+    virtual IStringVal & getColumnXmlType(IStringVal & s, int column) const;
+    virtual bool isSigned(int column) const;
+    virtual bool isEBCDIC(int column) const;
+    virtual bool isBigEndian(int column) const;
+    virtual unsigned getColumnRawType(int column) const;
+    virtual unsigned getColumnRawSize(int column) const;
+    virtual IStringVal & getXmlSchema(IStringVal & s, bool addHeader) const;
+    virtual IStringVal & getXmlXPathSchema(IStringVal & str, bool addHeader) const;
+    virtual unsigned getNumKeyedColumns() const;
+    virtual bool hasSetTranslation(int column) const;
+    virtual bool hasGetTranslation(int column) const;
+    virtual IStringVal & getNaturalColumnLabel(IStringVal &s, int column) const;
+    virtual bool isVirtual(int column) const;
+
+    void calcFieldOffsets(const byte * data, unsigned * offsets) const;
+
+    inline bool isFixedSize() const { return fixedSize; }
+    inline char queryFlags(int column) const                    { return columns.item(column).flag; }
+    inline const CResultSetColumnInfo & queryColumn(int column) const { return columns.item(column); }
+    inline ITypeInfo * queryType(int column) const              { return columns.item(column).type; }
+    inline const char * queryXPath(int column) const            { return meta->queryXPath(column); }
+
+    void getXmlSchema(ISchemaBuilder & builder, bool useXPath) const;
+    unsigned queryColumnIndex(unsigned firstField, const char * fieldName) const;
+
+protected:
+    IFvDataSourceMetaData * meta;
+    CIArrayOf<CResultSetColumnInfo> columns;
+    bool fixedSize;
+    bool alwaysUseXPath;
+};
+
+//Extend the Result interface with the stateless functions required by the cursors
+interface IExtendedNewResultSet : extends INewResultSet
+{
+    virtual IExtendedNewResultSet * cloneForFilter() = 0;
+    virtual bool fetch(MemoryBuffer & out, __int64 fileoffset) = 0;
+    virtual bool fetchRaw(MemoryBuffer & out, __int64 fileoffset) = 0;
+    virtual bool getRow(MemoryBuffer & out, __int64 row) = 0;
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row) = 0;
+    virtual void serialize(MemoryBuffer & out) = 0;
+    virtual bool isMappedIndexField(unsigned columnIndex) = 0;
+    virtual void onOpen() = 0;
+    virtual void onClose() = 0;
+    virtual IFvDataSource * queryDataSource() = 0;
+};
+
+class CResultSetBase : public CInterface, implements IExtendedNewResultSet
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+//interface IResultSet
+    virtual IResultSetCursor * createCursor();
+    virtual IResultSetCursor * createCursor(IDataVal & buffer);
+    virtual IFilteredResultSet * createFiltered();
+    virtual IResultSetCursor * createSortedCursor(unsigned column, bool descend);
+
+protected:
+    const CResultSetMetaData & getMeta() { return static_cast<const CResultSetMetaData &>(getMetaData()); }
+
+    CResultSetCursor * doCreateCursor(bool oldInterface);
+};
+
+
+class CResultSet : public CResultSetBase
+{
+public:
+    CResultSet(IFvDataSource * _dataSource, bool _useXPath);
+
+//interface IResultSet
+    virtual int findColumn(const char * columnName) const;
+    virtual const IResultSetMetaData & getMetaData() const;
+    virtual __int64 getNumRows() const;
+    virtual bool supportsRandomSeek() const;
+
+    virtual IExtendedNewResultSet * cloneForFilter();
+    virtual bool fetch(MemoryBuffer & out, __int64 fileoffset);
+    virtual bool fetchRaw(MemoryBuffer & out, __int64 fileoffset);
+    virtual bool getRow(MemoryBuffer & out, __int64 row);
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row);
+    virtual void serialize(MemoryBuffer & out);
+    virtual bool isMappedIndexField(unsigned columnIndex);
+    virtual void onOpen() { dataSource->onOpen(); }
+    virtual void onClose() { dataSource->onClose(); }
+    virtual IFvDataSource * queryDataSource() { return dataSource; }
+
+    inline CResultSetCursor * createOldStyleCursor() { return doCreateCursor(true); }
+
+    void setColumnMapping(IDistributedFile * df);
+
+protected:
+    void calcMappedFields();
+
+protected:
+    CResultSetMetaData  meta;
+    Linked<IFvDataSource> dataSource;
+    BoolArray mappedFields;
+    __int64 numRows;
+    CriticalSection cs;
+};
+
+
+class CColumnFilter : public CInterface
+{
+public:
+    CColumnFilter(unsigned _whichColumn, ITypeInfo * _type, bool _isMappedIndexField)
+        : whichColumn(_whichColumn)
+    { type.set(_type); optimized = false; isMappedIndexField = _isMappedIndexField; }
+
+    void addValue(unsigned lenText, const char * value);
+
+    void deserialize(MemoryBuffer & buffer);
+    bool isValid(const byte * rowValue, const unsigned * offsets);
+    bool optimizeFilter(IFvDataSource * dataSource);
+    void serialize(MemoryBuffer & buffer);
+
+protected:
+    bool matches(const byte * rowValue, unsigned valueLen, const byte * value);
+
+protected:
+    MemoryAttrArray values;
+    OwnedITypeInfo type;
+    unsigned subLen;
+    unsigned whichColumn;
+    bool optimized;
+    bool isMappedIndexField;
+};
+typedef CIArrayOf<CColumnFilter> ColumnFilterArray;
+
+
+class CIndirectResultSet : public CResultSetBase
+{
+public:
+    CIndirectResultSet(IExtendedNewResultSet * _parent) 
+        : parent(_parent), meta(static_cast<const CResultSetMetaData &>(_parent->getMetaData()))
+    {
+    }
+
+    virtual const IResultSetMetaData & getMetaData() const
+    {
+        return parent->getMetaData();
+    }
+    virtual __int64 getNumRows() const
+    {
+        return parent->getNumRows();
+    }
+    virtual bool supportsRandomSeek() const
+    {
+        return parent->supportsRandomSeek();
+    }
+    virtual IExtendedNewResultSet * cloneForFilter()
+    {
+        return NULL;
+    }
+    virtual bool fetch(MemoryBuffer & out, __int64 fileoffset)
+    {
+        return parent->fetch(out, fileoffset);
+    }
+    virtual bool fetchRaw(MemoryBuffer & out, __int64 fileoffset)
+    {
+        return parent->fetchRaw(out, fileoffset);
+    }
+    virtual bool getRow(MemoryBuffer & out, __int64 row)
+    {
+        return parent->getRow(out, row);
+    }
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row)
+    {
+        return parent->getRawRow(out, row);
+    }
+    virtual void serialize(MemoryBuffer & out)
+    {
+        parent->serialize(out);
+    }
+    virtual bool isMappedIndexField(unsigned columnIndex)
+    {
+        return parent->isMappedIndexField(columnIndex);
+    }
+    virtual void onOpen()
+    {
+        parent->onOpen();
+    }
+    virtual void onClose()
+    {
+        parent->onClose();
+    }
+    virtual IFvDataSource * queryDataSource() 
+    { 
+        return parent->queryDataSource(); 
+    }
+
+protected:
+    Linked<IExtendedNewResultSet> parent;
+    const CResultSetMetaData & meta;
+};
+
+
+class CFilteredResultSet : public CIndirectResultSet
+{
+public:
+    CFilteredResultSet(IExtendedNewResultSet * parent, ColumnFilterArray & _filters);
+//  CResultSetFilteredCursor(const CResultSetMetaData & _meta, CResultSet * _resultSet, MemoryBuffer & buffer);
+    ~CFilteredResultSet();
+
+    virtual bool getRow(MemoryBuffer & out, __int64 row);
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row);
+    virtual void serialize(MemoryBuffer & out);
+    virtual __int64 getNumRows() const;
+
+protected:
+    void initExtra();
+    bool rowMatchesFilter(const byte * row);
+    __int64 translateRow(__int64 row);
+
+protected:
+    ColumnFilterArray filters;
+    UInt64Array validPositions;
+    unsigned * offsets;
+    bool readAll;
+};
+
+class CFetchFilteredResultSet : public CIndirectResultSet
+{
+public:
+    CFetchFilteredResultSet(IExtendedNewResultSet * _parent, const CStrColumnFilter & _filters);
+
+    virtual IFilteredResultSet * createFiltered()           { UNIMPLEMENTED; }  // Would need to clone filters into new resultset
+    virtual bool getRow(MemoryBuffer & out, __int64 row);
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row);
+    virtual void serialize(MemoryBuffer & out);
+    virtual __int64 getNumRows() const;
+
+protected:
+    UInt64Array validOffsets;
+};
+
+class CResultSetCursor : public CInterface, implements IResultSet, implements IExtendedResultSetCursor
+{
+public:
+    CResultSetCursor(const CResultSetMetaData & _meta, IExtendedNewResultSet * _resultSet, bool _oldInterface);
+    CResultSetCursor(const CResultSetMetaData & _meta, IExtendedNewResultSet * _resultSet, MemoryBuffer & val);
+    ~CResultSetCursor();
+    IMPLEMENT_IINTERFACE;
+
+//interface IResultSet only
+    virtual int findColumn(const char * columnName) const;
+
+//interface IResultSet, IResultSetCursor shared
+    virtual bool absolute(__int64 row);
+    virtual void afterLast();
+    virtual void beforeFirst();
+    virtual bool fetch(__int64 fileoffset);
+    virtual bool first();
+    virtual bool getBoolean(int columnIndex);
+    virtual IDataVal & getBytes(IDataVal &d, int columnIndex);
+    virtual IResultSetCursor * getChildren(int columnIndex) const;
+    virtual xdouble getDouble(int columnIndex);
+    virtual int getFetchSize() const;
+    virtual bool getIsAll(int columnIndex) const;
+    virtual __int64 getInt(int columnIndex);
+    virtual IDataVal & getRaw(IDataVal &d, int columnIndex);
+    virtual IDataVal & getRawRow(IDataVal &d);
+    virtual int getType();
+    virtual const IResultSetMetaData & getMetaData() const;
+    virtual __int64 getNumRows() const;
+    virtual IStringVal & getString(IStringVal & ret, int columnIndex);
+    virtual bool isAfterLast() const;
+    virtual bool isBeforeFirst() const;
+    virtual bool isFirst() const;
+    virtual bool isLast() const;
+    virtual bool isNull(int columnIndex) const;
+    virtual bool isValid() const;
+    virtual bool last();
+    virtual bool next();
+    virtual bool previous();
+    virtual INewResultSet * queryResultSet() { return resultSet; }
+    virtual bool relative(__int64 rows);
+    virtual void setFetchSize(int rows);
+    virtual bool supportsRandomSeek() const;
+    virtual void serialize(IDataVal & d);
+    virtual IStringVal & getDisplayText(IStringVal &ret, int columnIndex);
+    virtual IStringVal & getXml(IStringVal & ret, int columnIndex);
+    virtual IStringVal & getXmlRow(IStringVal &ret);
+
+//IResultSetCursor
+    virtual void beginAccess();
+    virtual void endAccess();
+
+//IExtendedResultSetCursor
+    virtual void noteRelatedFileChanged() {}
+
+protected:
+    void calcFieldOffsets();
+    void init(IExtendedNewResultSet * _resultSet);
+    bool isMappedIndexField(unsigned columnIndex) { return resultSet->isMappedIndexField(columnIndex); }
+    const byte * getColumn(unsigned idx) const      { return (const byte *)curRowData.toByteArray() + offsets[idx]; }
+    void getXmlText(StringBuffer & out, int columnIndex);
+
+    virtual __int64 getCurRow() const;
+    virtual __int64 translateRow(__int64 row) const;
+    virtual void serializeType(MemoryBuffer & buffer);
+    virtual void serialize(MemoryBuffer & buffer);
+
+protected:
+    const CResultSetMetaData & meta;
+    Owned<IExtendedNewResultSet> resultSet;
+    MemoryBuffer curRowData;
+    __int64 curRow;
+    unsigned * offsets;
+    bool oldInterface;
+};
+
+//---------------------------------------------------------------------------
+
+class IndirectResultSetCursor : public CInterface, implements IExtendedResultSetCursor
+{
+public:
+    IMPLEMENT_IINTERFACE;
+
+//interface IResultSet, IResultSetCursor shared
+    virtual bool absolute(__int64 row);
+    virtual void afterLast();
+    virtual void beforeFirst();
+    virtual bool fetch(__int64 fileoffset);
+    virtual bool first();
+    virtual bool getBoolean(int columnIndex);
+    virtual IDataVal & getBytes(IDataVal &d, int columnIndex);
+    virtual IResultSetCursor * getChildren(int columnIndex) const;
+    virtual xdouble getDouble(int columnIndex);
+    virtual int getFetchSize() const;
+    virtual bool getIsAll(int columnIndex) const;
+    virtual __int64 getInt(int columnIndex);
+    virtual IDataVal & getRaw(IDataVal &d, int columnIndex);
+    virtual IDataVal & getRawRow(IDataVal &d);
+    virtual __int64 getNumRows() const;
+    virtual IStringVal & getString(IStringVal & ret, int columnIndex);
+    virtual bool isAfterLast() const;
+    virtual bool isBeforeFirst() const;
+    virtual bool isFirst() const;
+    virtual bool isLast() const;
+    virtual bool isNull(int columnIndex) const;
+    virtual bool isValid() const;
+    virtual bool last();
+    virtual bool next();
+    virtual bool previous();
+    virtual INewResultSet * queryResultSet();
+    virtual bool relative(__int64 rows);
+    virtual void serialize(IDataVal & d);
+    virtual IStringVal & getDisplayText(IStringVal &ret, int columnIndex);
+    virtual IStringVal & getXml(IStringVal & ret, int columnIndex);
+    virtual IStringVal & getXmlRow(IStringVal &ret);
+    virtual void noteRelatedFileChanged();
+
+protected:
+    virtual IExtendedResultSetCursor * queryBase() = 0;
+    virtual const IExtendedResultSetCursor * queryBase() const
+    {
+        return const_cast<IndirectResultSetCursor *>(this)->queryBase();
+    }
+};
+
+
+class NotifyingResultSetCursor : public IndirectResultSetCursor
+{
+public:
+    NotifyingResultSetCursor(IExtendedResultSetCursor * _cursor) : cursor(_cursor) {}
+
+    inline void addDependent(IExtendedResultSetCursor & next) { dependents.append(OLINK(next)); }
+
+    virtual bool absolute(__int64 row);
+    virtual void afterLast();
+    virtual void beforeFirst();
+    virtual bool fetch(__int64 fileoffset);
+    virtual bool first();
+    virtual bool last();
+    virtual bool next();
+    virtual bool previous();
+    virtual bool relative(__int64 rows);
+    virtual void noteRelatedFileChanged();
+    virtual IExtendedResultSetCursor * queryBase() { return cursor; }
+
+protected:
+    void notifyChanged();
+
+protected:
+    Linked<IExtendedResultSetCursor> cursor;
+    IArrayOf<IExtendedResultSetCursor> dependents;
+};
+
+class DelayedFilteredResultSetCursor : public IndirectResultSetCursor
+{
+public:
+    DelayedFilteredResultSetCursor(INewResultSet * _resultSet);
+
+    virtual IExtendedResultSetCursor * queryBase();
+    virtual void ensureFiltered();
+    virtual void noteRelatedFileChanged();
+
+    void clearFilters();
+    inline void clearFilter(unsigned columnIndex)
+    {
+        clearCursor();
+        filtered->clearFilter(columnIndex);
+    }
+    inline void addFilter(unsigned columnIndex, unsigned length, const char * utf8Value)
+    {
+        clearCursor();
+        filtered->addFilter(columnIndex, length, utf8Value);
+    }
+    inline void addNaturalFilter(unsigned columnIndex, unsigned length, const char * utf8Value)
+    {
+        clearCursor();
+        filtered->addNaturalFilter(columnIndex, length, utf8Value);
+    }
+
+protected:
+    void clearCursor();
+
+protected:
+    Linked<IFilteredResultSet> filtered;
+    Owned<INewResultSet> resultSet;
+    Owned<IExtendedResultSetCursor> cursor;
+};
+
+//---------------------------------------------------------------------------
+
+class CResultSetSortedCursor : public CResultSetCursor
+{
+public:
+    CResultSetSortedCursor(const CResultSetMetaData & _meta, IExtendedNewResultSet * _resultSet, unsigned _column, bool _desc);
+    CResultSetSortedCursor(const CResultSetMetaData & _meta, IExtendedNewResultSet * _resultSet, unsigned _column, bool _desc, MemoryBuffer & buffer);
+    ~CResultSetSortedCursor();
+
+    virtual bool absolute(__int64 row);
+
+protected:
+    void buildIndex();
+    virtual __int64 getCurRow() const;
+    virtual __int64 translateRow(__int64 row) const;
+    virtual void serializeType(MemoryBuffer & buffer);
+    virtual void serialize(MemoryBuffer & buffer);
+
+protected:
+    unsigned column;
+    bool desc;
+    unsigned numEntries;
+    unsigned * elements;
+    __int64 lastRow;
+};
+
+
+class CFilteredResultSetBuilder : public CInterface, implements IFilteredResultSet
+{
+public:
+    CFilteredResultSetBuilder(IExtendedNewResultSet * _resultSet);
+    IMPLEMENT_IINTERFACE
+
+    virtual INewResultSet * create();
+    virtual void addFilter(unsigned columnIndex, const char * value);
+    virtual void addFilter(unsigned columnIndex, unsigned len, const char * value);
+    virtual void addNaturalFilter(unsigned columnIndex, unsigned len, const char * value);
+    virtual void clearFilter(unsigned columnIndex);
+    virtual void clearFilters();
+
+protected:
+    CIArrayOf<CStrColumnFilter> filters;
+    Owned<IExtendedNewResultSet>    resultSet;
+};
+
+
+class ADataSource;
+class CResultSetFactoryBase : public CInterface, implements IResultSetFactory
+{
+public:
+    CResultSetFactoryBase(const char * _username, const char * _password);
+    CResultSetFactoryBase(ISecManager &secmgr, ISecUser &secuser);
+    IMPLEMENT_IINTERFACE
+
+protected:
+    StringAttr username;
+    StringAttr password;
+    Owned<ISecManager> secMgr;
+    Owned<ISecUser> secUser;
+};
+
+class CResultSetFactory : public CResultSetFactoryBase
+{
+public:
+    CResultSetFactory(const char * _username, const char * _password);
+    CResultSetFactory(ISecManager &secmgr, ISecUser &secuser);
+
+    virtual IResultSet * createResultSet(IConstWUResult * wuResult, const char * wuid);
+    virtual IResultSet * createFileResultSet(const char * logicalFile, const char * cluster);
+    virtual INewResultSet * createNewResultSet(IConstWUResult * wuResult, const char * wuid);
+    virtual INewResultSet * createNewFileResultSet(const char * logicalFile, const char * cluster);
+    virtual INewResultSet * createNewResultSet(const char * wuid, unsigned sequence, const char * name);
+    virtual INewResultSet * createNewFileResultSet(const char * logicalFile);
+    virtual IResultSetMetaData * createResultSetMeta(IConstWUResult * wuResult);
+    virtual IResultSetMetaData * createResultSetMeta(const char * wuid, unsigned sequence, const char * name);
+
+protected:
+    CResultSet * createResultSet(IFvDataSource * ds, bool _useXPath);
+    IDistributedFile * lookupLogicalName(const char * logicalName);
+};
+
+
+class CRemoteResultSetServer  : public CInterface
+{
+public:
+    void start();
+
+protected:
+    CResultSetFactory factory;
+};
+
+
+class CRemoteResultSetFactory : public CResultSetFactoryBase
+{
+public:
+    CRemoteResultSetFactory(const char * remoteServer, const char * _username, const char * _password);
+    CRemoteResultSetFactory(const char * remoteServer, ISecManager &secmgr, ISecUser &secuser);
+    IMPLEMENT_IINTERFACE
+
+    virtual IResultSet * createResultSet(IConstWUResult * wuResult, const char * wuid);
+    virtual IResultSet * createFileResultSet(const char * logicalFile, const char * cluster);
+    virtual INewResultSet * createNewResultSet(IConstWUResult * wuResult, const char * wuid);
+    virtual INewResultSet * createNewFileResultSet(const char * logicalFile, const char * cluster);
+    virtual INewResultSet * createNewResultSet(const char * wuid, unsigned sequence, const char * name);
+    virtual INewResultSet * createNewFileResultSet(const char * logicalFile);
+    virtual IResultSetMetaData * createResultSetMeta(IConstWUResult * wuResult);
+    virtual IResultSetMetaData * createResultSetMeta(const char * wuid, unsigned sequence, const char * name);
+
+protected:
+    SocketEndpoint serverEP;
+};
+
+
+#endif // _resultsetscm_SCM_INCL
+//end

common/fileview2/fvserver.cpp

@@ -0,0 +1,80 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include <jliball.hpp>
+
+#include "daclient.hpp"
+#include "dadfs.hpp"
+#include "dasds.hpp"
+#include "dalienv.hpp"
+#include "fileview.hpp"
+#include "dllserver.hpp"
+
+Semaphore sem;
+
+
+bool myAbortHandler()
+{
+    sem.signal();
+    return false;
+}
+
+
+int main(int argc, const char *argv[])
+{
+    InitModuleObjects();
+    if (argc < 4 || argv[1][0]=='/' && argv[1][1]=='?')
+    {
+        printf("fvserver <dali-server> <queue> <cluster>");
+        return 1;
+    }
+    
+    Owned<IGroup> serverGroup = createIGroup(argv[1],DALI_SERVER_PORT);
+    initClientProcess(serverGroup, DCR_Other, 9123, NULL, NULL, MP_WAIT_FOREVER);
+    setPasswordsFromSDS(); 
+
+    LocalAbortHandler localHandler(myAbortHandler);
+    startRemoteDataSourceServer(argv[2], argv[3]);
+    sem.wait();
+    stopRemoteDataSourceServer();
+
+    closeDllServer();
+    closeEnvironment();
+    closedownClientProcess();
+    releaseAtoms();
+    return 0;
+}
+

common/fileview2/fvsource.ipp

@@ -0,0 +1,395 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVSOURCE_IPP
+#define FVSOURCE_IPP
+
+#include "fvdatasource.hpp"
+#include "dllserver.hpp"
+#include "hqlexpr.hpp"
+#include "eclhelper.hpp"
+
+//Following constants configure different sizes etc.
+
+#define DISK_BLOCK_SIZE     8096            // Size of chunks read directly from file.
+#define PAGED_WU_LIMIT      0x20000         // Page load work unit results >= this size.
+#define WU_BLOCK_SIZE       0x4000          // Size of chunks read from Work unit
+#define DISKREAD_PAGE_SIZE  200             // Number of rows to read in each chunk from file.
+
+
+interface IRecordSizeEx : public IRecordSize
+{
+    IRecordSize::getRecordSize;
+    virtual size32_t getRecordSize(unsigned maxLength, const void *rec) = 0;
+};
+
+class RecordSizeToEx : public CInterface, implements IRecordSizeEx
+{
+public:
+    RecordSizeToEx(IRecordSize * _recordSize) : recordSize(_recordSize) {}
+    IMPLEMENT_IINTERFACE
+
+    virtual size32_t getRecordSize(const void *rec)
+    {
+        return recordSize->getRecordSize(rec);
+    }
+    virtual size32_t getRecordSize(unsigned maxLength, const void *rec)
+    {
+        return recordSize->getRecordSize(rec);
+    }
+    virtual size32_t getFixedSize() const
+    {
+        return recordSize->getFixedSize();
+    }
+private:
+    Linked<IRecordSize> recordSize;
+};
+    
+
+//NB: In the following the following convention is used:
+// storedX - size/structure in WU/on disk
+// returnedX - size/structure of the data actually sent to the program
+// transformedX - size/structure of data after applying transformation.
+// for workunit storedX == returnedX  for disk returnedX==transformedX
+
+class DataSourceMetaData;
+class DataSourceMetaItem : public CInterface
+{
+public:
+    DataSourceMetaItem(unsigned _flags, const char * _name, const char * _xpath, ITypeInfo * _type);
+    DataSourceMetaItem(unsigned flags, MemoryBuffer & in);
+    virtual void serialize(MemoryBuffer & out) const;
+    virtual DataSourceMetaData * queryChildMeta() { return NULL; }
+
+public:
+    StringAttr  name;
+    StringAttr  xpath;
+    OwnedITypeInfo type;
+    byte           flags;
+};
+
+class DataSourceMetaData : public CInterface, implements IFvDataSourceMetaData, public IRecordSizeEx
+{
+    friend class DataSourceSetItem;
+public:
+    DataSourceMetaData(IHqlExpression * _record, byte _numFieldsToIgnore, bool _randomIsOk, bool _isGrouped, unsigned _keyedSize);
+    DataSourceMetaData();           // for NULL implementation
+    DataSourceMetaData(type_t type);
+    DataSourceMetaData(MemoryBuffer & in);
+    IMPLEMENT_IINTERFACE
+
+    virtual unsigned numColumns() const;
+    virtual ITypeInfo * queryType(unsigned column) const;
+    virtual const char * queryName(unsigned column) const;
+    virtual const char * queryXPath(unsigned column) const;
+    virtual bool supportsRandomSeek() const;
+    virtual void serialize(MemoryBuffer & out) const;
+    virtual unsigned queryFieldFlags(unsigned column) const;
+    virtual IFvDataSourceMetaData * queryChildMeta(unsigned column) const;
+    virtual IFvDataSource * createChildDataSource(unsigned column, unsigned len, const void * data);
+    virtual unsigned numKeyedColumns() const;
+
+    void addFileposition();
+    void addGrouping();
+    void addVirtualField(const char * name, const char * xpath, ITypeInfo * type);
+
+    void extractKeyedInfo(UnsignedArray & offsets, TypeInfoArray & types);
+    unsigned fixedSize() { return storedFixedSize; }
+    bool isFixedSize() { return isStoredFixedWidth; }
+    bool isSingleSet() { return ((fields.ordinality() == 1) && (fields.item(0).type->getTypeCode() == type_set)); }
+    inline unsigned getMaxRecordSize()                      { return maxRecordSize; }
+    inline bool isKey()                                     { return keyedSize != 0; }
+
+//IRecordSizeEx....
+    virtual size32_t getRecordSize(const void *rec);
+    virtual size32_t getFixedSize() const;
+    virtual size32_t getRecordSize(unsigned maxLength, const void *rec)
+    {
+        return getRecordSize(rec);
+    }
+
+protected:
+    void addSimpleField(const char * name, const char * xpath, ITypeInfo * type);
+    void gatherFields(IHqlExpression * expr, bool isConditional);
+    void gatherChildFields(IHqlExpression * expr, bool isConditional);
+    void init();
+
+protected:
+    CIArrayOf<DataSourceMetaItem> fields;
+    unsigned keyedSize;
+    unsigned storedFixedSize;
+    unsigned maxRecordSize;
+    unsigned bitsRemaining;
+    unsigned numVirtualFields;
+    bool isStoredFixedWidth;
+    bool randomIsOk;
+    byte numFieldsToIgnore;
+};
+
+
+class DataSourceDatasetItem : public DataSourceMetaItem
+{
+public:
+    DataSourceDatasetItem(const char * _name, const char * _xpath, IHqlExpression * expr);
+    DataSourceDatasetItem(unsigned flags, MemoryBuffer & in);
+
+    virtual DataSourceMetaData * queryChildMeta() { return &record; }
+    virtual void serialize(MemoryBuffer & out) const;
+
+protected:
+    DataSourceMetaData record;
+};
+
+class DataSourceSetItem : public DataSourceMetaItem
+{
+public:
+    DataSourceSetItem(const char * _name, const char * _xpath, ITypeInfo * _type);
+    DataSourceSetItem(unsigned flags, MemoryBuffer & in);
+
+    virtual DataSourceMetaData * queryChildMeta() { return &record; }
+    virtual void serialize(MemoryBuffer & out) const;
+
+protected:
+    void createChild();
+
+protected:
+    DataSourceMetaData record;
+};
+
+
+//---------------------------------------------------------------------------
+
+class RowBlock : public CInterface
+{
+public:
+    RowBlock(MemoryBuffer & _buffer, __int64 _start, __int64 _startOffset);
+    RowBlock(__int64 _start, __int64 _startOffset);
+
+    virtual const void * fetchRow(__int64 offset, size32_t & len) = 0;
+    virtual const void * getRow(__int64 search, size32_t & len, unsigned __int64 & rowOffset) = 0;
+    __int64 getStartRow() const { return start; }
+    __int64 getNextRow()  const { return start + numRows; }
+
+    virtual void getNextStoredOffset(__int64 & row, offset_t & offset);
+
+protected:
+    MemoryBuffer buffer;
+    __int64 start;
+    __int64 startOffset;
+    unsigned numRows;
+};
+
+class FixedRowBlock : public RowBlock
+{
+public:
+    FixedRowBlock(MemoryBuffer & _buffer, __int64 _start, __int64 _startOffset, size32_t _fixedRecordSize);
+
+    virtual const void * fetchRow(__int64 offset, size32_t & len);
+    virtual const void * getRow(__int64 search, size32_t & len, unsigned __int64 & rowOffset);
+
+protected:
+    size32_t fixedRecordSize;
+};
+
+class VariableRowBlock : public RowBlock
+{
+public:
+    VariableRowBlock(MemoryBuffer & _buffer, __int64 _start, __int64 _startOffset, IRecordSizeEx * recordSize, bool isLast);
+    VariableRowBlock(MemoryBuffer & inBuffer, __int64 _start);  // used by remote 
+
+    virtual const void * fetchRow(__int64 offset, size32_t & len);
+    virtual const void * getRow(__int64 search, size32_t & len, unsigned __int64 & rowOffset);
+
+protected:
+    UnsignedArray rowIndex;
+};
+
+//---------------------------------------------------------------------------
+
+class FilePosFixedRowBlock : public FixedRowBlock
+{
+public:
+    FilePosFixedRowBlock(MemoryBuffer & _buffer, __int64 _start, __int64 _startOffset, size32_t _fixedRecordSize) : FixedRowBlock(_buffer, _start, _startOffset, _fixedRecordSize) {}
+
+    virtual void getNextStoredOffset(__int64 & row, offset_t & offset);
+};
+
+
+class FilePosVariableRowBlock : public VariableRowBlock
+{
+public:
+    FilePosVariableRowBlock(MemoryBuffer & _buffer, __int64 _start, __int64 _startOffset, IRecordSizeEx * recordSize, bool isLast) : VariableRowBlock(_buffer, _start, _startOffset, recordSize, isLast) {}
+
+    virtual void getNextStoredOffset(__int64 & row, offset_t & offset);
+};
+
+
+//---------------------------------------------------------------------------
+
+struct RowLocation
+{
+    RowLocation()           { matchRow = 0; matchLength = 0; bestRow = 0; bestOffset = 0; }
+
+    const void *    matchRow;
+    size32_t            matchLength;
+    __int64         bestRow;
+    offset_t        bestOffset;
+};
+
+class RowCache
+{
+enum { MaxBlocksCached = 20, MinBlocksCached = 10 };
+
+public:
+    void addRowsOwn(RowBlock * rows);
+    bool getCacheRow(__int64 row, RowLocation & location);
+
+protected:
+    void makeRoom();
+    unsigned getBestRow(__int64 row);
+    unsigned getInsertPosition(__int64 row);
+
+protected:
+    CIArrayOf<RowBlock> allRows;
+    Int64Array ages;
+};
+
+
+
+//---------------------------------------------------------------------------
+
+class FVDataSource : public ADataSource
+{
+public:
+    FVDataSource();
+    ~FVDataSource();
+
+    virtual IFvDataSourceMetaData * queryMetaData();
+
+    virtual bool fetchRow(MemoryBuffer & out, __int64 offset);
+    virtual bool fetchRawRow(MemoryBuffer & out, __int64 offset);
+    virtual bool getRow(MemoryBuffer & out, __int64 row);
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row);
+
+    virtual void onClose()  { openCount--; }
+    virtual void onOpen()   { openCount++; }
+
+protected:
+    virtual bool fetchRowData(MemoryBuffer & out, __int64 offset) = 0;
+    virtual bool getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset) = 0;
+
+protected:
+    void addFileposition();
+    void copyRow(MemoryBuffer & out, const void * src, size32_t length);
+    void loadDll(const char * wuid);
+    bool setReturnedInfoFromResult();
+
+protected:
+    StringAttr wuid;
+    Owned<IConstWUResult> wuResult;
+    HqlExprAttr returnedRecord;
+    Owned<DataSourceMetaData> returnedMeta;
+    Owned<IRecordSizeEx> returnedRecordSize;
+    Owned<DataSourceMetaData> transformedMeta;
+    HqlExprAttr transformedRecord;
+    Owned<ILoadedDllEntry> loadedDll;
+    Array pluginDlls;
+    rowTransformFunction transformer;
+    unsigned extraFieldsSize;
+    unsigned openCount;
+    bool appendFileposition;
+};
+
+class PagedDataSource : public FVDataSource
+{
+public:
+    PagedDataSource()   { totalRows = UNKNOWN_NUM_ROWS; }
+
+    virtual __int64 numRows(bool force = false);
+    virtual bool getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset);
+
+protected:
+    virtual bool loadBlock(__int64 startRow, offset_t startOffset) = 0;
+    virtual void improveLocation(__int64 row, RowLocation & location);
+
+protected:
+    unsigned __int64 totalRows;
+    RowCache cache;
+};
+
+
+
+class NullDataSource : public ADataSource
+{
+public:
+    NullDataSource() {}
+    NullDataSource(IHqlExpression * _record, bool _isGrouped, unsigned _keyedSize);
+
+    virtual bool init() { return true; }
+    virtual IFvDataSourceMetaData * queryMetaData()     { return &meta; }
+    virtual __int64 numRows(bool force = false)         { return 0; }
+    virtual bool fetchRow(MemoryBuffer & out, __int64 offset) { return false; }
+    virtual bool fetchRawRow(MemoryBuffer & out, __int64 offset) { return false; }
+    virtual bool getRow(MemoryBuffer & out, __int64 row){ return false; }
+    virtual bool getRawRow(MemoryBuffer & out, __int64 row){ return false; }
+    virtual bool isIndex() { return false; }
+    virtual bool optimizeFilter(unsigned offset, unsigned len, const void * data) { return true; }      // empty anyway...
+    virtual void onClose()  { }
+    virtual void onOpen()   { }
+
+protected:
+    DataSourceMetaData meta;
+};
+
+
+class NestedDataSource : public FVDataSource
+{
+public:
+    NestedDataSource(DataSourceMetaData & _meta, unsigned len, const void * data);
+
+//interface IFvDataSource
+    virtual bool fetchRowData(MemoryBuffer & out, __int64 offset)   { return false; }
+    virtual bool getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset);
+    virtual bool init();
+    virtual bool isIndex() { return false; }
+    virtual __int64 numRows(bool force = false);
+    virtual bool optimizeFilter(unsigned offset, unsigned len, const void * data) { return false; }
+
+protected:
+    unsigned __int64 totalSize;
+    Owned<RowBlock> rows;
+};
+
+
+class FailureDataSource : public NullDataSource
+{
+public:
+    FailureDataSource(IHqlExpression * _record, IException * _error, bool _isGrouped, unsigned _keyedSize);
+
+    virtual void onOpen()   { throw LINK(error); }
+
+protected:
+    Linked<IException> error;
+};
+
+
+#define FullStringMatch ((unsigned)-1)
+
+extern IHqlExpression * parseQuery(const char * text);
+
+#endif

common/fileview2/fvtransform.cpp

@@ -0,0 +1,948 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+#include "eclrtl.hpp"
+
+#include "fileview.hpp"
+#include "fverror.hpp"
+#include "fvrelate.ipp"
+#include "deftype.hpp"
+#include "fvresultset.ipp"
+#include "thorplugin.hpp"
+
+#include "eclrtl_imp.hpp"
+#include "hqlfold.hpp"
+#include "hqlvalid.hpp"
+#include "hqlrepository.hpp"
+
+static ViewTransformerRegistry * theTransformerRegistry;
+static ITypeInfo * stringType;
+static ITypeInfo * utf8Type;
+static ITypeInfo * unicodeType;
+static _ATOM addAtom;
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    addAtom = createIdentifierAtom("add");
+    stringType = makeStringType(UNKNOWN_LENGTH, NULL, NULL);
+    utf8Type = makeUtf8Type(UNKNOWN_LENGTH, NULL);
+    unicodeType = makeUnicodeType(UNKNOWN_LENGTH, NULL);
+
+    theTransformerRegistry = new ViewTransformerRegistry;
+    theTransformerRegistry->addTransformer(new ViewFailTransformer);
+    theTransformerRegistry->addTransformer(new ViewAddTransformer);
+    
+    return true;
+}
+MODULE_EXIT()
+{
+    delete theTransformerRegistry;
+    unicodeType->Release();
+    utf8Type->Release();
+    stringType->Release();
+}
+
+//---------------------------------------------------------------------------
+
+void CardinalityElement::init(unsigned len, const char * _text)
+{
+    if (len == 0)
+    {
+        text.set("1");
+        min = 1;
+        max = 1;
+        return;
+    }
+
+    text.set(_text, len);
+    const char * dotdot = strstr(text, "..");
+    if (dotdot)
+    {
+        min = rtlStrToUInt4(dotdot-text, text);
+        if (stricmp(dotdot+2, "M") == 0)
+            max = Unbounded;
+        else
+            max = rtlVStrToUInt4(dotdot+2);
+    }
+    else
+    {
+        if (stricmp(text, "M") == 0)
+        {
+            min = 0;
+            max = Unbounded;
+        }
+        else
+        {
+            min = rtlVStrToUInt4(text);
+            max = min;
+        }
+    }
+}
+
+
+
+void CardinalityMapping::init(const char * text)
+{
+    const char * colon = strchr(text, ':');
+    if (colon)
+    {
+        primary.init(colon-text, text);
+        secondary.init(strlen(colon+1), colon+1);
+    }
+    else
+    {
+        //May as well try and make some sense of it
+        primary.init(0, "");
+        secondary.init(strlen(text), text);
+    }
+}
+
+
+void getInvertedCardinality(StringBuffer & out, const char * cardinality)
+{
+    if (cardinality)
+    {
+        CardinalityMapping mapping(cardinality);
+        out.append(mapping.secondary.text).append(":").append(mapping.primary.text);
+    }
+}
+
+//---------------------------------------------------------------------------
+
+ViewFieldTransformer * ViewFieldTransformer::bind(const HqlExprArray & args)
+{
+    return LINK(this);
+}
+
+void ViewFieldTransformer::transform(MemoryAttr & utfTarget, const MemoryAttr & utfSrc)
+{
+    //NB: The system utf8 functions typically take a length, whilst MemoryAttr provide a size.
+    unsigned lenTarget;
+    char * target;
+    const char * source = static_cast<const char *>(utfSrc.get());
+    unsigned lenSource = rtlUtf8Length(utfSrc.length(), source);
+
+    transform(lenTarget, target, lenSource, source);
+
+    unsigned sizeTarget = rtlUtf8Size(lenTarget, target);
+    utfTarget.setOwn(lenTarget, target);
+}
+
+
+ViewFailTransformer::ViewFailTransformer() : ViewFieldTransformer(failAtom)
+{
+}
+
+void ViewFailTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
+{
+    throwError(FVERR_FailTransformation);
+    lenTarget = 0;
+    target = NULL;
+}
+
+
+ViewAddTransformer::ViewAddTransformer() : ViewFieldTransformer(addAtom)
+{
+}
+
+ViewAddTransformer::ViewAddTransformer(const HqlExprArray & _args) : ViewFieldTransformer(addAtom)
+{
+    appendArray(args, _args);
+}
+
+
+ViewFieldTransformer * ViewAddTransformer::bind(const HqlExprArray & args)
+{
+    return new ViewAddTransformer(args);
+}
+
+void ViewAddTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
+{
+    unsigned __int64 value = rtlUtf8ToInt(lenSource, source);
+    if (args.ordinality())
+        value += args.item(0).queryValue()->getIntValue();
+    rtlInt8ToStrX(lenTarget, target, value);
+}
+
+//---------------------------------------------------------------------------
+
+void ViewFieldUtf8Transformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
+{
+    (*function)(lenTarget, target, lenSource, source);
+}
+
+
+void ViewFieldUnicodeTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
+{
+    unsigned lenUnicodeSrc;
+    unsigned lenUnicodeTarget;
+    rtlDataAttr unicodeSrc;
+    rtlDataAttr unicodeTarget;
+
+    rtlUtf8ToUnicodeX(lenUnicodeSrc, unicodeSrc.refustr(), lenSource, source);
+    (*function)(lenUnicodeTarget, unicodeTarget.refustr(), lenUnicodeSrc, unicodeSrc.getustr());
+    rtlUnicodeToUtf8X(lenTarget, target, lenUnicodeTarget, unicodeTarget.getustr());
+}
+
+
+void ViewFieldStringTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
+{
+    unsigned lenStringSrc;
+    unsigned lenStringTarget;
+    rtlDataAttr stringSrc;
+    rtlDataAttr stringTarget;
+
+    rtlUtf8ToStrX(lenStringSrc, stringSrc.refstr(), lenSource, source);
+    (*function)(lenStringTarget, stringTarget.refstr(), lenStringSrc, stringSrc.getstr());
+    rtlStrToUtf8X(lenTarget, target, lenStringTarget, stringTarget.getstr());
+}
+
+
+//---------------------------------------------------------------------------
+
+ViewFieldTransformer * ViewFieldECLTransformer::bind(const HqlExprArray & args)
+{
+    if (args.ordinality() == 0)
+        return LINK(this);
+    return new ViewFieldBoundECLTransformer(this, args);
+}
+
+void ViewFieldECLTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source, const HqlExprArray & extraArgs)
+{
+    Owned<ITypeInfo> sourceType = makeUtf8Type(lenSource, 0);
+    IValue * sourceValue = createUtf8Value(source, LINK(sourceType));
+    OwnedHqlExpr sourceExpr = createConstant(sourceValue);
+    HqlExprArray actuals;
+    actuals.append(*LINK(sourceExpr));
+    appendArray(actuals, extraArgs);
+
+    ThrowingErrorReceiver errors;
+    OwnedHqlExpr call = createBoundFunction(&errors, function, actuals, NULL, true);
+    OwnedHqlExpr castValue = ensureExprType(call, utf8Type);
+    OwnedHqlExpr folded = quickFoldExpression(castValue, NULL, 0);
+    IValue * foldedValue = folded->queryValue();
+    assertex(foldedValue);
+    unsigned len = foldedValue->queryType()->getStringLen();
+    const char * data = static_cast<const char *>(foldedValue->queryValue());
+    unsigned size = rtlUtf8Size(len, data);
+    lenTarget = len;
+    target = (char *)rtlMalloc(size);
+    memcpy(target, data, size);
+}
+
+void ViewFieldECLTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
+{
+    HqlExprArray extraArgs;
+    transform(lenTarget, target, lenSource, source, extraArgs);
+}
+
+void ViewFieldBoundECLTransformer::transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source)
+{
+    transformer->transform(lenTarget, target, lenSource, source, args);
+}
+
+//---------------------------------------------------------------------------
+
+
+void ViewTransformerRegistry::addTransformer(ViewFieldTransformer * ownedTransformer)
+{
+    transformers.append(*ownedTransformer);
+}
+
+void ViewTransformerRegistry::addFieldUtf8Transformer(const char * name, utf8FieldTransformerFunction func)
+{
+    transformers.append(* new ViewFieldUtf8Transformer(createIdentifierAtom(name), func));
+}
+
+void ViewTransformerRegistry::addFieldStringTransformer(const char * name, stringFieldTransformerFunction func)
+{
+    transformers.append(* new ViewFieldStringTransformer(createIdentifierAtom(name), func));
+}
+
+void ViewTransformerRegistry::addFieldUnicodeTransformer(const char * name, unicodeFieldTransformerFunction func)
+{
+    transformers.append(* new ViewFieldUnicodeTransformer(createIdentifierAtom(name), func));
+}
+
+void ViewTransformerRegistry::addPlugins(const char * name)
+{
+    loadedPlugins.setown(new SafePluginMap(&pluginCtx, true));
+    loadedPlugins->loadFromList(name);
+
+    ThrowingErrorReceiver errors;
+    dataServer.setown(createSourceFileEclRepository(&errors, name, NULL, NULL, 0));
+    HqlScopeArray scopes;
+    HqlLookupContext ctx(NULL, &errors, NULL, dataServer);
+    dataServer->getRootScopes(scopes, ctx);
+    ForEachItemIn(i, scopes)
+    {
+        IHqlScope * scope = &scopes.item(i);
+        HqlExprArray symbols;
+        scope->getSymbols(symbols);
+
+        ForEachItemIn(j, symbols)
+        {
+            IHqlExpression & cur = symbols.item(j);
+            if (cur.getOperator() == no_service)
+                addServiceDefinition(&cur);
+        }
+    }
+}
+
+static bool matchesSimpleTransformer(IHqlExpression * definition, ITypeInfo * type)
+{
+    if (definition->queryBody()->queryType() != type)
+        return false;
+    if (definition->numChildren() != 2)
+        return false;
+    if (definition->queryChild(1)->queryType() != type)
+        return false;
+    return true;
+}
+
+void * ViewTransformerRegistry::resolveExternal(IHqlExpression * funcdef)
+{
+    IHqlExpression *body = funcdef->queryChild(0);
+
+    StringBuffer entry;
+    StringBuffer lib;
+    getProperty(body, entrypointAtom, entry);
+    getProperty(body, libraryAtom, lib);
+    if (!lib.length())
+        getProperty(body, pluginAtom, lib);
+
+    ensureFileExtension(lib, SharedObjectExtension);
+
+    Owned<ILoadedDllEntry> match = loadedPlugins->getPluginDll(lib.toCharArray(), NULL, false); // MORE - shouldn't it check the version????
+    if (!match)
+        return NULL;
+
+    return GetSharedProcedure(match->getInstance(), entry.toCharArray());
+}
+
+
+ViewFieldTransformer * ViewTransformerRegistry::createTransformer(IHqlExpression * funcdef)
+{
+    IHqlExpression *body = funcdef->queryChild(0);
+    if(!body) 
+        return NULL;
+
+    StringBuffer entry;
+    StringBuffer lib;
+    getProperty(body, entrypointAtom, entry);
+    getProperty(body, libraryAtom, lib);
+    if (!lib.length())
+        getProperty(body, pluginAtom, lib);
+    if ((entry.length() == 0) || (lib.length() == 0))
+        return NULL;
+
+    if(!body->hasProperty(pureAtom) && !body->hasProperty(templateAtom))
+        return NULL;
+
+    if(!body->hasProperty(cAtom)) 
+        return NULL;
+
+    if(body->hasProperty(gctxmethodAtom) || body->hasProperty(ctxmethodAtom) || body->hasProperty(omethodAtom)) 
+        return NULL;
+
+    if(body->hasProperty(contextAtom) || body->hasProperty(globalContextAtom)) 
+        return NULL;
+
+    //Special case string->string mapping (e.g., uppercase)
+    if (matchesSimpleTransformer(funcdef, stringType))
+    {
+        stringFieldTransformerFunction resolved = (stringFieldTransformerFunction)resolveExternal(funcdef);
+        if (resolved)
+            return new ViewFieldStringTransformer(funcdef->queryName(), resolved);
+    }
+
+    //Special case string->string mapping (e.g., uppercase)
+    if (matchesSimpleTransformer(funcdef, unicodeType))
+    {
+        unicodeFieldTransformerFunction resolved = (unicodeFieldTransformerFunction)resolveExternal(funcdef);
+        if (resolved)
+            return new ViewFieldUnicodeTransformer(funcdef->queryName(), resolved);
+    }
+
+    //MORE: special case string->string etc.
+    return new ViewFieldECLTransformer(funcdef);
+}
+
+void ViewTransformerRegistry::addServiceDefinition(IHqlExpression * service)
+{
+    Owned<ViewServiceEntry> entry = new ViewServiceEntry;
+    entry->name.set(service->queryName()->str());
+
+    HqlExprArray symbols;
+    service->queryScope()->getSymbols(symbols);
+    ForEachItemIn(i, symbols)
+    {
+        IHqlExpression & cur = symbols.item(i);
+        if (cur.getOperator() == no_funcdef && cur.queryChild(0)->getOperator() == no_external)
+        {
+            ViewFieldTransformer * transformer = createTransformer(&cur);
+            if (transformer)
+                entry->transformers.append(*transformer);
+        }
+    }
+    plugins.append(*entry.getClear());
+}
+
+
+ViewFieldTransformer * find(const ViewFieldTransformerArray & transformers, const char * name, const HqlExprArray & args)
+{
+    if (!name)
+        return NULL;
+    _ATOM search = createIdentifierAtom(name);
+    ForEachItemIn(i, transformers)
+    {
+        ViewFieldTransformer & cur = transformers.item(i);
+        if (cur.matches(search))
+            return cur.bind(args);
+    }
+    return NULL;
+}
+
+
+ViewFieldTransformer * ViewTransformerRegistry::resolve(const char * name, const HqlExprArray & args)
+{
+    return find(transformers, name, args);
+}
+
+ViewFieldTransformer * ViewTransformerRegistry::resolve(const char * servicename, const char * functionName, const HqlExprArray & args)
+{
+    if (!servicename)
+        return NULL;
+
+    ForEachItemIn(i, plugins)
+    {
+        ViewServiceEntry & cur = plugins.item(i);
+        if (stricmp(servicename, cur.name) == 0)
+            return find(cur.transformers, functionName, args);
+    }
+    return NULL;
+}
+
+IViewTransformerRegistry & queryTransformerRegistry()
+{
+    return *theTransformerRegistry;
+}
+
+
+
+//---------------------------------------------------------------------------
+
+bool containsFail(const ViewFieldTransformerArray & transforms)
+{
+    ForEachItemIn(i, transforms)
+    {
+        if (transforms.item(i).matches(failAtom))
+            return true;
+    }
+    return false;
+}
+
+void translateValue(MemoryAttr & result, const MemoryAttr & filterValue, const ViewFieldTransformerArray & transforms)
+{
+    unsigned numTransforms = transforms.ordinality();
+    if (numTransforms)
+    {
+        MemoryAttr tempValue[2];
+        unsigned whichTarget = 0;
+        const MemoryAttr * source = &filterValue;
+        for (unsigned i=0; i < numTransforms-1; i++)
+        {
+            MemoryAttr * target = &tempValue[whichTarget];
+            transforms.item(i).transform(*target, *source);
+            source = target;
+            whichTarget = 1-whichTarget;
+        }
+        transforms.item(numTransforms-1).transform(result, *source);
+    }
+    else
+        result.set(filterValue.length(), filterValue.get());
+}
+
+
+ViewJoinColumn::ViewJoinColumn(unsigned _whichColumn, const ViewFieldTransformerArray & _getTransforms, const ViewFieldTransformerArray & _setTransforms)
+{
+    whichColumn = _whichColumn;
+    appendArray(getTransforms, _getTransforms);
+    appendArray(setTransforms, _setTransforms);
+    getContainsFail = containsFail(getTransforms);
+    setContainsFail = containsFail(setTransforms);
+}
+
+void ViewJoinColumn::addFilter(IFilteredResultSet * resultSet, const MemoryAttr & value)
+{
+    const MemoryAttr * source = &value;
+    MemoryAttr tempValue;
+    if (setTransforms.ordinality())
+    {
+        translateValue(tempValue, value, setTransforms);
+        source = &tempValue;
+    }
+    resultSet->addFilter(whichColumn, source->length(), (const char *)source->get());
+}
+
+void ViewJoinColumn::clearFilter(IFilteredResultSet * resultSet)
+{
+    resultSet->clearFilter(whichColumn);
+}
+
+void ViewJoinColumn::getValue(MemoryAttr & value, IResultSetCursor * cursor)
+{
+    if (getTransforms.ordinality())
+    {
+        MemoryAttr rowValue;
+        MemoryAttr2IStringVal adaptor(rowValue);
+        cursor->getDisplayText(adaptor, whichColumn);
+        translateValue(value, rowValue, getTransforms);
+    }
+    else
+    {
+        MemoryAttr2IStringVal adaptor(value);
+        cursor->getDisplayText(adaptor, whichColumn);
+    }
+}
+
+//---------------------------------------------------------------------------
+
+struct TextReference
+{
+    TextReference() { len =0; text = NULL; }
+    TextReference(size32_t _len, const char * _text) : len(_len), text(_text) {}
+
+    inline bool eq(const char * search) const { return (len == strlen(search)) && (memicmp(text, search, len) == 0); }
+    inline void get(StringAttr & target) const { target.set(text, len); }
+    void set(size32_t _len, const char * _text) { len = _len; text = _text; }
+
+    IHqlExpression * createIntConstant();
+    IHqlExpression * createStringConstant();
+
+    size32_t len;
+    const char * text;
+};
+
+IHqlExpression * TextReference::createIntConstant()
+{
+    return createConstant(rtlStrToInt8(len, text));
+}
+
+IHqlExpression * TextReference::createStringConstant()
+{
+    return createConstant(createUnicodeValue(text+1, len-2, "", true, true));
+}
+
+class MappingParser
+{
+    enum { TokEof=256, TokId, TokInt, TokString };
+public:
+    MappingParser(const IResultSetMetaData & _fieldMeta, bool _datasetSelectorAllowed) : fieldMeta(_fieldMeta), datasetSelectorAllowed(_datasetSelectorAllowed) {}
+
+    void parseColumnMappingList(FieldTransformInfoArray & results, unsigned len, const char * text);
+
+protected:
+    unsigned lexToken();
+    void assertToken(int expected);
+    void getTokenText(StringAttr & target) { curToken.get(target); }
+
+    void parseAttribute(FieldTransformInfo & output);
+    void parseColumn(FieldTransformInfo & output);
+    void parseColumnMapping(FieldTransformInfo & output);
+    void parseConstantList(HqlExprArray & args);
+    void parseTransformList(ViewFieldTransformerArray & transforms);
+
+protected:
+    const IResultSetMetaData & fieldMeta;
+    bool datasetSelectorAllowed;
+    unsigned tokenType;
+    TextReference curToken;
+    unsigned offset;
+    unsigned lenInput;
+    const char * input;
+};
+
+inline bool isLeadingIdentChar(byte next)
+{
+    return isalpha(next) || (next == '_') || (next == '$');
+}
+
+inline bool isTrailingIdentChar(byte next)
+{
+    return isalnum(next) || (next == '_') || (next == '$');
+}
+
+//MORE: This should really use the hqllexer - especially if it gets any more complex!
+unsigned MappingParser::lexToken()
+{
+    const byte * buffer = (const byte *)input;
+    unsigned cur = offset;
+
+    while ((cur < lenInput) && isspace(buffer[cur]))
+        cur++;
+
+    if (cur < lenInput)
+    {
+        byte next = buffer[cur];
+        if (isLeadingIdentChar(next))
+        {
+            cur++;
+            while ((cur < lenInput) && (isTrailingIdentChar(buffer[cur])))
+                cur++;
+            tokenType = TokId;
+        }
+        else if (isdigit(next) ||
+                 ((next == '-') && (cur+1 < lenInput) && isdigit(buffer[cur+1])))
+        {
+            cur++;
+            while ((cur < lenInput) && (isdigit(buffer[cur])))
+                cur++;
+            tokenType = TokInt;
+        }
+        else if (next == '\'')
+        {
+            cur++;
+            while (cur < lenInput)
+            {
+                byte next = buffer[cur];
+                if (next == '\'')
+                    break;
+                else if (next == '\\')
+                {
+                    if (cur+1 < lenInput)
+                        cur++;
+                }
+                cur++;
+            }
+
+            if (cur == lenInput)
+                throwError2(FVERR_BadStringTermination, cur-offset, input+offset);
+            cur++;
+            tokenType = TokString;
+        }
+        else
+        {
+            tokenType = next;
+            cur++;
+        }
+    }
+    else
+    {
+        tokenType = TokEof;
+    }
+
+    curToken.set(cur-offset, input+offset);
+    offset = cur;
+    return tokenType;
+}
+
+void MappingParser::assertToken(int expected)
+{
+    if (tokenType != expected)
+    {
+        StringBuffer id;
+        switch (expected)
+        {
+        case TokId:
+            id.append("identifier");
+            break;
+        default:
+            id.append((char)expected);
+            break;
+        }
+        unsigned len = lenInput-offset;
+        if (len>10) len = 10;
+        throwError3(FVERR_ExpectedX, id.str(), len, input+offset);
+    }
+}
+
+void MappingParser::parseColumn(FieldTransformInfo & output)
+{
+    output.datasetColumn = NotFound;
+    output.column = NotFound;
+
+    unsigned firstFieldIndex = 0;
+    const IResultSetMetaData * curMeta = &fieldMeta;
+    loop
+    {
+        StringAttr fieldName;
+        assertToken(TokId);
+        getTokenText(fieldName);
+        lexToken();
+
+        //Cheat and cast the meta so the field lookup can be done more efficiently
+        const CResultSetMetaData & castFieldMeta = static_cast<const CResultSetMetaData &>(*curMeta);
+        unsigned matchColumn = castFieldMeta.queryColumnIndex(firstFieldIndex, fieldName);
+        if (matchColumn == NotFound)
+            throwError1(FVERR_UnrecognisedFieldX, fieldName.get());
+
+        DisplayType kind = fieldMeta.getColumnDisplayType(matchColumn);
+        switch (kind)
+        {
+        case TypeBoolean:
+        case TypeInteger:
+        case TypeUnsignedInteger:
+        case TypeReal:
+        case TypeString:
+        case TypeData:
+        case TypeUnicode:
+        case TypeUnknown:
+        case TypeSet:
+            output.column = matchColumn;
+            break;
+        case TypeBeginRecord:
+            //Restrict the search fields to the contents of the record.
+            firstFieldIndex = matchColumn+1;
+            break;
+        case TypeDataset:
+            {
+                if (!datasetSelectorAllowed)
+                    throwError1(FVERR_CannotSelectFromDatasetX, fieldName.get());
+                if (output.datasetColumn != NotFound)
+                    throwError1(FVERR_CannotSelectManyFromDatasetX, fieldName.get());
+
+                firstFieldIndex = 0;
+                curMeta = curMeta->getChildMeta(matchColumn);
+                output.datasetColumn = matchColumn;
+                break;
+            }
+        default:
+            throwUnexpected();
+        }
+
+        if (output.column != NotFound)
+            return;
+
+        assertToken('.');
+        lexToken();
+    }
+}
+
+void MappingParser::parseConstantList(HqlExprArray & args)
+{
+    loop
+    {
+        switch (tokenType)
+        {
+        case TokInt:
+            args.append(*curToken.createIntConstant());
+            break;
+        case TokString:
+            args.append(*curToken.createStringConstant());
+            break;
+        default:
+            unsigned len = lenInput - offset > 10 ? 10 : lenInput - offset;
+            throwError3(FVERR_ExpectedX, "int or string constant", len, input+offset);
+        }
+        lexToken();
+        if (tokenType != ',')
+            return;
+        lexToken();
+    }
+}
+
+void MappingParser::parseTransformList(ViewFieldTransformerArray & transforms)
+{
+    loop
+    {
+        assertToken(TokId);
+
+        StringAttr mappingName, childName, grandName;
+        curToken.get(mappingName);
+        lexToken();
+
+        if (tokenType == '.')
+        {
+            lexToken();
+            assertToken(TokId);
+            curToken.get(childName);
+            lexToken();
+        }
+
+        if (tokenType == '.')
+        {
+            lexToken();
+            assertToken(TokId);
+            curToken.get(grandName);
+            lexToken();
+        }
+
+        HqlExprArray args;
+        if (tokenType == '(')
+        {
+            lexToken();
+            if (tokenType != ')')
+                parseConstantList(args);
+            assertToken(')');
+            lexToken();
+        }
+
+        ViewFieldTransformer * transform;
+        if (childName)
+        {
+            transform = theTransformerRegistry->resolve(mappingName, childName, args);
+            if (!transform)
+            {
+                //Maybe they specified the module name - should provide a 3 valued lookup
+                transform = theTransformerRegistry->resolve(childName, grandName, args);
+            }
+            if (!transform)
+                throwError2(FVERR_UnrecognisedMappingFunctionXY, mappingName.get(), childName.get());
+        }
+        else
+        {
+            transform = theTransformerRegistry->resolve(mappingName, args);
+            if (!transform)
+                throwError1(FVERR_UnrecognisedMappingFunctionX, mappingName.get());
+        }
+
+        transforms.append(*transform);
+
+        if (tokenType != ',')
+            break;
+
+        lexToken();
+    }
+}
+
+void MappingParser::parseAttribute(FieldTransformInfo & output)
+{
+    assertToken(TokId);
+    if (curToken.eq("get"))
+    {
+        lexToken();
+        assertToken('(');
+        lexToken();
+        parseTransformList(output.getTransforms);
+        assertToken(')');
+        lexToken();
+    }
+    else if (curToken.eq("set"))
+    {
+        lexToken();
+        assertToken('(');
+        lexToken();
+        parseTransformList(output.setTransforms);
+        assertToken(')');
+        lexToken();
+    }
+    else if (curToken.eq("displayname"))
+    {
+        lexToken();
+        assertToken('(');
+        lexToken();
+        assertToken(TokId); // could allow a string I guess
+        curToken.get(output.naturalName);
+        lexToken();
+        assertToken(')');
+        lexToken();
+    }
+    else
+        throwError1(FVERR_ExpectedX, "Definition name");
+}
+
+void MappingParser::parseColumnMapping(FieldTransformInfo & output)
+{
+    parseColumn(output);
+    int endToken = '}';
+    //be flexible and allow () or {}?
+    if (tokenType == '(')
+        endToken = ')';
+    else if (tokenType != '{')
+        return;
+    lexToken();
+
+    if (tokenType != endToken)
+    {
+        loop
+        {
+            parseAttribute(output);
+
+            if (tokenType != ',')
+                break;
+
+            lexToken();
+        }
+    }
+
+    assertToken(endToken);
+    lexToken();
+}
+
+
+void MappingParser::parseColumnMappingList(FieldTransformInfoArray & results, unsigned len, const char * text)
+{
+    lenInput = len;
+    input = text;
+    offset = 0;
+    lexToken();
+    if (tokenType == TokEof)
+        return;
+
+    loop
+    {
+        FieldTransformInfo * next = new FieldTransformInfo;
+        results.append(*next);
+        parseColumnMapping(*next);
+
+        if (tokenType == TokEof)
+            break;
+
+        assertToken(',');
+        lexToken();
+    }
+}
+
+
+void parseColumnMappingList(FieldTransformInfoArray & results,
+                            const IResultSetMetaData & fieldMeta,
+                            bool isDatasetAllowed,      // if non null dataset.x is allowed, and column returned via pointer
+                            const char * text)
+{
+    MappingParser parser(fieldMeta, isDatasetAllowed);
+    parser.parseColumnMappingList(results, strlen(text), text);
+}
+
+
+void parseFileColumnMapping(FieldTransformInfoArray & results, const char * text, const IResultSetMetaData & fieldMeta)
+{
+    MappingParser parser(fieldMeta, false);
+    parser.parseColumnMappingList(results, strlen(text), text);
+}
+
+
+static void test()
+{
+    HqlExprArray args;
+    MemoryAttr source;
+    source.set(26,"Gavin H\303\243lliday !!\316\261\316\221\307\272!!");
+
+    {
+        Owned<ViewFieldTransformer> transform = theTransformerRegistry->resolve("stringlib","StringToUpperCase",args);
+        MemoryAttr target;
+        transform->transform(target, source);
+    }
+
+    {
+        Owned<ViewFieldTransformer> transform = theTransformerRegistry->resolve("unicodelib","UnicodeToUpperCase",args);
+        MemoryAttr target;
+        transform->transform(target, source);
+    }
+    source.get();
+}

common/fileview2/fvtransform.ipp

@@ -0,0 +1,276 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVTRANSFORM_IPP
+#define FVTRANSFORM_IPP
+
+#include "fvrelate.hpp"
+#include "hqlexpr.hpp"
+#include "thorplugin.hpp"
+
+struct CardinalityElement
+{
+//  static const unsigned Unbounded = (unsigned)-1;     // vc6 doesn't like this.
+    enum { Unbounded = -1 };        // almost the same
+
+public:
+    CardinalityElement() { min = 0; max = 0; }
+
+    void init(unsigned len, const char * _text);
+
+public:
+    unsigned min;
+    unsigned max;
+    StringAttr text;
+};
+
+
+struct CardinalityMapping
+{
+public:
+    inline CardinalityMapping() {}
+    inline CardinalityMapping(const char * _text) { init(_text); }
+
+    void init(const char * text);
+
+public:
+    CardinalityElement primary;
+    CardinalityElement secondary;
+};
+
+void getInvertedCardinality(StringBuffer & out, const char * cardinality);
+
+//---------------------------------------------------------------------------
+
+//Field values are always extracted/added as utf8 values.  The transformation functions may need to translate formats to work correctly
+//MORE: Support different types, and auto translate from string<->utf8<->unicode
+class FILEVIEW_API ViewFieldTransformer : public CInterface
+{
+public:
+    ViewFieldTransformer(_ATOM _name)
+        : name(_name)
+    {}
+
+    inline bool matches(_ATOM search) const { return name == search; }
+    inline _ATOM queryName() const { return name; }
+    
+    virtual ViewFieldTransformer * bind(const HqlExprArray & args);
+
+    void transform(MemoryAttr & utfTarget, const MemoryAttr & utfSrc);
+
+protected:
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source) = 0;
+
+protected:
+    _ATOM name;
+};
+
+class FILEVIEW_API ViewFailTransformer : public ViewFieldTransformer
+{
+public:
+    ViewFailTransformer();
+    
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source);
+};
+
+
+class FILEVIEW_API ViewAddTransformer : public ViewFieldTransformer
+{
+public:
+    ViewAddTransformer();
+    ViewAddTransformer(const HqlExprArray & _args);
+    
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source);
+    virtual ViewFieldTransformer * bind(const HqlExprArray & args);
+
+protected:
+    HqlExprArray args;
+};
+
+
+class FILEVIEW_API ViewFieldUtf8Transformer : public ViewFieldTransformer
+{
+public:
+    ViewFieldUtf8Transformer(_ATOM _name, utf8FieldTransformerFunction _function)
+        : ViewFieldTransformer(_name), function(_function)
+    {}
+
+protected:
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source);
+
+protected:
+    utf8FieldTransformerFunction function;
+};
+
+
+
+class FILEVIEW_API ViewFieldUnicodeTransformer : public ViewFieldTransformer
+{
+public:
+    ViewFieldUnicodeTransformer(_ATOM _name, unicodeFieldTransformerFunction _function)
+        : ViewFieldTransformer(_name), function(_function)
+    {}
+
+protected:
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source);
+
+protected:
+    unicodeFieldTransformerFunction function;
+};
+
+
+
+class FILEVIEW_API ViewFieldStringTransformer : public ViewFieldTransformer
+{
+public:
+    ViewFieldStringTransformer(_ATOM _name, stringFieldTransformerFunction _function)
+        : ViewFieldTransformer(_name), function(_function)
+    {}
+
+protected:
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source);
+
+protected:
+    stringFieldTransformerFunction function;
+};
+
+class FILEVIEW_API ViewFieldECLTransformer : public ViewFieldTransformer
+{
+    friend class ViewFieldBoundECLTransformer;
+public:
+    ViewFieldECLTransformer(IHqlExpression * _function)
+        : ViewFieldTransformer(_function->queryName()), function(_function)
+    {}
+
+protected:
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source);
+    virtual ViewFieldTransformer * bind(const HqlExprArray & args);
+
+    void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source, const HqlExprArray & extraArgs);
+
+protected:
+    LinkedHqlExpr function;
+};
+
+class FILEVIEW_API ViewFieldBoundECLTransformer : public ViewFieldTransformer
+{
+public:
+    ViewFieldBoundECLTransformer(ViewFieldECLTransformer * _transformer, const HqlExprArray & _args)
+        : ViewFieldTransformer(_transformer->queryName()), transformer(_transformer)
+    {
+        appendArray(args, _args);
+    }
+
+protected:
+    virtual void transform(unsigned & lenTarget, char * & target, unsigned lenSource, const char * source);
+
+protected:
+    Linked<ViewFieldECLTransformer> transformer;
+    HqlExprArray args;
+};
+
+typedef CIArrayOf<ViewFieldTransformer> ViewFieldTransformerArray;
+
+class ViewServiceEntry : public CInterface
+{
+public:
+    StringAttr name;
+    //HINSTANCE handle;         // to prevent it being loaded and unloaded.
+    ViewFieldTransformerArray transformers;
+};
+
+class FILEVIEW_API ViewTransformerRegistry : implements IViewTransformerRegistry
+{
+public:
+    virtual void addFieldStringTransformer(const char * name, stringFieldTransformerFunction func);
+    virtual void addFieldUtf8Transformer(const char * name, utf8FieldTransformerFunction func);
+    virtual void addFieldUnicodeTransformer(const char * name, unicodeFieldTransformerFunction func);
+    virtual void addPlugins(const char * pluginname);
+            
+    void addTransformer(ViewFieldTransformer * ownedTransformer);
+
+    ViewFieldTransformer * resolve(const char * name, const HqlExprArray & args);
+    ViewFieldTransformer * resolve(const char * servicename, const char * functionname, const HqlExprArray & args);
+
+protected:
+    void addServiceDefinition(IHqlExpression * service);
+    void * resolveExternal(IHqlExpression * funcdef);
+    ViewFieldTransformer * createTransformer(IHqlExpression * funcdef);
+
+protected:
+    //Could use a hash table if the number of entries is likely to get large, unlikely to be faster though
+    ViewFieldTransformerArray transformers;
+    CIArrayOf<ViewServiceEntry> plugins;
+    Owned<SafePluginMap> loadedPlugins;
+    Owned<IEclRepository> dataServer;
+    SimplePluginCtx pluginCtx;
+};
+
+
+//---------------------------------------------------------------------------
+
+typedef CIArrayOf<ViewFieldTransformer> ViewFieldTransformerArray;
+void translateValue(MemoryAttr & result, const MemoryAttr & value, const ViewFieldTransformerArray & transforms);
+
+bool containsFail(const ViewFieldTransformerArray & transforms);
+
+class FILEVIEW_API ViewJoinColumn : public CInterface
+{
+public:
+    ViewJoinColumn(unsigned _whichColumn, const ViewFieldTransformerArray & _getTransforms, const ViewFieldTransformerArray & _setTransforms);
+
+    void addFilter(IFilteredResultSet * resultSet, const MemoryAttr & value);
+    void clearFilter(IFilteredResultSet * resultSet);
+    void getValue(MemoryAttr & value, IResultSetCursor * cursor);
+    unsigned queryBaseColumn() const { return whichColumn; }
+
+    bool canGet() const { return !getContainsFail; }
+    bool canSet() const { return !setContainsFail; }
+    bool hasGetTranslation() const { return getTransforms.ordinality() != 0; }
+    bool hasSetTranslation() const { return setTransforms.ordinality() != 0; }
+
+protected:
+    unsigned whichColumn;
+    ViewFieldTransformerArray getTransforms;
+    ViewFieldTransformerArray setTransforms;
+    bool getContainsFail;
+    bool setContainsFail;
+};
+typedef CIArrayOf<ViewJoinColumn> ViewJoinColumnArray;
+
+
+struct FieldTransformInfo : public CInterface
+{
+//output 
+    unsigned datasetColumn;
+    unsigned column;
+    ViewFieldTransformerArray getTransforms;
+    ViewFieldTransformerArray setTransforms;
+    StringAttr naturalName;
+    Owned<ITypeInfo> type;
+};
+
+typedef CIArrayOf<FieldTransformInfo> FieldTransformInfoArray;
+
+void parseFileColumnMapping(FieldTransformInfoArray & mappings, const char * text, const IResultSetMetaData & fieldMeta);
+void parseColumnMappingList(FieldTransformInfoArray & results,
+                            const IResultSetMetaData & fieldMeta,
+                            bool isDatasetAllowed,      // if non null dataset.x is allowed, and column returned via pointer
+                            const char * text);
+
+#endif

common/fileview2/fvwugen.cpp

@@ -0,0 +1,310 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+
+#include "hqlexpr.hpp"
+#include "fvwugen.ipp"
+#include "fvsource.ipp"
+
+/*
+The format of the generated query is as follows:
+
+unsigned8 startPos := 0 : stored('__startPos__');
+unsigned4 numRecords := 100 : stored('__numRecords__');
+
+src := dataset(..., rec);                   //Need to add holepos or filepos to structure
+
+filter := src(curPos >= startPos);          //curPos == holepos or filepos depending on source.
+
+limited := choosen(filter, numRecords);
+
+simple := table(src, {simplify-record});
+
+output(simple);
+
+  */
+
+_ATOM fileposName;
+_ATOM insertedAtom;
+_ATOM recordlenName;
+
+MODULE_INIT(INIT_PRIORITY_STANDARD)
+{
+    fileposName = createIdentifierAtom("__filepos__");
+    insertedAtom = createLowerCaseAtom("inserted");
+    recordlenName = createIdentifierAtom("__recordlen__");
+    return true;
+}
+
+IHqlExpression * addFilter(IHqlExpression * dataset, IHqlExpression * limitField)
+{
+    IHqlExpression * lower = createConstant(limitField->queryType()->castFrom(true, (__int64)0));
+    lower = createValue(no_colon, lower, createValue(no_stored, createConstant(LOWER_LIMIT_ID)));
+    lower = createSymbol(createIdentifierAtom(LOWER_LIMIT_ID), lower->getType(), lower);
+    dataset = createDataset(no_filter, LINK(dataset), createBoolExpr(no_ge, LINK(limitField), lower));
+
+    IHqlExpression * upper = createConstant((int)DISKREAD_PAGE_SIZE);
+    upper = createValue(no_colon, upper, createValue(no_stored, createConstant(RECORD_LIMIT_ID)));
+    upper = createSymbol(createIdentifierAtom(RECORD_LIMIT_ID), upper->getType(), upper);
+    dataset = createDataset(no_choosen, dataset, upper);
+    dataset = createSymbol(createIdentifierAtom("_Filtered_"), dataset->getType(), dataset);
+    return dataset;
+}
+
+IHqlExpression * addOutput(IHqlExpression * dataset)
+{
+    return createValue(no_output, makeVoidType(), LINK(dataset));
+}
+
+
+IHqlExpression * addSimplifyProject(IHqlExpression * dataset)
+{
+    IHqlExpression * record = dataset->queryRecord();
+    IHqlExpression * projectRecord = getSimplifiedRecord(record, false);
+    if (!projectRecord)
+        return LINK(dataset);
+
+    projectRecord = createSymbol(createIdentifierAtom("_TargetRecord_"), projectRecord->getType(), projectRecord);
+    return createDataset(no_newusertable, LINK(dataset), createComma(projectRecord, getSimplifiedTransform(projectRecord, record, dataset)));
+}
+
+
+
+
+IHqlExpression * buildWorkUnitViewerEcl(IHqlExpression * record, const char * wuid, unsigned sequence, const char * name)
+{
+    OwnedHqlExpr newRecord = createSymbol(createIdentifierAtom("_SourceRecord_"), record->getType(), LINK(record));
+    IHqlExpression * arg = name ? createConstant(name) : createConstant((int)sequence);
+    OwnedHqlExpr dataset = createDataset(no_workunit_dataset, newRecord.getLink(), createComma(createConstant(wuid), arg));
+    OwnedHqlExpr projected = addSimplifyProject(dataset);
+    OwnedHqlExpr output = addOutput(projected);
+    return output.getClear();
+}
+
+
+IHqlExpression * buildDiskFileViewerEcl(const char * logicalName, IHqlExpression * record)
+{
+    //Add filepos to the incomming record structure...
+    IHqlExpression * filePosAttr = createAttribute(virtualAtom, createAttribute(filepositionAtom));
+    OwnedHqlExpr filepos = createField(fileposName, makeIntType(8, false), NULL, filePosAttr);
+    IHqlExpression * sizeofAttr = createAttribute(virtualAtom, createAttribute(sizeofAtom));
+    OwnedHqlExpr reclen = createField(recordlenName, makeIntType(2, false), NULL, sizeofAttr);
+    HqlExprArray fields;
+    unwindChildren(fields, record);
+    fields.append(*filepos.getLink());
+    fields.append(*reclen.getLink());
+
+    OwnedHqlExpr newRecord = createRecord(fields);
+    newRecord.setown(createSymbol(createIdentifierAtom("_SourceRecord_"), newRecord->getType(), newRecord.getLink()));
+
+    OwnedHqlExpr dataset = createNewDataset(createConstant(logicalName), newRecord.getLink(), createValue(no_thor), NULL, NULL, NULL);
+    OwnedHqlExpr filtered = addFilter(dataset, filepos);
+    OwnedHqlExpr projected = addSimplifyProject(filtered);
+    OwnedHqlExpr output = addOutput(projected);
+    return output.getClear();
+}
+
+//---------------------------------------------------------------------------
+
+IHqlExpression * buildDiskOutputEcl(const char * logicalName, IHqlExpression * record)
+{
+    OwnedHqlExpr dataset = createNewDataset(createConstant(logicalName), LINK(record), createValue(no_thor), NULL, NULL, NULL);
+    return addOutput(dataset);
+}
+
+//---------------------------------------------------------------------------
+
+//Add holepos/filepos/sizeof to the query, so that the browse has something to work on.
+static HqlTransformerInfo positionTransformerInfo("PositionTransformer");
+PositionTransformer::PositionTransformer()  : NewHqlTransformer(positionTransformerInfo)
+{ 
+    insertedAttr.setown(createAttribute(insertedAtom)); 
+}
+
+IHqlExpression * PositionTransformer::createTransformed(IHqlExpression * _expr)
+{
+    OwnedHqlExpr transformed = NewHqlTransformer::createTransformed(_expr);
+
+    switch (transformed->getOperator())
+    {
+    case no_table:
+        {
+            IHqlExpression * mode = transformed->queryChild(2);
+            HqlExprArray fields;
+            HqlExprArray args;
+
+            if (mode->getOperator() == no_thor)
+            {
+                unwindChildren(fields, transformed->queryChild(1));
+                IHqlExpression * filePosAttr = createComma(createAttribute(virtualAtom, createAttribute(filepositionAtom)), insertedAttr.getLink());
+                IHqlExpression * sizeofAttr = createComma(createAttribute(virtualAtom, createAttribute(sizeofAtom)), insertedAttr.getLink());
+                fields.append(*createField(fileposName, makeIntType(8, false), NULL, filePosAttr));
+                fields.append(*createField(recordlenName, makeIntType(2, false), NULL, sizeofAttr));
+
+                unwindChildren(args, transformed);
+                args.replace(*createRecord(fields), 1);
+                return transformed->clone(args);
+            }
+        }
+        break;
+    case no_iterate:
+    case no_hqlproject:
+        {
+            HqlExprArray args;
+            HqlExprArray assigns;
+            IHqlExpression * transform = transformed->queryChild(1);
+            unwindChildren(args, transformed);
+            unwindChildren(assigns, transform);
+            IHqlExpression * inRecord = transformed->queryChild(0)->queryRecord();
+            IHqlExpression * outRecord = transform->queryRecord();
+
+            HqlExprArray fields;
+            unwindChildren(fields, outRecord);
+            ForEachChild(idx, inRecord)
+            {
+                IHqlExpression * child = inRecord->queryChild(idx);
+                if (child->hasProperty(insertedAtom))
+                {
+                    IHqlExpression * newTarget = createField(child->queryName(), child->getType(), LINK(child), insertedAttr.getLink());
+                    fields.append(*newTarget);
+                    assigns.append(*createValue(no_assign, makeVoidType(), newTarget, createSelectExpr(createValue(no_left), LINK(newTarget))));
+                }
+            }
+            IHqlExpression * newRecord = createRecord(fields);
+            args.replace(*createValue(no_transform, newRecord->getType(), assigns), 1);
+            return transformed->clone(args);
+        }
+        break;
+    case no_join:
+        //only ok if join first
+    case no_rollup:
+    case no_newaggregate:
+    case no_aggregate:
+        fail();
+        break;
+    case no_usertable:
+    case no_selectfields:
+        {
+            IHqlExpression * grouping = transformed->queryChild(2);
+            if (grouping && (grouping->getOperator() != no_attr))
+                fail();
+            IHqlExpression * record = transformed->queryRecord();
+            unsigned max = record->numChildren();
+            HqlExprArray fields;
+            unwindChildren(fields, transformed->queryChild(1));
+            ForEachChild(idx, record)
+            {
+                IHqlExpression * child = record->queryChild(idx);
+                if (child->hasProperty(insertedAtom))
+                    fields.append(*createField(child->queryName(), child->getType(), LINK(child), insertedAttr.getLink()));
+            }
+
+            HqlExprArray args;
+            unwindChildren(args, transformed);
+            args.replace(*createRecord(fields), 1);
+            return transformed->clone(args);
+        }
+    case no_output:
+        {
+            IHqlExpression * file = transformed->queryChild(2);
+            if (file && (file->getOperator() != no_attr))
+            {
+                IHqlExpression * child = transformed->queryChild(0);
+                assertex(child->getOperator() == no_selectfields);
+                HqlExprArray args;
+                unwindChildren(args, child);
+
+                HqlExprArray fields;
+                IHqlExpression * record = child->queryChild(1);
+                if (record->getOperator() == no_null)
+                {
+                    //MORE: This might will not work for ifblocks, and may not cope with 
+                    //      alien(self.x), or nested records.
+                    IHqlExpression * record = child->queryRecord();
+                    ForEachChild(idx, record)
+                    {
+                        IHqlExpression * child = record->queryChild(idx);
+                        if (!child->hasProperty(insertedAtom))
+                            fields.append(*createField(child->queryName(), child->getType(), LINK(child)));
+                    }
+                }
+                else
+                {
+                    ForEachChild(idx, record)
+                    {
+                        IHqlExpression * child = record->queryChild(idx);
+                        if (!child->hasProperty(insertedAtom))
+                            fields.append(*LINK(child));
+                    }
+                }
+
+                args.replace(*createRecord(fields), 1);
+                IHqlExpression * dataset = createRecord(args);
+
+                args.kill();
+                unwindChildren(args, transformed);
+                args.replace(*dataset, 0);
+                return transformed->clone(args);
+            }
+        }
+        break;
+
+    default:
+        if (definesColumnList(transformed))
+            throw 2;
+        break;
+    }
+
+
+    return transformed.getClear();
+}
+
+
+void PositionTransformer::fail()
+{
+    throw 1;
+}
+
+
+IHqlExpression * addQueryPositionFields(IHqlExpression * selectFields)
+{
+    PositionTransformer transformer;
+    try
+    {
+        return transformer.transformRoot(selectFields);
+    }
+    catch (int)
+    {
+        return NULL;
+    }
+}
+
+
+IHqlExpression * buildQueryViewerEcl(IHqlExpression * selectFields)
+{
+    OwnedHqlExpr transformed = addQueryPositionFields(selectFields);
+    if (!transformed)
+        return NULL;
+    IHqlSimpleScope * scope = transformed->queryRecord()->querySimpleScope();
+    OwnedHqlExpr filterField = scope->lookupSymbol(fileposName);
+    OwnedHqlExpr filtered = addFilter(transformed, filterField);
+    OwnedHqlExpr output = addOutput(filtered);
+    return output.getClear();
+}
+

common/fileview2/fvwugen.hpp

@@ -0,0 +1,40 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVWUGEN_HPP
+#define FVWUGEN_HPP
+
+#ifdef _WIN32
+    #ifdef FILEVIEW2_EXPORTS
+        #define FILEVIEW_API __declspec(dllexport)
+    #else
+        #define FILEVIEW_API __declspec(dllimport)
+    #endif
+#else
+    #define FILEVIEW_API
+#endif
+
+#define LOWER_LIMIT_ID      "__startPos__"
+#define RECORD_LIMIT_ID     "__numRecords__"
+
+extern FILEVIEW_API IHqlExpression * buildWorkUnitViewerEcl(IHqlExpression * record, const char * wuid, unsigned sequence, const char * name);
+extern FILEVIEW_API IHqlExpression * buildDiskOutputEcl(const char * logicalName, IHqlExpression * record);
+extern FILEVIEW_API IHqlExpression * buildDiskFileViewerEcl(const char * logicalName, IHqlExpression * record);
+extern FILEVIEW_API IHqlExpression * buildQueryViewerEcl(IHqlExpression * selectFields);
+
+#endif

common/fileview2/fvwugen.ipp

@@ -0,0 +1,38 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVWUGEN_IPP
+#define FVWUGEN_IPP
+
+#include "fvwugen.hpp"
+#include "hqltrans.ipp"
+
+class PositionTransformer : public NewHqlTransformer
+{
+public:
+    PositionTransformer();
+
+    IHqlExpression *createTransformed(IHqlExpression * expr);
+
+protected:
+    void fail();
+    HqlExprAttr insertedAttr;
+};
+
+
+#endif

common/fileview2/fvwusource.cpp

@@ -0,0 +1,168 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "platform.h"
+#include "jliball.hpp"
+#include "eclrtl.hpp"
+
+#include "fvresultset.ipp"
+
+#include "fileview.hpp"
+#include "fvwusource.ipp"
+#include "eclhelper.hpp"
+
+#include "fvdatasource.hpp"
+
+WorkUnitDataSource::WorkUnitDataSource(IConstWUResult * _wuResult, const char * _wuid)
+{
+    wuResult.set(_wuResult);
+    wuid.set(_wuid);
+
+    totalRows = wuResult->getResultTotalRowCount();
+    if (totalRows == -1)
+        totalRows = 0;
+    totalSize = wuResult->getResultRawSize(NULL, NULL);
+}
+
+bool WorkUnitDataSource::init()
+{
+    return setReturnedInfoFromResult();
+}
+
+
+__int64 WorkUnitDataSource::numRows(bool force)
+{
+    return totalRows;
+}
+
+//---------------------------------------------------------------------------
+
+FullWorkUnitDataSource::FullWorkUnitDataSource(IConstWUResult * _wuResult, const char * _wuid) : WorkUnitDataSource(_wuResult, _wuid)
+{
+}
+
+bool FullWorkUnitDataSource::init()
+{
+    bool ok = WorkUnitDataSource::init();
+    if (ok)
+    {
+        MemoryBuffer temp;
+        MemoryBuffer2IDataVal xxx(temp);
+
+        //Nasty.  Single sets are represented as the same way as datasets (with an extra flag for all)
+        //however need to represent as a single row containing a set, which has a different format.
+        if (wuResult->isResultScalar() && returnedMeta->isSingleSet())
+        {
+            temp.append(wuResult->getResultIsAll());
+            temp.append((size32_t)wuResult->getResultRawSize(0, 0));
+        }
+        wuResult->getResultRaw(xxx, NULL, NULL);
+
+        if (returnedMeta->isFixedSize())
+            rows.setown(new FixedRowBlock(temp, 0, 0, returnedMeta->fixedSize()));
+        else
+            rows.setown(new VariableRowBlock(temp, 0, 0, returnedRecordSize, true));
+    }
+    return ok;
+}
+
+
+bool FullWorkUnitDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
+{
+    size32_t length;
+    const void * data = rows->fetchRow(offset, length);
+    if (!data)
+        return false;
+    out.append(length, data);
+    return true;
+}
+
+bool FullWorkUnitDataSource::getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset)
+{
+    data = rows->getRow(row, length, offset);
+    return (data != NULL);
+}
+
+//---------------------------------------------------------------------------
+
+PagedWorkUnitDataSource::PagedWorkUnitDataSource(IConstWUResult * _wuResult, const char * _wuid) : WorkUnitDataSource(_wuResult, _wuid)
+{
+}
+
+
+bool PagedWorkUnitDataSource::init()
+{
+    return WorkUnitDataSource::init();
+}
+
+bool PagedWorkUnitDataSource::getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset)
+{
+    if ((row < 0) || ((unsigned __int64)row > totalRows))
+        return false;
+
+    RowLocation location;
+    loop
+    {
+        if (cache.getCacheRow(row, location))
+        {
+            length = location.matchLength;
+            data = location.matchRow;
+            return true;
+        }
+
+        if (!loadBlock(location.bestRow, location.bestOffset))
+            return false;
+    }
+}
+
+
+bool PagedWorkUnitDataSource::fetchRowData(MemoryBuffer & out, __int64 offset)
+{
+    MemoryBuffer temp;
+    MemoryBuffer2IDataVal wrapper(out); 
+    wuResult->getResultRaw(wrapper, offset, returnedMeta->getMaxRecordSize(), NULL, NULL);
+    if (temp.length() == 0)
+        return false;
+    return true;
+}
+
+
+bool PagedWorkUnitDataSource::loadBlock(__int64 startRow, offset_t startOffset)
+{
+    MemoryBuffer temp;
+    MemoryBuffer2IDataVal xxx(temp); 
+    RowBlock * rows;
+    if (returnedMeta->isFixedSize())
+    {
+        unsigned fixedSize = returnedMeta->fixedSize();
+        unsigned readSize = (WU_BLOCK_SIZE / fixedSize) * fixedSize;
+        wuResult->getResultRaw(xxx, startOffset, readSize, NULL, NULL);
+        if (temp.length() == 0)
+            return false;
+        rows = new FixedRowBlock(temp, startRow, startOffset, fixedSize);
+    }
+    else
+    {
+        wuResult->getResultRaw(xxx, startOffset, WU_BLOCK_SIZE, NULL, NULL);
+        if (temp.length() == 0)
+            return false;
+        rows = new VariableRowBlock(temp, startRow, startOffset, returnedRecordSize, startOffset + temp.length() == totalSize);
+    }
+    cache.addRowsOwn(rows);
+    return true;
+}

common/fileview2/fvwusource.ipp

@@ -0,0 +1,75 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef FVWUSOURCE_IPP
+#define FVWUSOURCE_IPP
+
+#include "fvdatasource.hpp"
+#include "dllserver.hpp"
+#include "hqlexpr.hpp"
+#include "eclhelper.hpp"
+
+#include "fvsource.ipp"
+
+class WorkUnitDataSource : public FVDataSource
+{
+public:
+    WorkUnitDataSource(IConstWUResult * _wuResult, const char * _wuid);
+
+//interface IFvDataSource
+    virtual bool isIndex() { return false; }
+    virtual __int64 numRows(bool force = false);
+
+    virtual bool init();
+
+protected:
+    unsigned __int64 totalRows;
+    unsigned __int64 totalSize;
+};
+
+
+class FullWorkUnitDataSource : public WorkUnitDataSource
+{
+public:
+    FullWorkUnitDataSource(IConstWUResult * _wuResult, const char * _wuid);
+
+    virtual bool fetchRowData(MemoryBuffer & out, __int64 offset);
+    virtual bool getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset);
+    virtual bool init();
+
+protected:
+    Owned<RowBlock> rows;
+};
+
+
+class PagedWorkUnitDataSource : public WorkUnitDataSource
+{
+public:
+    PagedWorkUnitDataSource(IConstWUResult * _wuResult, const char * _wuid);
+
+    virtual bool fetchRowData(MemoryBuffer & out, __int64 offset);
+    virtual bool getRowData(__int64 row, size32_t & length, const void * & data, unsigned __int64 & offset);
+    virtual bool init();
+
+    bool loadBlock(__int64 startRow, offset_t startOffset);
+
+protected:
+    RowCache cache;
+};
+
+#endif

common/fileview2/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/fileview2</title>
+
+    <para>
+        The common/fileview2 directory contains the sources for the common/fileview2 library.
+    </para>
+</section>

common/monitoring/preflight

@@ -0,0 +1,65 @@
+#!/bin/bash
+################################################################################
+#
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#################################################################################
+
+
+declare -a arr0
+declare -a arr1
+declare -a arr2
+i=0
+if [ $# -lt 2 ]; then
+        echo usage: $0 dir command
+        exit 1
+fi
+#full=$1/$2
+#pid=`ps -ef | grep $full | grep -v grep | awk '{print $2}'`
+full=$1/$2.pid
+pid=`cat $full`
+echo ProcessID: $pid
+elapsed=`ps -o etime $pid | sed -e '1d' | tail -n 1`
+echo ProcessUpTime: $elapsed
+mem=`free | head -n 3 | tail -n 1 | awk '{print $3,$4}'`
+cpu=`vmstat 1 2 | tail -n 1 | awk '{print $15}'`
+echo CPU-Idle: $cpu%
+cuptime=`uptime | cut -f 1,2 -d ','`
+echo ComputerUpTime: $cuptime
+echo ---SpaceUsedAndFree---
+swap=`free | tail -n 1 | awk '{print $3,$4}'`
+echo Physical Memory: $mem
+echo Virtual Memory: $swap
+for name in `df -l | tail -n +2 | awk '{if(NF>=4) print $NF}'`
+do
+arr0[i]=$name
+i=$((i+1))
+done
+i=0
+for used in `df -l | tail -n +2 | awk '{if(NF>=4) print $(NF-3)}'`
+do
+arr1[i]=$used
+i=$((i+1))
+done
+i=0
+for free in `df -l | tail -n +2 | awk '{if(NF>=4) print $(NF-2)}'`
+do
+arr2[i]=$free
+i=$((i+1))
+done
+for j in $(seq 0 $((i-1)))
+do
+echo ${arr0[$j]}: ${arr1[$j]} ${arr2[$j]}
+done

common/monitoring/prosysinfo/CMakeLists.txt

@@ -0,0 +1,33 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: prosysinfo 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for prosysinfo
+#####################################################
+
+project( prosysinfo ) 
+
+set (    SRCS 
+         main.cpp 
+    )
+
+add_executable ( prosysinfo ${SRCS} )
+install ( TARGETS prosysinfo DESTINATION ${OSSDIR}/bin )

common/monitoring/prosysinfo/main.cpp

@@ -0,0 +1,155 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include <stdlib.h>
+#include <stdio.h>
+
+#if defined(_WIN32) || defined(_WIN64)
+#include <windows.h>
+#define Li2Double(x) ((double)((x).HighPart) * 4.294967296E9 + (double)((x).LowPart))
+typedef union
+{
+    LONGLONG li;
+    FILETIME ft;
+} BIGTIME;
+class TSpan
+{
+public:
+    int s, m, h, d;
+    TSpan(int secs)
+    {
+        s = secs % 60;
+        m = (secs % 3600) / 60;
+        h = (secs % 86400) / 3600;
+        d = secs / 86400;
+    }
+};
+#endif
+
+int main(int argc, char** argv)
+{
+#if !defined(_WIN32) && !defined(_WIN64)
+    printf("Only Windows OS is supported.\n");
+#else
+    if(argc < 3)
+    {
+        printf("usage: %s <dir> <command>\n", argv[0]);
+        return -1;
+    }
+
+    char path[512];
+    sprintf_s(path, 511, "%s\\%s.pid", argv[1], argv[2]);
+
+    DWORD pid = 0;
+    FILE* f = NULL;
+    fopen_s(&f, path, "r");
+    if(!f)
+    {
+        fprintf(stderr, "Can't open file %s\n", path);
+    }
+    else
+    {   
+        char* pidbuf[32];
+        int numread = fread(pidbuf, sizeof(char), 31, f);
+        if(numread > 0)
+        {
+            pidbuf[numread] = '\0';
+            pid = atoi((const char*)pidbuf);
+        }
+    }
+    if(pid > 0)
+    {
+        printf("ProcessID: %d\n", pid);
+        HANDLE h = OpenProcess(PROCESS_QUERY_INFORMATION, false, pid);
+        if(h <= 0)
+        {
+            fprintf(stderr, "Process %d can't be opened.\n", pid);
+            printf("ProcessUpTime: \n");
+        }
+        else
+        {
+            //Process elapsed time.
+            BIGTIME CreateTime, ExitTime, ElapsedTime, Now;
+            FILETIME KernelTime, UserTime;
+            GetProcessTimes(h, &CreateTime.ft, &ExitTime.ft, &KernelTime, &UserTime);
+            if(ExitTime.li > CreateTime.li)
+                ElapsedTime.li = ExitTime.li - CreateTime.li;
+            else
+            {
+                GetSystemTimeAsFileTime(&Now.ft);
+                ElapsedTime.li = Now.li - CreateTime.li;
+            }
+            unsigned elapsedsecs = (unsigned)(ElapsedTime.li/10000000);
+            TSpan span(elapsedsecs);
+            printf("ProcessUpTime: %d-%02d:%02d:%02d\n", span.d, span.h, span.m, span.s);
+        }
+    }
+    else
+    {
+        printf("ProcessID: \nProcessUpTime: \n");
+    }
+
+    //CPU usage
+    BIGTIME idle1, kernel1, user1, idle2, kernel2, user2;
+    GetSystemTimes(&idle1.ft, &kernel1.ft, &user1.ft);
+    Sleep(1000);
+    GetSystemTimes(&idle2.ft, &kernel2.ft, &user2.ft);
+    int IdleTime = (int)(idle2.li - idle1.li);
+    int TotalTime = (int)((kernel2.li + user2.li) - (kernel1.li + user1.li));
+    int idleRate = (int)(100.0 * IdleTime / TotalTime);
+    printf("CPU-Idle: %d%%\n", idleRate);
+
+    //Computer uptime
+    LARGE_INTEGER ticks, unit;
+    QueryPerformanceCounter(&ticks);
+    QueryPerformanceFrequency(&unit);
+    int secs = (int)(ticks.QuadPart/unit.QuadPart);
+    TSpan u((int)secs);
+    printf("ComputerUpTime: %d days, %d:%d\n", u.d, u.h, u.m);
+
+    printf("---SpaceUsedAndFree---\n");
+
+    //Physical and virtual memory usage.
+    MEMORYSTATUS memstatus;
+    GlobalMemoryStatus(&memstatus);
+    printf("Physical Memory: %d %d\nVirtual Memory: %d %d\n", 
+        (memstatus.dwTotalPhys - memstatus.dwAvailPhys)/1024, memstatus.dwAvailPhys/1024,
+        (memstatus.dwTotalVirtual - memstatus.dwAvailVirtual)/1024, memstatus.dwAvailVirtual/1024);
+
+    // Disk Usage
+    char        drivePath[] = "?:\\";
+    char        driveName;
+    for( driveName = 'A'; driveName <= 'Z'; driveName++ ) 
+    {
+        drivePath[0] = driveName;
+        int dtype = GetDriveTypeA(drivePath);
+        if(dtype == DRIVE_FIXED || dtype == DRIVE_RAMDISK || dtype == DRIVE_REMOVABLE || dtype == DRIVE_CDROM) 
+        {
+            ULARGE_INTEGER diskAvailStruct;
+            ULARGE_INTEGER diskTotalStruct;
+            diskAvailStruct.QuadPart = 0;
+            diskTotalStruct.QuadPart = 0;
+            GetDiskFreeSpaceExA(drivePath, &diskAvailStruct, &diskTotalStruct, 0);
+            double DiskSize = diskTotalStruct.QuadPart / 1024.0; 
+            double FreeSize = diskAvailStruct.QuadPart / 1024.0;
+            printf("%s: %.0f %.0f\n", drivePath, DiskSize - FreeSize, FreeSize);
+        }
+    }
+#endif
+    return 0;
+}

common/monitoring/prosysinfo/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/monitoring/prosysinfo</title>
+
+    <para>
+        The common/monitoring/prosysinfo directory contains the sources for the common/monitoring/prosysinfo library.
+    </para>
+</section>

common/monitoring/sourcedoc.xml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!DOCTYPE section PUBLIC "-//OASIS//DTD DocBook XML V4.3//EN" "http://www.oasis-open.org/docbook/xml/4.3/docbookx.dtd">
+<section>
+    <title>common/monitoring</title>
+
+    <para>
+        The common/monitoring directory contains the sources for the common/monitoring library.
+    </para>
+</section>

common/remote/CMakeLists.txt

@@ -0,0 +1,60 @@
+################################################################################
+#    Copyright (C) 2011 HPCC Systems.
+#
+#    All rights reserved. This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU Affero General Public License as
+#    published by the Free Software Foundation, either version 3 of the
+#    License, or (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU Affero General Public License for more details.
+#
+#    You should have received a copy of the GNU Affero General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+################################################################################
+
+
+# Component: remote 
+#####################################################
+# Description:
+# ------------
+#    Cmake Input File for remote
+#####################################################
+
+project( remote ) 
+
+set (    SRCS 
+         rmtfile.cpp 
+         rmtpass.cpp 
+         rmtspawn.cpp 
+         rmtssh.cpp 
+         rmtsmtp.cpp 
+         sockfile.cpp 
+         
+         remoteerr.hpp
+         rmtfile.hpp
+         rmtpass.hpp
+         rmtsmtp.hpp
+         rmtspawn.hpp
+         rmtssh.hpp
+         sockfile.hpp
+    )
+
+include_directories ( 
+         ./../../system/hrpc 
+         ./../../system/mp 
+         ./../../system/include 
+         ./../../system/jlib 
+    )
+
+ADD_DEFINITIONS( -D_USRDLL -DREMOTE_EXPORTS )
+
+HPCC_ADD_LIBRARY( remote SHARED ${SRCS}  )
+install ( TARGETS remote DESTINATION ${OSSDIR}/lib )
+
+target_link_libraries ( remote 
+    jlib 
+    mp 
+    )

common/remote/remoteerr.hpp

@@ -0,0 +1,93 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef REMOTEERR_HPP
+#define REMOTEERR_HPP
+
+#define ERR_REMOTE_FIRST                        8000
+#define ERR_REMOTE_LAST                         8049
+
+#define RFSERR_InvalidCommand                   8000
+#define RFSERR_NullFileIOHandle                 8001
+#define RFSERR_InvalidFileIOHandle              8002
+#define RFSERR_TimeoutFileIOHandle              8003
+#define RFSERR_OpenFailed                       8004
+#define RFSERR_ReadFailed                       8005
+#define RFSERR_WriteFailed                      8006
+#define RFSERR_RenameFailed                     8007
+#define RFSERR_ExistsFailed                     8009
+#define RFSERR_RemoveFailed                     8010
+#define RFSERR_CloseFailed                      8011
+#define RFSERR_IsFileFailed                     8012
+#define RFSERR_IsDirectoryFailed                8013
+#define RFSERR_IsReadOnlyFailed                 8014
+#define RFSERR_SetReadOnlyFailed                8015
+#define RFSERR_GetTimeFailed                    8016
+#define RFSERR_SetTimeFailed                    8017
+#define RFSERR_CreateDirFailed                  8018
+#define RFSERR_GetDirFailed                     8019
+#define RFSERR_GetCrcFailed                     8020
+#define RFSERR_MoveFailed                       8021
+#define RFSERR_ExtractBlobElementsFailed        8022
+#define RFSERR_CopyFailed                       8023
+#define RFSERR_AppendFailed                     8024
+#define RFSERR_AuthenticateFailed               8025
+#define RFSERR_CopySectionFailed                8026
+#define RFSERR_TreeCopyFailed                   8027
+
+
+#define RAERR_InvalidUsernamePassword           8040
+#define RFSERR_MasterSeemsToHaveDied            8041
+#define RFSERR_TimeoutWaitSlave                 8042
+#define RFSERR_TimeoutWaitConnect               8043
+#define RFSERR_TimeoutWaitMaster                8044
+#define RFSERR_NoConnectSlave                   8045
+#define RFSERR_NoConnectSlaveXY                 8046
+#define RFSERR_VersionMismatch                  8047
+
+//---- Text for all errors (make it easy to internationalise) ---------------------------
+
+#define RFSERR_InvalidCommand_Text              "Unrecognised command %d"
+#define RFSERR_NullFileIOHandle_Text            "Remote file operation on NULL fileio"
+#define RFSERR_InvalidFileIOHandle_Text         "Remote file operation on invalid fileio"
+#define RFSERR_TimeoutFileIOHandle_Text         "Remote fileio has been closed because of timeout"
+#define RFSERR_MasterSeemsToHaveDied_Text       "Master program seems to have died..."
+#define RFSERR_VersionMismatch_Text             "Slave version does not match, expected %d got %d"
+
+#define RFSERR_TimeoutWaitSlave_Text            "Timeout waiting for slave %s to respond"
+#define RFSERR_TimeoutWaitConnect_Text          "Timeout waiting to connect to slave %s"
+#define RFSERR_TimeoutWaitMaster_Text           "Timeout waiting to connect to master"
+#define RFSERR_NoConnectSlave_Text              "Failed to start slave program"
+#define RFSERR_NoConnectSlaveXY_Text            "Failed to start slave program %s on %s"
+
+#define RAERR_InvalidUsernamePassword_Text      "Invalid (upper case U) in username/password"
+
+
+interface IDAFS_Exception: extends IException
+{ // Raise by dafilesrv calls
+};
+
+enum DAFS_ERROR_CODES {
+    DAFSERR_connection_failed               = -1,   
+    DAFSERR_authenticate_failed             = -2,
+    DAFSERR_protocol_failure                = -3
+};
+
+
+
+#endif

common/remote/rmtfile.cpp

@@ -0,0 +1,641 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "platform.h"
+#include "portlist.h"
+
+#include "jlib.hpp"
+#include "jio.hpp"
+#include "jlog.hpp"
+
+#include "jmutex.hpp"
+#include "jfile.hpp"
+
+#include "sockfile.hpp"
+#include "rmtfile.hpp"
+#include "remoteerr.hpp"
+
+static CBuildVersion _bv("$HeadURL: https://svn.br.seisint.com/ecl/trunk/common/remote/rmtfile.cpp $ $Id: rmtfile.cpp 64457 2011-05-09 17:28:42Z yma $");
+
+//----------------------------------------------------------------------------
+
+//#define TEST_DAFILESRV_FOR_UNIX_PATHS     // probably not needed
+
+
+unsigned short getDaliServixPort()
+{
+    return DAFILESRV_PORT;
+}
+
+
+void setCanAccessDirectly(RemoteFilename & file,bool set)
+{
+    if (set)
+        file.setPort(0);
+    else if (file.getPort()==0)                 // foreign daliservix may be passed in
+        file.setPort(getDaliServixPort());
+
+}
+
+bool canAccessDirectly(const RemoteFilename & file) // not that well named but historical
+{
+    return (file.getPort()==0);
+}
+
+void setLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir)
+{
+    setDafsLocalMountRedirect(ip,dir,mountdir);
+}
+
+
+
+
+class CDaliServixIntercept: public CInterface, implements IFileCreateHook
+{
+public:
+    IMPLEMENT_IINTERFACE;
+    virtual IFile * createIFile(const RemoteFilename & filename)
+    {
+        SocketEndpoint ep = filename.queryEndpoint();
+        bool noport = (ep.port==0);
+        setDafsEndpointPort(ep);
+        if (!filename.isLocal()||(ep.port!=DAFILESRV_PORT)) {   // assume standard port is running on local machine 
+#ifdef __linux__
+#ifndef USE_SAMBA   
+            return createDaliServixFile(filename);  
+#endif
+#endif
+            if (!noport)            // expect all filenames that specify port to be dafilesrc or daliservix
+                return createDaliServixFile(filename);  
+            if (filename.isUnixPath()
+#ifdef TEST_DAFILESRV_FOR_UNIX_PATHS        
+                &&testDaliServixPresent(ep)
+#endif
+                )
+                return createDaliServixFile(filename);  
+        }
+        return NULL;
+    }   
+} *DaliServixIntercept = NULL;
+
+bool testDaliServixPresent(const SocketEndpoint &_ep)
+{
+    SocketEndpoint ep(_ep);
+    setDafsEndpointPort(ep);
+    if (ep.isNull())
+        return false;
+    try {
+        Owned<ISocket> socket = ISocket::connect_timeout(ep,10000);
+        return true;
+    }
+    catch (IException *e)
+    {
+        e->Release();
+    }
+    return false;
+}
+
+bool testDaliServixPresent(const IpAddress &ip)
+{
+    SocketEndpoint ep(0,ip);
+    return testDaliServixPresent(ep);
+}
+
+unsigned getDaliServixVersion(const SocketEndpoint &_ep,StringBuffer &ver)
+{
+    SocketEndpoint ep(_ep);
+    setDafsEndpointPort(ep);
+    if (ep.isNull())
+        return false;
+    try {
+        Owned<ISocket> socket = ISocket::connect_timeout(ep,10000);
+        return getRemoteVersion(socket,ver);
+    }
+    catch (IException *e)
+    {
+        EXCLOG(e,"getDaliServixVersion");
+        e->Release();
+    }
+    return 0;
+}
+
+struct CDafsOsCacheEntry
+{
+    SocketEndpoint ep;
+    DAFS_OS os;
+    time_t at;
+};
+
+class CDafsOsCache: public SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>
+{
+    void onAdd(void *) {}
+
+    void onRemove(void *et)
+    {
+        CDafsOsCacheEntry *e = (CDafsOsCacheEntry *)et;
+        delete e;
+    }
+    unsigned getHashFromElement(const void *e) const
+    {
+        const CDafsOsCacheEntry &elem=*(const CDafsOsCacheEntry *)e;        
+        return elem.ep.hash(0);
+    }
+
+    unsigned getHashFromFindParam(const void *fp) const
+    {
+        return ((const SocketEndpoint *)fp)->hash(0);
+    }
+
+    const void * getFindParam(const void *p) const
+    {
+        const CDafsOsCacheEntry &elem=*(const CDafsOsCacheEntry *)p;        
+        return (void *)&elem.ep;
+    }
+
+    bool matchesFindParam(const void * et, const void *fp, unsigned) const
+    {
+        return ((CDafsOsCacheEntry *)et)->ep.equals(*(SocketEndpoint *)fp);
+    }
+
+    IMPLEMENT_SUPERHASHTABLEOF_REF_FIND(CDafsOsCacheEntry,SocketEndpoint);
+
+public:
+    static CriticalSection crit;
+
+    CDafsOsCache() 
+    {
+    }
+    ~CDafsOsCache()
+    {
+        SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>::releaseAll();
+    }
+    DAFS_OS lookup(const SocketEndpoint &ep,ISocket *sock)
+    {
+        CriticalBlock block(crit);
+        CDafsOsCacheEntry *r = SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>::find(&ep);
+        bool needupdate=false;
+        unsigned t = (unsigned)time(NULL);
+        if (!r) {
+            r = new CDafsOsCacheEntry;
+            r->ep = ep;
+            needupdate = true;
+            SuperHashTableOf<CDafsOsCacheEntry,SocketEndpoint>::add(*r);
+        }
+        else
+            needupdate = (t-r->at>60*5);        // update every 5 mins
+        if (needupdate) {
+            r->os = DAFS_OSunknown;
+            StringBuffer ver;
+            unsigned ret;
+            if (sock)
+                ret = getRemoteVersion(sock,ver);
+            else
+                ret = getDaliServixVersion(ep,ver);
+            if (ret!=0) { // if cross-os needs dafilesrv
+                if (strstr(ver.str(),"Linux")!=NULL)
+                    r->os = DAFS_OSlinux;
+                else if (strstr(ver.str(),"Windows")!=NULL)
+                    r->os = DAFS_OSwindows;
+                else if (strstr(ver.str(),"Solaris")!=NULL)
+                    r->os = DAFS_OSsolaris;
+            }
+            r->at = t;
+        }
+        return r->os;
+    }
+};
+
+
+CriticalSection CDafsOsCache::crit;
+
+
+DAFS_OS getDaliServixOs(const SocketEndpoint &ep,ISocket *socket)
+{
+#ifdef _DEBUG
+    if (ep.isLocal())
+#ifdef _WIN32
+        return DAFS_OSwindows;
+#else
+        return DAFS_OSlinux;
+#endif
+#endif
+    static CDafsOsCache cache;
+    return cache.lookup(ep,socket);
+}
+
+
+
+DAFS_OS getDaliServixOs(const SocketEndpoint &ep)
+{
+    return getDaliServixOs(ep,NULL);
+}
+
+
+unsigned getDaliServixVersion(const IpAddress &ip,StringBuffer &ver)
+{
+    SocketEndpoint ep(0,ip);
+    return getDaliServixVersion(ep,ver);
+}
+
+int remoteExec(const SocketEndpoint &_ep,const char *cmdline, const char *workdir,bool sync,
+                                 size32_t insize, void *inbuf, MemoryBuffer *outbuf)
+{
+    SocketEndpoint ep(_ep);
+    setDafsEndpointPort(ep);
+    if (ep.isNull())
+        return false;
+    try {
+        Owned<ISocket> socket = ISocket::connect_wait(ep,5000);
+        return remoteExec(socket, cmdline, workdir, sync, insize, inbuf, outbuf);
+    }
+    catch (IException *e)
+    {
+        EXCLOG(e,"remoteExec");
+        e->Release();
+    }
+    return -2;
+}
+
+extern REMOTE_API int setDafileSvrTraceFlags(const SocketEndpoint &_ep,byte flags)
+{
+    SocketEndpoint ep(_ep);
+    setDafsEndpointPort(ep);
+    if (ep.isNull())
+        return -3;
+    try {
+        Owned<ISocket> socket = ISocket::connect_wait(ep,5000);
+        return setDafsTrace(socket, flags);
+    }
+    catch (IException *e)
+    {
+        EXCLOG(e,"setDafileSvrTraceFlags");
+        e->Release();
+    }
+    return -2;
+}
+
+extern REMOTE_API int getDafileSvrInfo(const SocketEndpoint &_ep,StringBuffer &retstr)
+{
+    SocketEndpoint ep(_ep);
+    setDafsEndpointPort(ep);
+    if (ep.isNull())
+        return false;
+    try {
+        Owned<ISocket> socket = ISocket::connect_wait(ep,5000);
+        return getDafsInfo(socket, retstr);
+    }
+    catch (IException *e)
+    {
+        EXCLOG(e,"getDafileSvrInfo");
+        e->Release();
+    }
+    return -2;
+}
+
+
+void remoteExtractBlobElements(const char * prefix, const RemoteFilename &file, ExtractedBlobArray & extracted)
+{
+    SocketEndpoint ep(file.queryEndpoint());
+    setDafsEndpointPort(ep);
+    if (ep.isNull())
+        return;
+    StringBuffer filename;
+    remoteExtractBlobElements(ep, prefix, file.getLocalPath(filename).str(), extracted);
+}
+
+
+IFile *createDaliServixFile(const RemoteFilename & file)
+{
+    SocketEndpoint ep(file.queryEndpoint());
+    setDafsEndpointPort(ep);
+    if (ep.isNull())
+        return NULL;
+    StringBuffer path;
+    file.getLocalPath(path);
+    return createRemoteFile(ep, path.str());
+}
+
+void setDaliServixSocketCaching(bool set)
+{
+    clientSetDaliServixSocketCaching(set);
+}
+
+void cacheFileConnect(IFile *file,unsigned timeout)
+{
+    RemoteFilename rfn;
+    rfn.setRemotePath(file->queryFilename());
+    if (!rfn.isLocal()&&!rfn.isNull()) {
+        SocketEndpoint ep = rfn.queryEndpoint();
+        if (ep.port)
+            clientCacheFileConnect(ep,timeout);
+    }
+}
+
+void disconnectRemoteFile(IFile *file)
+{
+    clientDisconnectRemoteFile(file);
+}
+
+void disconnectRemoteIoOnExit(IFileIO *fileio,bool set)
+{
+    clientDisconnectRemoteIoOnExit(fileio,set);
+}
+
+
+bool resetRemoteFilename(IFile *file, const char *newname)
+{
+    return clientResetFilename(file,newname); 
+}
+
+
+void enableAuthentication(bool set)
+{
+    enableDafsAuthentication(set);
+}
+
+bool asyncCopyFileSection(const char *uuid,                 // from genUUID - must be same for subsequent calls
+                            IFile *from,                        // expected to be remote
+                            RemoteFilename &to,
+                            offset_t toofs,                     // (offset_t)-1 created file and copies to start
+                            offset_t fromofs,
+                            offset_t size,                      // (offset_t)-1 for all file
+                            ICopyFileProgress *progress,
+                            unsigned timeout                    // 0 to start, non-zero to wait
+                        )
+{
+    return  clientAsyncCopyFileSection(uuid,from,to,toofs,fromofs,size,progress,timeout);
+}
+
+
+void setRemoteFileTimeouts(unsigned maxconnecttime,unsigned maxreadtime)
+{
+    clientSetRemoteFileTimeouts(maxconnecttime,maxreadtime);
+}
+
+class CScriptThread : public Thread
+{
+    StringAttr script;
+    SocketEndpoint ep;
+    Semaphore done;
+    bool ok;
+public:
+    IMPLEMENT_IINTERFACE;
+    CScriptThread(SocketEndpoint &_ep,const char *_script)
+        : ep(_ep), script(_script)
+    {
+        ok = false;
+    }
+
+    int run()
+    {
+        try {
+            int ret = remoteExec(ep,script.get(),"/c$",true,0,NULL,NULL);
+            if (ret==0)
+                ok = true;
+        }
+        catch (IException *e) {
+            EXCLOG(e,"validateNodes CScriptThread");
+            e->Release();
+        }
+        done.signal();
+        return 0;
+    }
+
+    bool waitok(unsigned timeout)
+    {
+        done.wait(timeout);
+        return ok;
+    }
+};
+
+unsigned validateNodes(const SocketEndpointArray &eps,bool chkc,bool chkd,bool chkver,const char *script,unsigned scripttimeout,SocketEndpointArray &failures,UnsignedArray &failedcodes,StringArray &failedmessages, const char *filename)
+{
+    // used for detecting duff nodes
+    PointerIArrayOf<ISocket> sockets;
+    unsigned to=30*1000;
+    unsigned n=eps.ordinality();    // use approx log scale (timeout is long but only for failure situation)
+    while (n>1) {
+        n/=2;
+        to+=30*1000;
+    }
+    multiConnect(eps,sockets,to);
+    ForEachItemIn(i,eps) {
+        if (sockets.item(i)==NULL) {
+            failures.append(eps.item(i));
+            failedcodes.append(DAFS_VALIDATE_CONNECT_FAIL);
+            failedmessages.append("Connect failure");
+        }
+    }
+
+    CriticalSection sect;
+    class casyncfor: public CAsyncFor
+    {
+        const SocketEndpointArray &eps;
+        const PointerIArrayOf<ISocket> &sockets;
+        SocketEndpointArray &failures;
+        StringArray &failedmessages;
+        UnsignedArray &failedcodes;
+        CriticalSection &sect;
+        bool chkc;
+        bool chkd;
+        bool chkv;
+        const char *filename;
+        const char *script;
+        unsigned scripttimeout;
+public:
+        casyncfor(const SocketEndpointArray &_eps,const PointerIArrayOf<ISocket> &_sockets,bool _chkc,bool _chkd,bool _chkv, const char *_script, unsigned _scripttimeout, const char *_filename,SocketEndpointArray &_failures, StringArray &_failedmessages,UnsignedArray &_failedcodes,CriticalSection &_sect) 
+            : eps(_eps), sockets(_sockets), 
+              failures(_failures), failedmessages(_failedmessages), failedcodes(_failedcodes), sect(_sect)
+        { 
+            chkc = _chkc;
+            chkd = _chkd;
+            chkv = _chkv;
+            filename = _filename;
+            script = _script;
+            scripttimeout = (script&&*script)?_scripttimeout:0;
+        }
+        void Do(unsigned i)
+        {
+#ifdef NIGEL_TESTING            
+            IpAddress badip("10.173.34.70");
+#endif
+            ISocket *sock = sockets.item(i);
+            if (!sock)
+                return;
+            SocketEndpoint ep = eps.item(i);
+            bool iswin;
+            unsigned code = 0;
+            StringBuffer errstr;
+            StringBuffer ver;
+            try {
+                getRemoteVersion(sock,ver);
+                iswin = (strstr(ver.str(),"Windows")!=NULL);
+            }
+            catch (IException *e)
+            {
+                code = DAFS_VALIDATE_CONNECT_FAIL;
+                e->errorMessage(errstr);
+                e->Release();
+            }
+            if (!code&&chkv) {
+                const char *rv = ver.str();
+                const char *v = remoteServerVersionString();
+                while (*v&&(*v!='-')&&(*v==*rv)) {
+                    v++;
+                    rv++;
+                }
+                if (*rv!=*v) {
+                    if (*rv) {
+                        while (*rv&&(*rv!='-'))
+                            rv++;
+                        while (*v&&(*v!='-'))
+                            v++;
+                        StringBuffer wanted(v-remoteServerVersionString(),remoteServerVersionString());
+                        ver.setLength(rv-ver.str());
+                        if (strcmp(ver.str(),wanted.str())<0) { // allow >
+                            code = DAFS_VALIDATE_BAD_VERSION;
+                            errstr.appendf("Mismatch dafilesrv version ");
+                            errstr.append(rv-ver.str(),ver.str());
+                            errstr.append(", wanted ");
+                            errstr.append(v-remoteServerVersionString(),remoteServerVersionString());
+                        }
+                    }
+                    else {
+                        code = DAFS_VALIDATE_CONNECT_FAIL;
+                        errstr.appendf("could not contact dafilesrv");
+                    }
+                }
+            }
+            if (!code&&(chkc||chkd)) {
+                clientAddSocketToCache(ep,sock);
+                StringBuffer path;
+                if (iswin) 
+                    path.append("c:\\");
+                else
+                    path.append("/c$/");
+                if (filename)
+                    path.append(filename);
+                else {
+                    path.append("dafs_");
+                    genUUID(path);
+                    path.append(".tmp");
+                }
+                for (unsigned drive=chkc?0:1;drive<(chkd?2U:1U);drive++) {
+                    RemoteFilename rfn;
+                    setPathDrive(path,drive);   
+                    rfn.setPath(ep,path);
+                    Owned<IFile> file = createIFile(rfn);
+                    size32_t sz;
+                    StringBuffer ds;
+                    try {
+                        Owned<IFileIO> fileio = file->open(IFOcreate);
+                        CDateTime dt;
+                        dt.setNow();
+                        dt.getString(ds);
+                        sz = ds.length()+1;
+                        assertex(sz<64);
+                        fileio->write(0,sz,ds.str());
+                    }
+                    catch (IException *e) {
+                        if (e->errorCode()==DISK_FULL_EXCEPTION_CODE)
+                            code |=  (drive?DAFS_VALIDATE_DISK_FULL_C:DAFS_VALIDATE_DISK_FULL_D);
+                        else
+                            code |=  (drive?DAFS_VALIDATE_WRITE_FAIL_C:DAFS_VALIDATE_WRITE_FAIL_D);
+                        if (errstr.length())
+                            errstr.append(',');
+                        e->errorMessage(errstr);
+                        e->Release();
+                        continue; // no use trying read
+                    }
+                    try {
+                        Owned<IFileIO> fileio = file->open(IFOread);
+                        char buf[64];
+                        size32_t rd = fileio->read(0,sizeof(buf)-1,buf);
+                        if ((rd!=sz)||(memcmp(buf,ds.str(),sz)!=0)
+#ifdef NIGEL_TESTING            
+                            ||(drive&&(badip.ipequals(ep)))
+#endif
+                            ) {
+                            StringBuffer s;
+                            ep.getIpText(s);
+                            throw MakeStringException(-1,"Data discrepancy on disk read of %c$ of %s",'c'+drive,s.str());
+                        }
+                    }
+                    catch (IException *e) {
+                        code |=  (drive?DAFS_VALIDATE_READ_FAIL_C:DAFS_VALIDATE_READ_FAIL_D);
+                        if (errstr.length())
+                            errstr.append(',');
+                        e->errorMessage(errstr);
+                        e->Release();
+                    }
+                    if (!filename||!*filename) {
+                        // delete file created
+                        try {
+                            file->remove();
+                        }
+                        catch (IException *e) {
+                            e->Release();           // supress error
+                        }
+                    }
+
+                }
+            }
+            if (!code&&scripttimeout) { // use a second thread to implement script timeout
+                Owned<CScriptThread> thread = new CScriptThread(ep,script);
+                thread->start();
+                if (!thread->waitok(scripttimeout)) {
+                    code |=  DAFS_SCRIPT_FAIL;
+                    if (errstr.length())
+                        errstr.append(',');
+                    errstr.append("FAILED: ").append(script);
+                }
+            }
+            if (code) {
+                CriticalBlock block(sect);
+                failures.append(ep);
+                failedcodes.append(code);
+                failedmessages.append(errstr.str());
+            }
+        }
+    } afor(eps,sockets,chkc,chkd,chkver,script,scripttimeout,filename,failures,failedmessages,failedcodes,sect);
+    afor.For(eps.ordinality(), 10, false, true);
+    return failures.ordinality();
+}
+
+
+MODULE_INIT(INIT_PRIORITY_REMOTE_RMTFILE)
+{
+    if(!DaliServixIntercept)
+    {
+        DaliServixIntercept = new CDaliServixIntercept;
+        addIFileCreateHook(DaliServixIntercept);
+    }
+    return true;
+}
+
+MODULE_EXIT()
+{
+    if(DaliServixIntercept)
+    {
+        // delete ConnectionTable;              // too late to delete (jsocket closed down)
+        removeIFileCreateHook(DaliServixIntercept);
+        ::Release(DaliServixIntercept);
+        DaliServixIntercept = NULL;
+    }
+}
+

common/remote/rmtfile.hpp

@@ -0,0 +1,102 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef RMTFILE_HPP
+#define RMTFILE_HPP
+
+#include "jsocket.hpp"
+#include "jfile.hpp"
+
+#ifdef DAFILESRV_LOCAL
+#define REMOTE_API
+#else
+#ifdef REMOTE_EXPORTS
+#define REMOTE_API __declspec(dllexport)
+#else
+#define REMOTE_API __declspec(dllimport)
+#endif
+#endif
+
+enum DAFS_OS
+{
+    DAFS_OSunknown,
+    DAFS_OSwindows,
+    DAFS_OSlinux,
+    DAFS_OSsolaris
+};
+
+extern REMOTE_API void filenameToUrl(StringBuffer & out, const char * filename);
+
+extern REMOTE_API unsigned short getDaliServixPort();  // assumed just the one for now
+extern REMOTE_API void setCanAccessDirectly(RemoteFilename & file,bool set);
+extern REMOTE_API void setDaliServixSocketCaching(bool set);
+extern REMOTE_API void cacheFileConnect(IFile *file,unsigned timeout);
+extern REMOTE_API bool canAccessDirectly(const RemoteFilename & file);
+extern REMOTE_API IFile *createDaliServixFile(const RemoteFilename & file);
+extern REMOTE_API bool testDaliServixPresent(const IpAddress &ip);
+extern REMOTE_API bool testDaliServixPresent(const SocketEndpoint &ep);
+extern REMOTE_API unsigned getDaliServixVersion(const IpAddress &ip,StringBuffer &ver);
+extern REMOTE_API unsigned getDaliServixVersion(const SocketEndpoint &ep,StringBuffer &ver);
+extern REMOTE_API DAFS_OS getDaliServixOs(const SocketEndpoint &ep);
+
+extern REMOTE_API void setLocalMountRedirect(const IpAddress &ip,const char *dir,const char *mountdir);
+// redirects a daliservix file to a local mount. To remove redirect use NULL for mount dir or NULL for dir
+
+
+extern REMOTE_API int remoteExec(const SocketEndpoint &ep,const char *cmdline, const char *workdir,bool sync,
+                                 size32_t insize, void *inbuf, MemoryBuffer *outbuf);
+
+extern REMOTE_API void remoteExtractBlobElements(const char * prefix, const RemoteFilename &file, ExtractedBlobArray & extracted);
+
+extern REMOTE_API int setDafileSvrTraceFlags(const SocketEndpoint &ep,byte flags);
+extern REMOTE_API int getDafileSvrInfo(const SocketEndpoint &ep,StringBuffer &retstr);
+
+extern REMOTE_API void disconnectRemoteFile(IFile *file);
+extern REMOTE_API void disconnectRemoteIoOnExit(IFileIO *fileio,bool set=true);
+
+extern REMOTE_API bool resetRemoteFilename(IFile *file, const char *newname); // returns false if not remote
+
+
+extern REMOTE_API void enableAuthentication(bool set=true); // default enabled for clients, disabled for server
+
+extern REMOTE_API bool asyncCopyFileSection(const char *uuid,   // from genUUID - must be same for subsequent calls
+                            IFile *from,                        // expected to be remote
+                            RemoteFilename &to,
+                            offset_t toofs,                     // (offset_t)-1 created file and copies to start
+                            offset_t fromofs,
+                            offset_t size,                      // (offset_t)-1 for all file
+                            ICopyFileProgress *progress,
+                            unsigned timeout                    // 0 to start, non-zero to wait
+                        ); // returns true when done
+
+extern REMOTE_API void setRemoteFileTimeouts(unsigned maxconnecttime,unsigned maxreadtime);
+
+#define DAFS_VALIDATE_CONNECT_FAIL  (0x01)
+#define DAFS_VALIDATE_BAD_VERSION   (0x02)
+#define DAFS_VALIDATE_WRITE_FAIL_C  (0x12)
+#define DAFS_VALIDATE_READ_FAIL_C   (0x14)
+#define DAFS_VALIDATE_DISK_FULL_C   (0x18)
+#define DAFS_VALIDATE_WRITE_FAIL_D  (0x22)
+#define DAFS_VALIDATE_READ_FAIL_D   (0x24)
+#define DAFS_VALIDATE_DISK_FULL_D   (0x28)
+#define DAFS_SCRIPT_FAIL            (0x40)
+                                
+extern REMOTE_API unsigned validateNodes(const SocketEndpointArray &ep,bool chkc,bool chkd,bool chkver,const char *script,unsigned scripttimeout,SocketEndpointArray &failures,UnsignedArray &failedcodes, StringArray &failedmessages, const char *filename=NULL);
+
+
+#endif

common/remote/rmtpass.cpp

@@ -0,0 +1,119 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+
+#include "platform.h"
+
+#include "jlib.hpp"
+#include "jsem.hpp"
+#include "jthread.hpp"
+
+#include "portlist.h"
+#include "jsocket.hpp"
+
+#include "rmtfile.hpp"
+#include "rmtpass.hpp"
+
+static CBuildVersion _bv("$HeadURL: https://svn.br.seisint.com/ecl/trunk/common/remote/rmtpass.cpp $ $Id: rmtpass.cpp 62376 2011-02-04 21:59:58Z sort $");
+
+void CachedPasswordProvider::addPasswordForFilename(const char * filename)
+{
+    RemoteFilename remote;
+    remote.setRemotePath(filename);
+    addPasswordForIp(remote.queryIP());
+}
+
+
+void CachedPasswordProvider::addPasswordForFilename(RemoteFilename & filename)
+{
+    addPasswordForIp(filename.queryIP());
+}
+
+void CachedPasswordProvider::addPasswordForIp(const IpAddress & ip)
+{
+    if (!hasPassword(ip))
+    {
+        IPasswordProvider * provider = queryPasswordProvider();
+        StringBuffer username, password;
+
+        if (provider && provider->getPassword(ip, username, password))
+        {
+            CachedPassword & cur = * new CachedPassword;
+            cur.ip.ipset(ip);
+            cur.password.set(password.str());
+            cur.username.set(username.str());
+            passwords.append(cur);
+        }
+    }
+}
+
+
+void CachedPasswordProvider::clear()
+{
+    passwords.kill();
+}
+
+void CachedPasswordProvider::deserialize(MemoryBuffer & in)
+{
+    unsigned num;
+    in.read(num);
+    while (num--)
+    {
+        CachedPassword & cur = * new CachedPassword;
+        cur.deserialize(in);
+        passwords.append(cur);
+    }
+}
+
+void CachedPasswordProvider::serialize(MemoryBuffer & out)
+{
+    unsigned num = passwords.ordinality();
+    out.append(num);
+    ForEachItemIn(idx, passwords)
+        passwords.item(idx).serialize(out);
+}
+
+bool CachedPasswordProvider::getPassword(const IpAddress & ip, StringBuffer & username, StringBuffer & password)
+{
+    ForEachItemIn(idx, passwords)
+    {
+        CachedPassword & cur = passwords.item(idx);
+        if (cur.ip.ipequals(ip))
+        {
+            username.append(cur.username);
+            password.append(cur.password);
+            return true;
+        }
+    }
+    StringBuffer iptext;
+    LOG(MCdebugInfo, unknownJob, "cached:getPassword(%s) failed", ip.getIpText(iptext).str());
+    return false;
+}
+
+bool CachedPasswordProvider::hasPassword(const IpAddress & ip) const
+{
+    ForEachItemIn(idx, passwords)
+    {
+        CachedPassword & cur = passwords.item(idx);
+        if (cur.ip.ipequals(ip))
+            return true;
+    }
+    return false;
+}
+

common/remote/rmtpass.hpp

@@ -0,0 +1,62 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef RMTPASS_HPP
+#define RMTPASS_HPP
+
+#ifdef REMOTE_EXPORTS
+#define REMOTE_API __declspec(dllexport)
+#else
+#define REMOTE_API __declspec(dllimport)
+#endif
+
+class REMOTE_API CachedPassword : public CInterface
+{
+public:
+    void deserialize(MemoryBuffer & in)     { ip.ipdeserialize(in); in.read(password).read(username); }
+    void serialize(MemoryBuffer & out)      { ip.ipserialize(out); out.append(password).append(username); }
+
+public:
+    IpAddress ip;
+    StringAttr password;
+    StringAttr username;
+};
+
+
+class RemoteFilename;
+class REMOTE_API CachedPasswordProvider : public CInterface, implements IPasswordProvider
+{
+public:
+    IMPLEMENT_IINTERFACE
+
+    virtual bool getPassword(const IpAddress & ip, StringBuffer & username, StringBuffer & password);
+
+    void addPasswordForFilename(const char * filename);
+    void addPasswordForFilename(RemoteFilename & filename);
+    void addPasswordForIp(const IpAddress & ip);
+    void clear();
+    void deserialize(MemoryBuffer & in);
+    void serialize(MemoryBuffer & out);
+    bool hasPassword(const IpAddress & ip) const;
+
+protected:
+    CIArrayOf<CachedPassword>   passwords;
+};
+
+
+#endif

common/remote/rmtsmtp.cpp

@@ -0,0 +1,831 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "platform.h"
+
+#include "jlib.hpp"
+#include "jlog.hpp"
+#include "jsocket.hpp"
+#include "jbuff.hpp"
+
+#include "rmtsmtp.hpp"
+
+
+static CBuildVersion _bv("$HeadURL: https://svn.br.seisint.com/ecl/trunk/common/remote/rmtsmtp.cpp $ $Id: rmtfile.cpp 59036 2010-08-31 17:54:39Z nhicks $");
+
+
+class CSMTPValidator
+{
+public:
+    CSMTPValidator() : scanlist(false) {}
+
+    void validateValue(char const * _value, char const * _label)
+    {
+        value = finger = _value;
+        label = _label;
+        while(*finger)
+        {
+            if(badChar(*finger))
+                fail("illegal character");
+            ++finger;
+        }
+    }
+
+    void validateAddress(char const * _address, char const * _label)
+    {
+        value = finger = _address;
+        label = _label;
+        scanlist = false;
+        validateLocalPart();
+        validateDomain();
+    }
+
+    void validateDomain(char const * _domain, char const * _label)
+    {
+        value = finger = _domain;
+        label = _label;
+        scanlist = false;
+        validateDomain();
+    }
+
+    void scanAddressListStart(char const * _addrlist, char const * _label)
+    {
+        value = finger = _addrlist;
+        label = _label;
+        if(!skipListSep())
+            fail("empty address list");
+        scanlist = true;
+    }
+
+    bool scanAddressListNext(StringBuffer & out)
+    {
+        if(!scanlist)
+            return false;
+        char const * start = finger;
+        validateLocalPart();
+        scanlist = validateDomain();
+        out.append(finger-start, start);
+        if(scanlist)
+            scanlist = skipListSep();
+        return true;
+    }
+
+    void escapeQuoted(char const * in, StringBuffer & out, char const * _label)
+    {
+        value = finger = in;
+        label = _label;
+        while(*finger)
+        {
+            if(badChar(*finger))
+                fail("illegal character");
+            else if((*finger == '"') || (*finger == '\\'))
+            {
+                if(finger>in)
+                    out.append(finger-in, in);
+                out.append('\\');
+                in = finger;
+            }
+            ++finger;
+        }
+        if(finger>in)
+            out.append(finger-in, in);
+    }
+
+private:
+    bool skipListSep()
+    {
+        while(*finger && isListSep(*finger))
+            ++finger;
+        return (*finger != 0);
+    }
+
+    void validateLocalPart()
+    {
+        if(*finger == '"')
+            validateQuotedLocal();
+        else
+            validateDotStringLocal();
+    }
+
+    bool validateDomain()
+    {
+        if(*finger == '[')
+            return validateAddressLiteral();
+        else
+            return validateNamedDomain();
+    }
+
+    void validateQuotedLocal()
+    {
+        ++finger;
+        while(*finger != '"')
+        {
+            if(*finger == '\\')
+            {
+                ++finger;
+                if(!*finger)
+                    fail("unexpected end-of-string in quoted local part");
+                else if(badChar(*finger))
+                    fail("illegal escaped character in quoted local part");
+            }
+            else if(!*finger)
+                fail("unexpected end-of-string in quoted local part");
+            else if(badQuotedChar(*finger))
+                fail("illegal character in quoted local part (may need escaping)");
+            ++finger;
+        }
+        ++finger;
+        if(!*finger)
+            fail("address had quoted local part but no domain (reached end-of-string)");
+        else if(*finger != '@')
+            fail("quoted local part was not followed by @");
+        ++finger;
+    }
+
+    void validateDotStringLocal()
+    {
+        enum { Start, StartAtom, Main };
+        unsigned mode = Start;
+        while(*finger != '@')
+        {
+            if(*finger == '.')
+                switch(mode)
+                {
+                case Start:
+                    fail("illegal . at start of local part");
+                case StartAtom:
+                    fail("illegal .. in local part");
+                case Main:
+                    mode = StartAtom;
+                    break;
+                }
+            else if(!*finger)
+                if(mode == Start)
+                    fail("blank address (reached end-of-string)");
+                else
+                    fail("address had dotted-atom-string local part but no domain (reached end-of-string)");
+            else if(scanlist && isListSep(*finger))
+                if(mode == Start)
+                    fail("blank address (reached comma/semicolon/space indicating next address in list)");
+                else
+                    fail("address had dotted-atom-string local part but no domain (reached comma/semicolon/space indicating next address in list)");
+            else if(badAtomChar(*finger))
+                fail("illegal character in dotted-atom-string local part (may need quoting)");
+            else
+                mode = Main;
+            ++finger;
+        }
+        switch(mode)
+        {
+        case Start:
+            fail("empty local part");
+        case StartAtom:
+            fail("illegal . at end of local part");
+        }
+        ++finger;
+    }
+
+    bool validateAddressLiteral()
+    {
+        ++finger;
+        unsigned digitcount = 0;
+        unsigned groupcount = 0;
+        while(*finger != ']')
+        {
+            if(isdigit(*finger))
+                if(digitcount == 3)
+                    fail("more than three digits in octet in address literal");
+                else
+                    ++digitcount;
+            else if(*finger == '.')
+                if(digitcount == 0)
+                {
+                    fail("empty octet in address literal");
+                }
+                else
+                {
+                    digitcount = 0;
+                    ++groupcount;
+                    if(groupcount == 4)
+                        fail("too many octets in address literal (sorry, only IPv4 supported)");
+                }
+            else if(!*finger)
+                fail("unexpected end-of-string in address literal");
+            else
+                fail("illegal character in address literal (sorry, only IPv4 supported)");
+            ++finger;
+        }
+        if(digitcount == 0)
+            fail("empty octet in address literal");
+        digitcount = 0;
+        ++groupcount;
+        if(groupcount < 4)
+            fail("too few octets in address literal");
+        ++finger;
+        if(scanlist && isListSep(*finger))
+            return true;
+        if(*finger)
+            fail("unexpected character after end of address literal");
+        return false;
+    }
+
+    bool validateNamedDomain()
+    {
+        unsigned subcount = 0;
+        unsigned charcount = 0;
+        bool ret = false;
+        while(*finger)
+        {
+            if(isalnum(*finger))
+                ++charcount;
+            else if(*finger == '_')
+                if(charcount == 0)
+                    fail("illegal _ at start of subdomain");
+                else if(!*(finger+1) || (*(finger+1) == '.') || (scanlist && isListSep(*(finger+1))))
+                    fail("illegal _ at end of subdomain");
+                else
+                    ++charcount;
+            else if(*finger == '.')
+                if(charcount == 0)
+                    if(subcount == 0)
+                        fail("illegal . at start of domain");
+                    else
+                        fail("illegal .. in domain");
+                else
+                {
+                    ++subcount;
+                    charcount = 0;
+                }
+            else if(scanlist && isListSep(*finger))
+            {
+                ret = true;
+                break;
+            }
+            else
+                fail("illegal character in domain");
+            ++finger;
+        }
+        if(charcount == 0)
+            if(subcount == 0)
+                fail("empty domain");
+            else
+                fail("illegal . at end of domain");
+        ++subcount;
+        if(subcount < 2)
+            fail("domain has only 1 subdomain");
+        return ret;
+    }
+
+    void fail(char const * msg)
+    {
+        throw MakeStringException(0, "bad %s (%s at character %u): %s", label, msg, finger-value, value);
+    }
+
+    bool badAtomChar(char c)
+    {
+        if((c<33) || (c>126)) return true;
+        switch(c)
+        {
+        case '"':
+        case '(':
+        case ')':
+        case ',':
+        case '.':
+        case ':':
+        case ';':
+        case '<':
+        case '>':
+        case '@':
+        case '[':
+        case '\\':
+        case ']':
+            return true;
+        }
+        return false;
+    }
+
+    bool badQuotedChar(char c)
+    {
+        if((c < 1) || (c>126)) return true;
+        switch(c)
+        {
+        case '\t':
+        case '\r':
+        case '\n':
+        case ' ':
+        case '"':
+        case '\\':
+            return true;
+        }
+        return false;
+    }
+
+    bool badChar(char c)
+    {
+        if((c < 1) || (c>126)) return true;
+        switch(c)
+        {
+        case '\r':
+        case '\n':
+            return true;
+        }
+        return false;
+    }
+
+    bool isListSep(char c)
+    {
+        switch(c)
+        {
+        case ',':
+        case ';':
+        case ' ':
+            return true;
+        }
+        return false;
+    }
+
+private:
+    char const * value;
+    char const * finger;
+    char const * label;
+    bool scanlist;
+    bool scanmore;
+};
+
+// escapes text for mail transfer, returns true if quoted-printable encoding was required
+
+bool mailEncode(char const * in, StringBuffer & out)
+{
+    bool esc = false;
+    size32_t outlinelen = 0;
+    char const * finger = in;
+    while(*finger)
+    {
+        //max line length 76, use soft line break =\r\n to split (RFC 1521 section 5.1 rule #5)
+        if(outlinelen+finger-in == 75)
+        {
+            out.append(finger-in, in).append("=\r\n");
+            outlinelen = 0;
+            in = finger;
+            esc = true;
+        }
+
+        //printable chars except = and - and . are left alone (RFC 1521 section 5.1 rule #2)
+        if((*finger >= 33) && (*finger <= 126) && (*finger != '=') && (*finger != '-') && (*finger != '.'))
+        {
+            ++finger;
+            continue;
+        }
+
+        //- is left alone, except for -- at start of line to protect multipart boundary (RFC 1341 section 7.2.1)
+        if(*finger == '-')
+        {
+            if((outlinelen != 0) || (*(finger+1) != '-'))
+            {
+                ++finger;
+                continue;
+            }
+        }
+
+        //. is left alone, except that an extra . is added when at start of line to protect SMTP 'end of data' signal (RFC 8211 section 4.5.2)
+        if(*finger == '.')
+        {
+            if(outlinelen == 0)
+            {
+                out.append('.');
+                ++outlinelen;
+            }
+            ++finger;
+            continue;
+        }
+
+        //tab and space are left alone except at EOL (RFC 1521 section 5.1 rule #3)
+        if((*finger == '\t') || (*finger == ' '))
+        {
+            char nxt = *(finger+1);
+            if(nxt && (nxt != 10) && (nxt != 13))
+            {
+                ++finger;
+                continue;
+            }
+        }
+
+        //CR, LF, and CRLF are all converted to CRLF (RFC 1521 section 5.1 rule #4)
+        if(*finger == 10)
+        {
+            if(finger>in)
+                out.append(finger-in, in);
+            ++finger;
+            if(*finger == 13)
+                ++finger;
+            out.append("\r\n");
+            outlinelen = 0;
+            in = finger;
+            continue;
+        }
+        if(*finger == 13)
+        {
+            if(finger>in)
+                out.append(finger-in, in);
+            ++finger;
+            out.append("\r\n");
+            outlinelen = 0;
+            in = finger;
+            continue;
+        }
+
+        //everything else is escaped (RFC 1521 section 5.1 rule #1)
+        if(finger>in)
+            out.append(finger-in, in);
+        if(outlinelen+finger-in > 72)
+        {
+            out.append("=\r\n");
+            outlinelen = 3;
+        }
+        else
+        {
+            outlinelen += (finger-in)+3;
+        }
+        out.appendf("=%02X", (unsigned char)*finger);
+        in = ++finger;
+        esc = true;
+    }
+    if(finger > in)
+        out.append(finger-in, in);
+    return esc;
+}
+
+//#define SMTP_TRACE
+
+class CMailInfo
+{
+    StringArray *warnings;
+    StringArray recipients;
+    StringBuffer to;
+    StringAttr subject;
+    StringAttr mailServer;
+    unsigned port;
+    StringAttr sender;
+    Owned<ISocket> socket;
+    StringBuffer lastAction;
+    char inbuff[200];
+    unsigned inlen;
+
+    static char const * toHeader;
+    static char const * subjectHeader;
+    static char const * senderHeader;
+public:
+    CMailInfo(char const * _to, char const * _subject, char const * _mailServer, unsigned _port, char const * _sender, StringArray *_warnings) 
+        : subject(_subject), mailServer(_mailServer), port(_port), sender(_sender), lastAction("process initialization")
+    {
+        warnings = _warnings;
+        CSMTPValidator validator;
+        validator.validateDomain(mailServer.get(), "mail server address");
+
+        if(strlen(senderHeader) + sender.length() > 998)
+            throw MakeStringException(0, "email sender address too long: %u characters", sender.length());
+        validator.validateAddress(sender.get(), "email sender address");
+
+        getRecipients(validator, _to);
+        if(strlen(toHeader) + to.length() > 998)
+            throw MakeStringException(0, "Email recipient address list too long: %u characters", to.length());
+
+        if(strlen(subjectHeader) + subject.length() > 998)
+            throw MakeStringException(0, "Email subject too long: %u characters", subject.length());
+        validator.validateValue(subject.get(), "email subject");
+    }
+
+    void open()
+    {
+        SocketEndpoint address(mailServer.get());
+        if (address.isNull())
+            throw MakeStringException(MSGAUD_operator, 0, "Could not resolve mail server address %s in SendEmail*", mailServer.get());
+        address.port = port;
+        try
+        {
+            socket.setown(ISocket::connect(address));
+        }
+        catch(IException *E)
+        {
+            E->Release();
+            throw MakeStringException(MSGAUD_operator, 0, "Failed to connect to mail server at %s:%u in SendEmail*", mailServer.get(), port);
+        }
+        lastAction.clear().append("connection to server");
+    }
+
+    void write(char const * out, size32_t len, char const * action = NULL)
+    {
+        if(action)
+            lastAction.clear().append(action);
+        else
+            lastAction.clear().append(len, out).clip();
+        try
+        {
+            socket->write(out, len);
+#ifdef SMTP_TRACE
+            DBGLOG("SMTP write: [%s]", out);
+#endif
+        }
+        catch(IException * e)
+        {
+            int code = e->errorCode();
+            StringBuffer buff;
+            e->errorMessage(buff);
+            e->Release();
+            throw MakeStringException(MSGAUD_operator, 0, "Exception %d (%s) in SendEmail* while writing %s to mail server %s:%u", code, buff.str(), lastAction.str(), mailServer.get(), port);
+        }
+    }
+
+    void read()
+    {
+        try
+        {
+            socket->read(inbuff,1,sizeof(inbuff),inlen);
+            //MORE: the following is somewhat primitive and not RFC compliant (see bug 25951) - but it is a lot better than nothing
+            if((*inbuff == '4') || (*inbuff == '5'))
+            {
+                StringBuffer b;
+                b.append("Negative reply from mail server at ").append(mailServer.get()).append(":").append(port).append(" after writing ").append(lastAction.str()).append(" in SendEmail*: ").append(inlen, inbuff).clip();
+                WARNLOG("%s", b.str());
+                if (warnings)
+                    warnings->append(b.str());
+            }
+#ifdef SMTP_TRACE
+            else
+            {
+                StringBuffer b(inlen, inbuff);
+                b.clip();
+                DBGLOG("SMTP read: [%s]", b.str());
+            }
+#endif
+        }
+        catch(IException * e)
+        {
+            int code = e->errorCode();
+            StringBuffer buff;
+            e->errorMessage(buff);
+            e->Release();
+            throw MakeStringException(MSGAUD_operator, 0, "Exception %d (%s) in SendEmail* while reading from mail server %s:%u following %s", code, buff.str(), mailServer.get(), port, lastAction.str());
+        }
+    }
+
+    void getHeader(StringBuffer & header) const
+    {
+        header.append(senderHeader).append(sender.get()).append("\r\n");
+        header.append(toHeader).append(to.str()).append("\r\n");
+        header.append(subjectHeader).append(subject.get()).append("\r\n");
+        header.append("MIME-Version: 1.0\r\n");
+    }
+
+    void getHelo(StringBuffer & out) const
+    {
+        out.append("HELO ").append(mailServer.get()).append("\r\n");
+    }
+
+    void getMailFrom(StringBuffer & out) const
+    {
+        out.append("MAIL FROM:<").append(sender.get()).append(">\r\n");
+    }
+
+    unsigned numRecipients() const
+    {
+        return recipients.ordinality();
+    }
+
+    void getRecipient(unsigned i, StringBuffer & out) const
+    {
+        char const * rcpt = recipients.item(i);
+        out.append("RCPT TO:<").append(rcpt).append(">\r\n");
+    }
+
+private:
+    void getRecipients(CSMTPValidator & validator, char const * _to)
+    {
+        StringBuffer rcpt;
+        validator.scanAddressListStart(_to, "recipient email address list");
+        while(validator.scanAddressListNext(rcpt.clear()))
+        {
+            if(recipients.ordinality())
+                to.append(",");
+            to.append(rcpt.str());
+            recipients.append(rcpt.str());
+        }
+    }
+
+};
+
+char const * CMailInfo::toHeader = "To: ";
+char const * CMailInfo::subjectHeader = "Subject: ";
+char const * CMailInfo::senderHeader = "From: ";
+
+class CMailPart
+{
+public:
+    CMailPart(char const * mimeType, char const * filename)
+    {
+        if(strlen(mimeTypeHeader) + strlen(mimeType) > 998)
+            throw MakeStringException(0, "Email attachment mime type too long: %u characters", strlen(mimeType));
+        CSMTPValidator validator;
+        validator.validateValue(mimeType, "email attachment mime type");
+        mime.append(mimeType);
+
+        if(filename)
+        {
+            StringBuffer qfilename;
+            validator.escapeQuoted(filename, qfilename, "email attachment filename");
+            if(strlen(dispositionHeader) + strlen("attachment; filename=\"\"") + qfilename.length() > 998)
+                throw MakeStringException(0, "Email attachment filename too long: %u characters", strlen(filename));
+            disposition.append("attachment; filename=\"").append(qfilename.str()).append("\"");
+        }
+        else
+        {
+            disposition.append("inline");
+        }
+
+        encoding = NULL;
+    }
+
+    void getHeader(StringBuffer & header) const
+    {
+        header.append(mimeTypeHeader).append(mime.str()).append("\r\n");
+        header.append(dispositionHeader).append(disposition).append("\r\n");
+        if(encoding)
+            header.append(encodingHeader).append(encoding).append("\r\n");
+    }
+
+    virtual void write(CMailInfo & info) const = 0;
+
+protected:
+    char const * encoding;
+    StringBuffer mime;
+    StringBuffer disposition;
+
+private:
+    static char const * mimeTypeHeader;
+    static char const * dispositionHeader;
+    static char const * encodingHeader;
+};
+
+char const * CMailPart::mimeTypeHeader = "Content-Type: ";
+char const * CMailPart::dispositionHeader = "Content-Disposition: ";
+char const * CMailPart::encodingHeader = "Content-Transfer-Encoding: ";
+
+class CTextMailPart : public CMailPart
+{
+public:
+    CTextMailPart(char const * text, char const * mimeType, char const * filename) : CMailPart(mimeType, filename)
+    {
+        if(mailEncode(text, buff))
+            encoding = "quoted-printable";
+    }
+
+    void write(CMailInfo & info) const
+    {
+        info.write(buff.str(), buff.length(), "mail body");
+    }
+
+private:
+    StringBuffer buff;
+};
+
+class CDataMailPart : public CMailPart
+{
+public:
+    CDataMailPart(size32_t len, const void * data, char const * mimeType, char const * filename) : CMailPart(mimeType, filename)
+    {
+        JBASE64_Encode(data, len, buff);
+        encoding = "base64";
+    }
+
+    void write(CMailInfo & info) const
+    {
+        info.write(buff.str(), buff.length(), "mail body");
+    }
+
+private:
+    StringBuffer buff;
+};
+
+class CMultiMailPart : public CMailPart
+{
+public:
+    CMultiMailPart(CMailPart const & _inlined, CMailPart const & _attachment) : CMailPart("multipart/mixed", NULL), inlined(_inlined), attachment(_attachment)
+    {
+        unsigned char rndm[12];
+        for(unsigned i=0; i<12; ++i)
+            rndm[i] = getRandom() % 256;
+        JBASE64_Encode(rndm, 12, boundary);
+        mime.append("; boundary=\"").append(boundary).append("\"");
+    }
+
+    void write(CMailInfo & info) const
+    {
+        writePart(inlined, info);
+        writePart(attachment, info);
+        writePartEnd(info);
+    }
+
+private:
+    void writePart(CMailPart const & part, CMailInfo & info) const
+    {
+        StringBuffer outbuff;
+        outbuff.append("\r\n").append("--").append(boundary).append("\r\n");
+        part.getHeader(outbuff);
+        outbuff.append("\r\n");
+        info.write(outbuff.str(), outbuff.length(), "mail body");
+        part.write(info);
+    }
+
+    void writePartEnd(CMailInfo & info) const
+    {
+        StringBuffer outbuff;
+        outbuff.append("\r\n").append("--").append(boundary).append("--").append("\r\n");
+        info.write(outbuff.str(), outbuff.length(), "mail body");
+    }
+
+private:
+    StringBuffer boundary;
+    CMailPart const & inlined;
+    CMailPart const & attachment;
+};
+
+static const char *data="DATA\r\n";
+static const char *endMail="\r\n\r\n.\r\n";
+static const char *quit="QUIT\r\n";
+
+static void doSendEmail(CMailInfo & info, CMailPart const & part)
+{
+    info.open();
+    StringBuffer outbuff;
+
+    info.read();
+    info.getHelo(outbuff);
+    info.write(outbuff.str(), outbuff.length());
+    info.read();
+
+    info.getMailFrom(outbuff.clear());
+    info.write(outbuff.str(), outbuff.length());
+    info.read();
+
+    unsigned numRcpt = info.numRecipients();
+    for(unsigned i=0; i<numRcpt; ++i)
+    {
+        info.getRecipient(i, outbuff.clear());
+        info.write(outbuff.str(), outbuff.length());
+        info.read();
+    }
+
+    info.write(data, strlen(data));
+    info.read();
+    info.getHeader(outbuff.clear());
+    part.getHeader(outbuff);
+    outbuff.append("\r\n");
+    info.write(outbuff.str(), outbuff.length(), "mail header");
+    part.write(info);
+    info.write(endMail, strlen(endMail), "end of mail body");
+    info.read();
+
+    info.write(quit, strlen(quit));
+    info.read();
+}
+
+void sendEmail(const char * to, const char * subject, const char * body, const char * mailServer, unsigned port, const char * sender, StringArray *warnings)
+{
+    CMailInfo info(to, subject, mailServer, port, sender, warnings);
+    CTextMailPart bodyPart(body, "text/plain; charset=ISO-8859-1", NULL);
+    doSendEmail(info, bodyPart);
+}
+
+void sendEmailAttachText(const char * to, const char * subject, const char * body, const char * attachment, const char * mimeType, const char * attachmentName, const char * mailServer, unsigned int port, const char * sender, StringArray *warnings)
+{
+    CMailInfo info(to, subject, mailServer, port, sender, warnings);
+    CTextMailPart inlinedPart(body, "text/plain; charset=ISO-8859-1", NULL);
+    CTextMailPart attachmentPart(attachment, mimeType, attachmentName);
+    CMultiMailPart multiPart(inlinedPart, attachmentPart);
+    doSendEmail(info, multiPart);
+}
+
+void sendEmailAttachData(const char * to, const char * subject, const char * body, size32_t lenAttachment, const void * attachment, const char * mimeType, const char * attachmentName, const char * mailServer, unsigned int port, const char * sender, StringArray *warnings)
+{
+    CMailInfo info(to, subject, mailServer, port, sender, warnings);
+    CTextMailPart inlinedPart(body, "text/plain; charset=ISO-8859-1", NULL);
+    CDataMailPart attachmentPart(lenAttachment, attachment, mimeType, attachmentName);
+    CMultiMailPart multiPart(inlinedPart, attachmentPart);
+    doSendEmail(info, multiPart);
+}
+

common/remote/rmtsmtp.hpp

@@ -0,0 +1,36 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef RMTSMTP_HPP
+#define RMTSMTP_HPP
+
+#include "jbuff.hpp"
+#include "jstring.hpp"
+
+#ifdef REMOTE_EXPORTS
+#define REMOTE_API __declspec(dllexport)
+#else
+#define REMOTE_API __declspec(dllimport)
+#endif
+
+extern REMOTE_API void sendEmail( const char * to, const char * subject, const char * body, const char * mailServer, unsigned port, const char * sender, StringArray *warnings=NULL);
+extern REMOTE_API void sendEmailAttachText(const char * to, const char * subject, const char * body, const char * attachment, const char * mimeType, const char * attachmentName, const char * mailServer, unsigned int port, const char * sender, StringArray *warnings=NULL);
+extern REMOTE_API void sendEmailAttachData(const char * to, const char * subject, const char * body, size32_t lenAttachment, const void * attachment, const char * mimeType, const char * attachmentName, const char * mailServer, unsigned int port, const char * sender, StringArray *warnings=NULL);
+
+
+#endif

common/remote/rmtspawn.cpp

@@ -0,0 +1,576 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#include "jliball.hpp"
+
+#include "platform.h"
+#include "portlist.h"
+
+
+#include "remoteerr.hpp"
+
+#include "rmtspawn.hpp"
+#include "rmtssh.hpp"
+#include "rmtpass.hpp"
+
+
+
+static CBuildVersion _bv("$HeadURL: https://svn.br.seisint.com/ecl/trunk/common/remote/rmtspawn.cpp $ $Id: rmtspawn.cpp 64028 2011-04-14 14:28:10Z nhicks $");
+
+LogMsgCategory MCdetailDebugInfo(MCdebugInfo(1000));
+
+/*
+How the remote spawning works:
+
+  i) the master starts a slave program using hoagent/ssh, passing a) who the master is and b) what reply tag to use
+  ii) the slave starts up, and starts listening on a socket based on the reply tag.
+  iii) the master connects to the socket, and is returned the ip/mpsocket that the slave is listening on.
+  iv) The master connects to the slave on that mp channel.
+
+  Complications:
+    a) slave could fail to start
+    b) slave/master could die at any point.
+    c) more than one slave can be being started on the same socket/reply tag.
+
+  Timeouts:
+    master->slave socket connect 300 seconds + buffer read + delay * 20 attempts (assuming bad connect throws an exception)
+    slave for master 5 minutes normally, max 5 mins * 20 * 20 attempts in weird cicumstances
+        read buffer with no timeout - could it get stuck here?
+
+  Q's
+    What if always connect to an orphaned slave?
+
+
+MORE: This could be improved.  Really there should be one thing connecting to the socket, that shares all the
+      attempted connections.  That would solve the problem of connecting for the wrong slave.  However, since
+      it is only a problem for running all the slaves on the same machine its probably not worth worrying about.
+*/
+
+static unsigned nextReplyTag;
+static StringAttr SSHidentfilename;
+static StringAttr SSHusername;
+static StringAttr SSHpasswordenc;
+static unsigned SSHtimeout;
+static unsigned SSHretries;
+static StringAttr SSHexeprefix;
+
+void setRemoteSpawnSSH(
+                const char *identfilename,
+                const char *username, // if NULL then disable SSH
+                const char *passwordenc,
+                unsigned timeout,
+                unsigned retries,
+                const char *exeprefix)
+{
+    SSHidentfilename.set(identfilename);
+    SSHusername.set(username);
+    SSHpasswordenc.set(passwordenc);
+    SSHtimeout = timeout;
+    SSHretries = retries;
+    SSHexeprefix.set(exeprefix);
+}
+
+void getRemoteSpawnSSH(
+                StringAttr &identfilename,
+                StringAttr &username, // if isEmpty then disable SSH
+                StringAttr &passwordenc,
+                unsigned &timeout,
+                unsigned &retries,
+                StringAttr &exeprefix)
+{
+    identfilename.set(SSHidentfilename);
+    username.set(SSHusername);
+    passwordenc.set(SSHpasswordenc);
+    timeout = SSHtimeout;
+    retries = SSHretries;
+    exeprefix.set(SSHexeprefix);
+}
+
+
+ISocket * spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint & childEP, unsigned version, const char *logdir, IAbortRequestCallback * abort, bool debug, const char *extra)
+{
+    SocketEndpoint myEP;
+    myEP.setLocalHost(0);
+    unsigned replyTag = ++nextReplyTag;
+    unsigned port = SLAVE_CONNECT_PORT + ((unsigned)kind * NUM_SLAVE_CONNECT_PORT) + getRandom() % NUM_SLAVE_CONNECT_PORT;
+    StringBuffer args;
+
+    myEP.getUrlStr(args);
+    args.append(' ').append(replyTag).append(' ').append((unsigned)kind).append(" ").append(port);
+    if (extra)
+        args.append(' ').append(extra);
+    else
+        args.append(" _");
+    if (logdir)
+        args.append(' ').append(logdir);
+
+
+    //Run the program directly if it is being run on the local machine - so hoagent/ssh doesn't need to be running...
+#if 0 // this doesn't necessarily work as relies on hoagent account dir so disable
+    if (!debug && childEP.isHost())
+    {
+        StringBuffer command;
+        command.append(exe).append(" ").append(args);
+        DWORD runcode;
+        if (!invoke_program(command.str(), runcode, false))
+        {
+            //Try running remote if not on the path.
+            if (!runRemoteProgram(exe, args.str(), childEP, debug))
+                return NULL;
+        }
+    }
+    else
+#endif
+    if (SSHusername.isEmpty()) 
+        throw MakeStringException(-1,"SSH user not specified");
+    else {
+        Owned<IFRunSSH> runssh = createFRunSSH();
+        StringBuffer cmd;
+        if (SSHexeprefix.isEmpty())
+            cmd.append(exe);
+        else {
+            const char * tail = splitDirTail(exe,cmd);
+            size32_t l = strlen(tail);
+            addPathSepChar(cmd).append(SSHexeprefix);
+            if ((l>4)&&(memcmp(tail+l-4,".exe",4)==0))  // bit odd but want .bat if prefix on windows
+                cmd.append(l-4,tail).append(".bat");
+            else
+                cmd.append(tail);
+        }
+        cmd.append(' ').append(args);
+        runssh->init(cmd.str(),SSHidentfilename,SSHusername,SSHpasswordenc,SSHtimeout,SSHretries);
+        runssh->exec(childEP,NULL,true); // need workdir? TBD
+    }
+    //Have to now try and connect to the child and get back the port it is listening on
+    bool connected;
+    unsigned slaveTag;
+    unsigned attempts = 20;
+    SocketEndpoint connectEP(childEP);
+    connectEP.port = port;
+    LOG(MCdetailDebugInfo, unknownJob, "Start connect to correct slave (%3d)", replyTag);
+    IException * error = NULL;
+    ISocket * result = NULL;
+    while (!result && attempts)
+    {
+        if (abort && abort->abortRequested())
+            break;
+
+        try
+        {
+            StringBuffer tmp;
+            connectEP.getUrlStr(tmp);
+            LOG(MCdetailDebugInfo, unknownJob, "Try to connect to slave %s",tmp.str());
+            Owned<ISocket> socket = ISocket::connect_wait(connectEP,MASTER_CONNECT_SLAVE_TIMEOUT);
+            if (socket)
+            {
+                try
+                {
+                    MemoryBuffer buffer;
+                    buffer.setEndian(__BIG_ENDIAN);
+                    buffer.append(version);
+                    myEP.ipserialize(buffer);
+                    buffer.append((unsigned)kind);
+                    buffer.append(replyTag);
+                    writeBuffer(socket, buffer);
+
+                    readBuffer(socket, buffer.clear(), 100*1000);
+                    buffer.read(connected);
+                    buffer.read(slaveTag);
+                    SocketEndpoint childEP;
+                    childEP.deserialize(buffer);
+                    if (connected)
+                    {
+                        assertex(slaveTag == replyTag);
+                        LOG(MCdetailDebugInfo, unknownJob, "Connected to correct slave (%3d)", replyTag);
+                        result = socket.getClear();
+                        break;
+                    }
+                    unsigned slaveVersion = 5;
+                    unsigned slaveKind = kind;
+                    if (buffer.getPos() < buffer.length())
+                        buffer.read(slaveVersion);
+                    if (buffer.getPos() < buffer.length())
+                        buffer.read(slaveKind);
+                    if ((slaveVersion != version) && (slaveKind == kind))
+                    {
+                        error = MakeStringException(RFSERR_VersionMismatch, RFSERR_VersionMismatch_Text, version, slaveVersion);
+                        break;
+                    }
+                    if (slaveKind != kind)
+                        LOG(MCdetailDebugInfo, unknownJob, "Connected to wrong kind of slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag);
+                    else
+                        LOG(MCdetailDebugInfo, unknownJob, "Failed to connect to correct slave (%d,%d/%d) - try again later",connected,replyTag,slaveTag);
+
+                    //Wrong slave listening, need to leave time for the other, don't count as an attempt
+                    MilliSleep(rand() % 5000 + 5000);
+                }
+                catch (IException * e)
+                {
+                    StringBuffer s;
+                    s.appendf("Retry after exception talking to slave (%d): ",replyTag);
+                    e->errorMessage(s);
+                    LOG(MCdetailDebugInfo, unknownJob, "%s", s.str());
+                    e->Release();
+                    //Probably another element just connected, and the listening socket has just been killed.
+                    //So try again.  Wait just long enough to give another thread a chance.
+                    MilliSleep(10);
+                }
+            }
+        }
+        catch (IException * e)
+        {
+            StringBuffer s;
+            LOG(MCdetailDebugInfo, unknownJob, e, s.appendf("Failed to connect to slave (%d) (try again): ", replyTag).str());
+            e->Release();
+            // No socket listening or contention - try again fairly soon
+            MilliSleep(rand()%400+100);
+            attempts--;
+        }
+    }
+    if (error)
+        throw error;
+    if (!result)
+        ERRLOG("Failed to connect to slave (%d)", replyTag);
+    return result;
+}
+
+
+//---------------------------------------------------------------------------
+
+CRemoteParentInfo::CRemoteParentInfo()
+{
+}
+
+
+bool CRemoteParentInfo::processCommandLine(int argc, char * argv[], StringBuffer &logdir)
+{
+    if (argc <= 4)
+        return false;
+
+    parent.set(argv[1]);
+    replyTag = atoi(argv[2]);
+    kind = (SpawnKind)atoi(argv[3]);
+    port = atoi(argv[4]);
+    // 5 is extra (only used in logging)
+    if (argc>6)
+        logdir.clear().append(argv[6]);
+
+    return true;
+}
+
+void CRemoteParentInfo::log()
+{
+    StringBuffer temp;
+    LOG(MCdebugProgress, unknownJob, "Starting remote slave.  Master=%s reply=%d port=%d", parent.getUrlStr(temp).str(), replyTag, port);
+}
+
+bool CRemoteParentInfo::sendReply(unsigned version)
+{
+    unsigned listenAttempts = 20;
+    MemoryBuffer buffer;
+    buffer.setEndian(__BIG_ENDIAN);
+    while (listenAttempts--)
+    {
+        try
+        {
+            LOG(MCdebugInfo(1000), unknownJob, "Ready to listen. reply=%d port=%d", replyTag, port);
+            Owned<ISocket> listen = ISocket::create(port, 1);
+            if (listen)
+            {
+                unsigned receiveAttempts = 10;
+                unsigned connectVersion;
+                unsigned connectTag;
+                unsigned connectKind;
+                StringBuffer masterIPtext;
+                while (receiveAttempts--)
+                {
+                    try
+                    {
+                        LOG(MCdebugInfo(1000), unknownJob, "Ready to accept connection. reply=%d", replyTag);
+
+                        if (!listen->wait_read(SLAVE_LISTEN_FOR_MASTER_TIMEOUT))
+                        {
+                            LOG(MCdebugInfo(1000), unknownJob, "Gave up waiting for a connection. reply=%d", replyTag);
+                            return false;
+                        }
+
+                        Owned<ISocket> connect = listen->accept();
+                        readBuffer(connect, buffer.clear());
+                        buffer.read(connectVersion);
+                        bool same = false;
+                        unsigned replyVersion = version;
+                        IpAddress masterIP;
+                        masterIP.ipdeserialize(buffer);
+                        buffer.read(connectKind);
+                        if (version == connectVersion)
+                        {
+                            buffer.read(connectTag);
+                            masterIP.getIpText(masterIPtext.clear());
+
+                            LOG(MCdebugInfo(1000), unknownJob, "Process incoming connection. reply=%d got(%d,%s)", replyTag,connectTag,masterIPtext.str());
+
+                            same = (kind == connectKind) && masterIP.ipequals(parent) && (connectTag == replyTag);
+                        }
+                        else
+                        {
+                            //If connected to a different kind of slave, fake the version number
+                            //so it doesn't think there is a version mismatch
+                            //can remove when all .exes have new code.
+                            if (connectKind != kind)
+                            {
+                                LOG(MCdebugInfo(1000), unknownJob, "Connection for wrong slave kind - ignore", connectKind, kind);
+                                replyVersion = connectVersion;
+                            }
+                        }
+
+                        buffer.clear().append(same).append(replyTag);
+                        SocketEndpoint ep(1U);
+                        ep.serialize(buffer);
+                        buffer.append(version);
+                        buffer.append(kind);
+                        writeBuffer(connect, buffer);
+
+                        if (same)
+                        {
+                            socket.setown(connect.getClear());
+                            LOG(MCdebugInfo(1000), unknownJob, "Connection matched - continue....");
+                            return true;
+                        }
+                        if ((connectKind == kind) && (version != connectVersion))
+                        {
+                            LOG(MCdebugInfo, unknownJob, "Version mismatch - terminating slave process expected %d got %d", version, connectVersion);
+                            return false;
+                        }
+                    }
+                    catch (IException * e)
+                    {
+                        EXCLOG(e, "Error reading information from master: ");
+                        e->Release();
+                    }
+                    MilliSleep(50);
+                }
+            }
+        }
+        catch (IException * e)
+        {
+            EXCLOG(e, "Failed to create master listener: ");
+            e->Release();
+        }
+        MilliSleep(rand() % 3000 + 2000);
+    }
+
+    return false;
+}
+
+//---------------------------------------------------------------------------
+
+
+CRemoteSlave::CRemoteSlave(const char * _name, unsigned _tag, unsigned _version, bool _stayAlive)
+{
+    slaveName.set(_name);
+    tag = _tag;
+    stayAlive = _stayAlive;
+    version = _version;
+}
+
+void CRemoteSlave::run(int argc, char * argv[])
+{
+    StringBuffer logFile;
+    CRemoteParentInfo info;
+
+    bool paramsok = info.processCommandLine(argc, argv, logFile);
+    if (logFile.length()==0) { // not expected!
+#ifdef _WIN32
+        //logFile.append("c:\\");   // don't write to root on windows!
+#else
+        if (checkDirExists("/c$"))
+            logFile.append("/c$/");
+#endif
+    }
+    if (logFile.length())
+        addPathSepChar(logFile);
+    logFile.append(slaveName);
+    addFileTimestamp(logFile, true);
+    logFile.append(".log");
+    attachStandardFileLogMsgMonitor(logFile.str(), 0, MSGFIELD_STANDARD, MSGAUD_all, MSGCLS_all, TopDetail, false, true, true);
+    queryLogMsgManager()->removeMonitor(queryStderrLogMsgHandler());        // no point logging output to screen if run remote!
+    LOG(MCdebugProgress, unknownJob, "Starting %s %s %s %s %s %s %s",slaveName.get(),(argc>1)?argv[1]:"",(argc>2)?argv[2]:"",(argc>3)?argv[3]:"",(argc>4)?argv[4]:"",(argc>5)?argv[5]:"",(argc>6)?argv[6]:"");
+
+
+    if (paramsok)
+    {
+        info.log();
+        EnableSEHtoExceptionMapping();
+
+        CachedPasswordProvider passwordProvider;
+        setPasswordProvider(&passwordProvider);
+        try
+        {
+            if (info.sendReply(version))
+            {
+                ISocket * masterSocket = info.queryMasterSocket();
+
+                unsigned timeOut = RMTTIME_RESPONSE_MASTER;
+                do
+                {
+                    MemoryBuffer msg;
+                    MemoryBuffer results;
+                    results.setEndian(__BIG_ENDIAN);
+
+                    bool ok = false;
+                    Linked<IException> error;
+                    try
+                    {
+                        if (!catchReadBuffer(masterSocket, msg, timeOut))
+                            throwError(RFSERR_MasterSeemsToHaveDied);
+                        msg.setEndian(__BIG_ENDIAN);
+                        byte action;
+                        msg.read(action);
+                        passwordProvider.clear();
+                        passwordProvider.deserialize(msg);
+
+                        ok = processCommand(action, masterSocket, msg, results);
+                    }
+                    catch (IException * e)
+                    {
+                        PrintExceptionLog(e, slaveName.get());
+                        error.setown(e);
+                    }
+                    catch (RELEASE_CATCH_ALL)
+                    {
+                        LOG(MCwarning, unknownJob, "Server seems to have crashed - close done gracefully");
+                        error.setown(MakeStringException(999, "Server seems to have crashed - close done gracefully"));
+                    }
+
+                    msg.setEndian(__BIG_ENDIAN);
+                    msg.clear().append(true).append(ok);
+                    serializeException(error, msg);
+                    msg.append(results);
+                    catchWriteBuffer(masterSocket, msg);
+
+                    LOG(MCdebugProgress, unknownJob, "Results sent from slave %d", info.replyTag);
+
+                    //Acknowledgement before closing down...
+                    msg.clear();
+                    if (catchReadBuffer(masterSocket, msg, RMTTIME_RESPONSE_MASTER))
+                    {
+                        msg.read(ok);
+                        assertex(ok);
+                    }
+
+                    if (error)
+                        break;
+                    timeOut = 24*60*60*1000;
+                } while (stayAlive);
+
+                LOG(MCdebugProgress, unknownJob, "Terminate acknowledgement received from master for slave %d", info.replyTag);
+            }
+        }
+        catch (IException * e)
+        {
+            PrintExceptionLog(e, slaveName.get());
+            e->Release();
+        }
+
+        setPasswordProvider(NULL);
+    }
+    LOG(MCdebugProgress, unknownJob, "Stopping %s", slaveName.get());
+}
+
+
+
+#if 0
+void checkForRemoteAbort(ICommunicator * communicator, mptag_t tag)
+{
+    if (communicator->probe(1, tag, NULL, 0)!=0)
+    {
+        CMessageBuffer msg;
+        if (!communicator->recv(msg, 1, tag, NULL))
+            throwError(RFSERR_TimeoutWaitMaster);
+
+        bool aborting;
+        msg.setEndian(__BIG_ENDIAN);
+        msg.read(aborting);
+        if (aborting)
+            throwAbortException();
+    }
+}
+
+void sendRemoteAbort(INode * node, mptag_t tag)
+{
+    CMessageBuffer msg;
+    msg.clear().append(true);
+    queryWorldCommunicator().send(msg, node, tag, MP_ASYNC_SEND);
+}
+#endif
+
+void checkForRemoteAbort(ISocket * socket)
+{
+    if (socket->wait_read(0))
+    {
+        MemoryBuffer msg;
+        if (!catchReadBuffer(socket, msg))
+            throwError(RFSERR_TimeoutWaitMaster);
+
+        bool aborting;
+        msg.setEndian(__BIG_ENDIAN);
+        msg.read(aborting);
+        if (aborting)
+            throwAbortException();
+    }
+}
+
+bool sendRemoteAbort(ISocket * socket)
+{
+    LOG(MCdebugInfo, unknownJob, "Send abort to remote slave (%d)", isAborting());
+
+    MemoryBuffer msg;
+    msg.append(true);
+    return catchWriteBuffer(socket, msg);
+}
+
+#if 0
+bool sendSlaveCommand(INode * remote, CMessageBuffer & msg, unsigned tag)
+{
+    if (!queryWorldCommunicator().send(msg, remote, tag, FTTIME_CONNECT_SLAVE))
+        throwError1(DFTERR_TimeoutWaitConnect, url.str());
+
+    bool done;
+    loop
+    {
+        msg.clear();
+        if (!queryWorldCommunicator().recv(msg, remote, tag, NULL, FTTIME_PROGRESS))
+            throwError1(DFTERR_TimeoutWaitSlave, url.str());
+
+        msg.setEndian(__BIG_ENDIAN);
+        msg.read(done);
+        if (!done)
+            return SCcontinue;
+    }
+
+    bool ok;
+    msg.read(ok);
+    error.setown(deserializeException(msg));
+    if (error)
+        sprayer.setHadError();
+
+    msg.clear().append(true);
+    queryWorldCommunicator().send(msg, remote, tag);
+}
+
+#endif

common/remote/rmtspawn.hpp

@@ -0,0 +1,102 @@
+/*##############################################################################
+
+    Copyright (C) 2011 HPCC Systems.
+
+    All rights reserved. This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU Affero General Public License as
+    published by the Free Software Foundation, either version 3 of the
+    License, or (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU Affero General Public License for more details.
+
+    You should have received a copy of the GNU Affero General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+############################################################################## */
+
+#ifndef RMTSPAWN_HPP
+#define RMTSPAWN_HPP
+
+#ifdef REMOTE_EXPORTS
+#define REMOTE_API __declspec(dllexport)
+#else
+#define REMOTE_API __declspec(dllimport)
+#endif
+
+#define SLAVE_LISTEN_FOR_MASTER_TIMEOUT             5 * 60 * 1000           // 2 minutes
+#define MASTER_CONNECT_SLAVE_TIMEOUT                5 * 60 * 1000
+
+#define RMTTIME_CLOSEDOWN                           1000 * 1000         // How long slave to wait for acknowledge of closedown notification
+#define RMTTIME_CONNECT_SLAVE                       1000 * 1000         // How long to wait for connection from slave
+#define RMTTIME_RESPONSE_MASTER                     10 * 60 * 1000      // How long to wait for a response from master - should be immediate!
+
+enum SpawnKind
+{
+    SPAWNdfu, SPAWNlast
+};
+
+interface IAbortRequestCallback;
+
+extern REMOTE_API ISocket * spawnRemoteChild(SpawnKind kind, const char * exe, const SocketEndpoint & remoteEP, unsigned version, const char *logdir, IAbortRequestCallback * abort = NULL, bool debug=false, const char *extra=NULL);
+extern REMOTE_API void setRemoteSpawnSSH(
+                const char *identfilename,
+                const char *username, // if NULL then disable SSH
+                const char *passwordenc,
+                unsigned timeout,
+                unsigned retries,
+                const char *exeprefix);
+extern REMOTE_API void getRemoteSpawnSSH(
+                StringAttr &identfilename,
+                StringAttr &username, // if isEmpty then disable SSH
+                StringAttr &passwordenc,
+                unsigned &timeout,
+                unsigned &retries,
+                StringAttr &exeprefix);
+
+class REMOTE_API CRemoteParentInfo
+{
+public:
+    CRemoteParentInfo();
+
+    ISocket * queryMasterSocket()               { return socket; }
+    bool processCommandLine(int argc, char * argv[], StringBuffer &logdir);
+    void log();
+    bool sendReply(unsigned version);
+
+public:
+    Owned<ISocket>      socket;
+    SocketEndpoint      parent;
+    unsigned            replyTag;
+    unsigned            port;
+    SpawnKind           kind;
+};
+
+
+class REMOTE_API CRemoteSlave
+{
+public:
+    CRemoteSlave(const char * name, unsigned _tag, unsigned _version, bool _stayAlive);
+
+    void run(int argc, char * argv[]);
+    virtual bool processCommand(byte action, ISocket * masterSocket, MemoryBuffer & msg, MemoryBuffer & results) = 0;
+
+protected:
+    void stopRunning()              { stayAlive = false; }
+
+protected:
+    StringAttr      slaveName;
+    unsigned        tag;
+    bool            stayAlive;
+    unsigned        version;
+};
+
+
+//extern REMOTE_API void checkForRemoteAbort(ICommunicator * communicator, mptag_t tag);
+//extern REMOTE_API void sendRemoteAbort(INode * node, mptag_t tag);
+extern REMOTE_API void checkForRemoteAbort(ISocket * socket);
+extern REMOTE_API bool sendRemoteAbort(ISocket * socket);
+
+
+#endif