Merge pull request #11988 from richardkchapman/java-persist-experiment

HPCC-21142 Add persistence options to Java embed

Reviewed-by: Gavin Halliday <ghalliday@hpccsystems.com>

ecl/hql/hqlatoms.cpp

@@ -35,6 +35,7 @@ IIdAtom * bindUnicodeParamId;
 IIdAtom * bindUnsignedParamId;
 IIdAtom * bindUnsignedSizeParamId;
 IIdAtom * bindUtf8ParamId;
+IIdAtom * checkImportId;
 IIdAtom * compileEmbeddedScriptId;
 IIdAtom * getEmbedContextId;
 IIdAtom * getBooleanResultId;
@@ -505,6 +506,7 @@ MODULE_INIT(INIT_PRIORITY_HQLATOM)
     MAKEID(bindUnsignedParam);
     MAKEID(bindUnsignedSizeParam);
     MAKEID(bindUtf8Param);
+    MAKEID(checkImport);
     MAKEID(compileEmbeddedScript);
     defaultFieldNameId = createIdAtom("__f1__");
     MAKEID(fail);

ecl/hql/hqlatoms.hpp

@@ -35,6 +35,7 @@ extern HQL_API IIdAtom * bindUnicodeParamId;
 extern HQL_API IIdAtom * bindUnsignedParamId;
 extern HQL_API IIdAtom * bindUnsignedSizeParamId;
 extern HQL_API IIdAtom * bindUtf8ParamId;
+extern HQL_API IIdAtom * checkImportId;
 extern HQL_API IIdAtom * compileEmbeddedScriptId;
 extern HQL_API IIdAtom * defaultFieldNameId;
 extern HQL_API IIdAtom * failId;

ecl/hql/hqlgram2.cpp

@@ -946,9 +946,9 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
         OwnedHqlExpr threadlocal = pluginScope->lookupSymbol(threadlocalId, LSFpublic, lookupCtx);
         if (matchesBoolean(threadlocal, true))
             args.append(*createAttribute(_threadlocal_Atom));
-        OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(syntaxCheckId, LSFpublic, lookupCtx);
-        OwnedHqlExpr precompile = pluginScope->lookupSymbol(precompileId, LSFpublic, lookupCtx);
-        if (!isImport & (syntaxCheckFunc || precompile))
+        OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(isImport ? checkImportId : syntaxCheckId, LSFpublic, lookupCtx);
+        OwnedHqlExpr precompile = isImport ? nullptr : pluginScope->lookupSymbol(precompileId, LSFpublic, lookupCtx);
+        if (syntaxCheckFunc || precompile)
             args.append(*createExprAttribute(_original_Atom, LINK(language)));  // Add this so that we can complete the syntax check/precompile later (in checkEmbedBody), as we don't know func name until then
         args.append(*createExprAttribute(languageAtom, getEmbedContextFunc.getClear()));
         IHqlExpression *projectedAttr = queryAttribute(projectedAtom, args);
@@ -1017,7 +1017,7 @@ IHqlExpression * HqlGram::processEmbedBody(const attribute & errpos, IHqlExpress
 
     result.setown(createLocationAnnotation(result.getClear(), errpos.pos));
 
-    if (queryParametered())
+    if (queryParametered())  // MORE - this code should be in checkEmbedBody?
     {
         HqlExprArray args;
         args.append(*LINK(result));
@@ -9743,7 +9743,8 @@ IHqlExpression * HqlGram::checkEmbedBody(const attribute & errpos, DefineIdSt *
         language = language->queryChild(0);
         IHqlExpression *embedText = body->queryChild(0);
         IHqlScope *pluginScope = language->queryScope();
-        OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(syntaxCheckId, LSFpublic, lookupCtx);
+        bool isImport = body->hasAttribute(importAtom);
+        OwnedHqlExpr syntaxCheckFunc = pluginScope->lookupSymbol(isImport ? checkImportId : syntaxCheckId, LSFpublic, lookupCtx);
         bool failedSyntaxCheck = false;
         if (syntaxCheckFunc)
         {
@@ -9804,28 +9805,31 @@ IHqlExpression * HqlGram::checkEmbedBody(const attribute & errpos, DefineIdSt *
             }
             else
             {
-                reportWarning(CategoryError, WRN_EMBEDFOLD, errpos.pos, "Embedded syntax check function could not be called");
+                DBGLOG("Embedded syntax check function could not be called");  // Don't make it a warning - it happens on client machines all the time...
             }
         }
-        OwnedHqlExpr precompile = pluginScope->lookupSymbol(precompileId, LSFpublic, lookupCtx);
-        if (precompile && !failedSyntaxCheck && !lookupCtx.syntaxChecking())
+        if (!isImport)
         {
-            HqlExprArray precompileArgs;
-            precompileArgs.append(*createConstant(defineid->id->queryStr()));
-            embedText->unwindList(precompileArgs, no_comma);
-            // Replace queryText with compiled version of it
-            precompileArgs.append(*argNamesParam.getLink());
-            precompileArgs.append(*compilerOptions.getLink());
-            precompileArgs.append(*persistOptions.getLink());
-            OwnedHqlExpr compiled = createBoundFunction(this, precompile, precompileArgs, lookupCtx.functionCache, true);
-            OwnedHqlExpr folded = foldHqlExpression(compiled);  // Best to fold here if we can, as the syntax check may cache on the assumption that next call is the compile. There may be a better way
-            if (folded)
-                bodyArgs.replace(*folded.getClear(), 0);
-            else
-                bodyArgs.replace(*compiled.getClear(), 0);
-            bodyArgs.append(*createAttribute(_precompile_Atom));
+            OwnedHqlExpr precompile = pluginScope->lookupSymbol(precompileId, LSFpublic, lookupCtx);
+            if (precompile && !failedSyntaxCheck && !lookupCtx.syntaxChecking())
+            {
+                HqlExprArray precompileArgs;
+                precompileArgs.append(*createConstant(defineid->id->queryStr()));
+                embedText->unwindList(precompileArgs, no_comma);
+                // Replace queryText with compiled version of it
+                precompileArgs.append(*argNamesParam.getLink());
+                precompileArgs.append(*compilerOptions.getLink());
+                precompileArgs.append(*persistOptions.getLink());
+                OwnedHqlExpr compiled = createBoundFunction(this, precompile, precompileArgs, lookupCtx.functionCache, true);
+                OwnedHqlExpr folded = foldHqlExpression(compiled);  // Best to fold here if we can, as the syntax check may cache on the assumption that next call is the compile. There may be a better way
+                if (folded)
+                    bodyArgs.replace(*folded.getClear(), 0);
+                else
+                    bodyArgs.replace(*compiled.getClear(), 0);
+                bodyArgs.append(*createAttribute(_precompile_Atom));
+            }
+            return body->clone(bodyArgs);
         }
-        return body->clone(bodyArgs);
     }
     return nullptr;
 }

plugins/javaembed/HpccClassLoader.java

@@ -19,12 +19,15 @@ package com.HPCCSystems;
 
 import java.net.*;
 import java.util.Hashtable;
+import java.lang.reflect.Method;
+import java.lang.Throwable;
+
 public class HpccClassLoader extends java.net.URLClassLoader
 {
     private long bytecode;
     private int bytecodeLen;
-    private native Class defineClassForEmbed(int bytecodeLen, long bytecode, String name);
-    private Hashtable<String, Class> classes = new Hashtable<>();
+    private native Class<?> defineClassForEmbed(int bytecodeLen, long bytecode, String name);
+    private Hashtable<String, Class<?>> classes = new Hashtable<>();
     private HpccClassLoader(java.net.URL [] urls, ClassLoader parent, int _bytecodeLen, long _bytecode, String dllname)
     {
         super(urls, parent);
@@ -34,7 +37,7 @@ public class HpccClassLoader extends java.net.URLClassLoader
     }
     public synchronized Class<?> findClass(String className) throws ClassNotFoundException
     {
-        Class result = (Class) classes.get(className);
+        Class<?> result = classes.get(className);
         if (result == null)
         {
             result = defineClassForEmbed(bytecodeLen, bytecode, className.replace(".","/"));
@@ -46,4 +49,33 @@ public class HpccClassLoader extends java.net.URLClassLoader
     {
         return new HpccClassLoader(urls, parent, _bytecodeLen, _bytecode, dllname); 
     }
+    
+    public static String getSignature(Method m)
+    {
+        StringBuilder sb = new StringBuilder("(");
+        for(Class<?> c : m.getParameterTypes())
+        { 
+            String sig=java.lang.reflect.Array.newInstance(c, 0).toString();
+            sb.append(sig.substring(1, sig.indexOf('@')));
+        }
+        sb.append(')');
+        if (m.getReturnType()==void.class)
+            sb.append("V");
+        else
+        {
+            String sig=java.lang.reflect.Array.newInstance(m.getReturnType(), 0).toString();
+            sb.append(sig.substring(1, sig.indexOf('@')));
+        }
+        return sb.toString();
+    }
+
+    /* get signature for first method with given name */
+    public static String getSignature ( Class<?> clazz, String simpleName )
+    {
+        Method[] methods = clazz.getMethods();
+        for (Method m : methods)
+            if (m.getName().equals(simpleName))
+                return getSignature(m);
+        return null;
+    }
 }

plugins/javaembed/java.ecllib

@@ -19,10 +19,12 @@ EXPORT Language := SERVICE : plugin('javaembed')
   integer getEmbedContext():cpp,pure,namespace='javaembed',entrypoint='getEmbedContext',prototype='IEmbedContext* getEmbedContext()',fold;
   DATA precompile(const varstring funcname, UTF8 body, const varstring argnames, const varstring compileOptions, const varstring persistOptions):cpp,pure,namespace='javaembed',entrypoint='precompile',fold;
   STRING syntaxCheck(const varstring funcname, UTF8 body, const varstring argnames, const varstring compileOptions, const varstring persistOptions):cpp,pure,namespace='javaembed',entrypoint='syntaxCheck',fold;
+  STRING checkImport(const varstring funcname, UTF8 importString, const varstring argnames, const varstring compileOptions, const varstring persistOptions):cpp,pure,namespace='javaembed',entrypoint='checkImport',fold;
 END;
 EXPORT getEmbedContext := Language.getEmbedContext;
 EXPORT precompile := Language.precompile;
 EXPORT syntaxCheck := Language.syntaxCheck;
+EXPORT checkImport := Language.checkImport;
 EXPORT boolean supportsImport := true;
 EXPORT boolean supportsScript := true;
 EXPORT boolean threadlocal := true;

roxie/ccd/ccdmain.cpp

@@ -427,7 +427,16 @@ int STARTQUERY_API start_query(int argc, const char *argv[])
     setTerminateOnSEH();
     init_signals();
     // We need to do the above BEFORE we call InitModuleObjects
-    InitModuleObjects();
+    try
+    {
+        InitModuleObjects();
+    }
+    catch (IException *E)
+    {
+        EXCLOG(E);
+        E->Release();
+        return EXIT_FAILURE;
+    }
     init_signals();
 
     // stand alone usage only, not server

testing/regress/ecl/java-fold.ecl

@@ -0,0 +1,27 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2014 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+
+import java;
+string jcat(string a, string b) := IMPORT(java, 'JavaCat.cat:(Ljava/lang/String;Ljava/lang/String;)Ljava/lang/String;' : classpath('/opt/HPCCSystems/classes'),FOLD);
+integer jadd(integer a, integer b) := IMPORT(java, 'JavaCat.add:(II)I' :FOLD);
+
+ASSERT(jcat('Hello',' world')='Hello world', CONST);
+ASSERT(jadd(1,2)=3, CONST);
+OUTPUT('ok');

testing/regress/ecl/javapersist.ecl

@@ -0,0 +1,51 @@
+/*##############################################################################
+
+    HPCC SYSTEMS software Copyright (C) 2018 HPCC Systems.
+
+    Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+    Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+############################################################################## */
+
+//class=embedded
+//class=3rdparty
+//nohthor
+
+import java;
+
+integer accumulate(integer a) := EMBED(Java: persist('thread'))
+public class persisty
+{
+  public persisty()
+  {
+  }
+  public synchronized int accumulate(int a)
+  {
+    tot = tot + a;
+    return tot;
+  }
+  private int tot = 0;
+}
+ENDEMBED;
+
+r := record
+  integer i;
+end;
+
+r t(r l) := TRANSFORM
+  SELF.i := accumulate(l.i);
+END;
+
+d1 := DATASET([{1}, {2}, {3}], r);
+d2 := DATASET([{3}, {4}, {5}], r);
+
+accumulated := PROJECT(d1, t(LEFT))+PROJECT(d2, t(LEFT));
+max(accumulated, i) = 3+4+5;  // The order cannot be predicted but the max should be consistent.

testing/regress/ecl/key/java-fold.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>ok</Result_1></Row>
+</Dataset>

testing/regress/ecl/key/javapersist.xml

@@ -0,0 +1,3 @@
+<Dataset name='Result 1'>
+ <Row><Result_1>true</Result_1></Row>
+</Dataset>