Merge pull request #8633 from JamesDeFabia/14846MultiPartPackages

HPCC-14846 Document Multipart Packagemaps use

Reviewed-By: Anthony Fishbeck <anthony.fishbeck@lexisnexis.com>
Reviewed-By: Richard Chapman <rchapman@hpccsystems.com>

docs/HPCCClientTools/CT_Mods/CT_ECL_CLI.xml

@@ -2393,6 +2393,406 @@ ecl packagemap add roxie mypackagemap.pkg --daliip=192.168.11.11
           </informaltable></para>
       </sect2>
 
+      <sect2 id="eclpackagemapaddpart" role="brk">
+        <title>ecl packagemap add-part</title>
+
+        <para><emphasis role="bold">ecl packagemap add-part &lt;target&gt;
+        &lt;pmid&gt; &lt;filename&gt; </emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl packagemap add-part roxie multipart.pkg addresses.pkg 
+</programlisting>
+
+        <para>The packagemap add-part command adds additional packagemap
+        content to an existing packagemap</para>
+
+        <para><informaltable colsep="1" frame="all" rowsep="1">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="175.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl packagemap add-part</entry>
+
+                  <entry>Calls the packagemap add-part command.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Actions</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>add-part</entry>
+
+                  <entry><para>Adds additional packagemap content to an
+                  existing packagemap</para></entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Arguments</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>target</entry>
+
+                  <entry>Name of target to use when adding packagemap
+                  part</entry>
+                </row>
+
+                <row>
+                  <entry>pmid</entry>
+
+                  <entry>Identifier of packagemap to add the part to</entry>
+                </row>
+
+                <row>
+                  <entry>filename</entry>
+
+                  <entry>one or more part files</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>--part-name</entry>
+
+                  <entry>Name of part being added (defaults to
+                  filename)</entry>
+                </row>
+
+                <row>
+                  <entry>--delete-prev</entry>
+
+                  <entry>Replace an existing part with matching name</entry>
+                </row>
+
+                <row>
+                  <entry> --daliip=&lt;ip&gt;</entry>
+
+                  <entry>IP of the remote Dali to use for logical file
+                  lookups</entry>
+                </row>
+
+                <row>
+                  <entry>--global-scope</entry>
+
+                  <entry>The specified packagemap is shared across multiple
+                  targets</entry>
+                </row>
+
+                <row>
+                  <entry>--source-process=&lt;value&gt;</entry>
+
+                  <entry>Process cluster to copy files from</entry>
+                </row>
+
+                <row>
+                  <entry>--allow-foreign</entry>
+
+                  <entry>Do not fail if foreign files are used in
+                  packagemap</entry>
+                </row>
+
+                <row>
+                  <entry>--preload-all</entry>
+
+                  <entry>Set preload files option for all packages</entry>
+                </row>
+
+                <row>
+                  <entry>--update-super-files </entry>
+
+                  <entry>Update local DFS superfiles if remote DALI has
+                  changed</entry>
+                </row>
+
+                <row>
+                  <entry>--update-clone-from</entry>
+
+                  <entry>Update local clone from location if remote DALI has
+                  changed</entry>
+                </row>
+
+                <row>
+                  <entry>--dont-append-cluster</entry>
+
+                  <entry>Use only to avoid locking issues due to adding
+                  cluster to file</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+
+                <row>
+                  <entry>-s, --server</entry>
+
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
+                </row>
+
+                <row>
+                  <entry>--port</entry>
+
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
+                </row>
+
+                <row>
+                  <entry>-ssl</entry>
+
+                  <entry>Use SSL to secure the connection to the
+                  server.</entry>
+                </row>
+
+                <row>
+                  <entry>-u, --username</entry>
+
+                  <entry>The username (if necessary)</entry>
+                </row>
+
+                <row>
+                  <entry>-pw, --password</entry>
+
+                  <entry>The password (if necessary)</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 id="eclpackagemapgetpart" role="brk">
+        <title>ecl packagemap get-part</title>
+
+        <para><emphasis role="bold">ecl packagemap get-part &lt;target&gt;
+        &lt;packagemap&gt; &lt;partname&gt; </emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl packagemap get-part roxie multipart.pkg contacts
+</programlisting>
+
+        <para>The get-part command fetches the given part from the given
+        packagemap <informaltable colsep="1" frame="all" rowsep="1">
+            <tgroup cols="2">
+              <colspec align="left" colwidth="175.55pt" />
+
+              <colspec colwidth="384.85pt" />
+
+              <tbody>
+                <row>
+                  <entry>ecl packagemap get-part</entry>
+
+                  <entry>Calls the packagemap get-part command.</entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Actions</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>get-part</entry>
+
+                  <entry><para>Fetches the given part from the given
+                  packagemap </para></entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Arguments</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>target</entry>
+
+                  <entry>Name of target to use when adding packagemap
+                  part</entry>
+                </row>
+
+                <row>
+                  <entry>packagemap</entry>
+
+                  <entry>Name of the packagemap containing the part</entry>
+                </row>
+
+                <row>
+                  <entry>partname</entry>
+
+                  <entry>Name of the part to retrieve </entry>
+                </row>
+
+                <row>
+                  <entry><emphasis role="bold">Options</emphasis></entry>
+                </row>
+
+                <row>
+                  <entry>--global-scope</entry>
+
+                  <entry>The specified packagemap is sharable across multiple
+                  targets</entry>
+                </row>
+
+                <row>
+                  <entry>-v, --verbose</entry>
+
+                  <entry>Output additional tracing information</entry>
+                </row>
+
+                <row>
+                  <entry>-s, --server</entry>
+
+                  <entry>The IP Address or hostname of ESP server running ECL
+                  Watch services</entry>
+                </row>
+
+                <row>
+                  <entry>--port</entry>
+
+                  <entry>The ECL Watch services port (Default is 8010)</entry>
+                </row>
+
+                <row>
+                  <entry>-ssl</entry>
+
+                  <entry>Use SSL to secure the connection to the
+                  server.</entry>
+                </row>
+
+                <row>
+                  <entry>-u, --username</entry>
+
+                  <entry>The username (if necessary)</entry>
+                </row>
+
+                <row>
+                  <entry>-pw, --password</entry>
+
+                  <entry>The password (if necessary)</entry>
+                </row>
+              </tbody>
+            </tgroup>
+          </informaltable></para>
+      </sect2>
+
+      <sect2 id="eclpackagemapremovepart" role="brk">
+        <title>ecl packagemap remove-part</title>
+
+        <para><emphasis role="bold">ecl packagemap remove-part &lt;target&gt;
+        &lt;pmid&gt; &lt;partname&gt; </emphasis></para>
+
+        <para>Examples:</para>
+
+        <programlisting>ecl packagemap remove-part roxie multipart.pkg contacts
+</programlisting>
+
+        <para>The remove-part command will remove the given part from the
+        given packagemap</para>
+
+        <para></para>
+
+        <informaltable colsep="1" frame="all" rowsep="1">
+          <tgroup cols="2">
+            <colspec align="left" colwidth="175.55pt" />
+
+            <colspec colwidth="384.85pt" />
+
+            <tbody>
+              <row>
+                <entry>ecl packagemap remove-part</entry>
+
+                <entry>Calls the packagemap remove-part command.</entry>
+              </row>
+
+              <row>
+                <entry><emphasis role="bold">Actions</emphasis></entry>
+              </row>
+
+              <row>
+                <entry>remove-part</entry>
+
+                <entry><para>Removes the given part from the given
+                packagemap</para></entry>
+              </row>
+
+              <row>
+                <entry><emphasis role="bold">Arguments</emphasis></entry>
+              </row>
+
+              <row>
+                <entry>target</entry>
+
+                <entry>Name of target to use </entry>
+              </row>
+
+              <row>
+                <entry>packagemap</entry>
+
+                <entry>Name of the packagemap containing the part</entry>
+              </row>
+
+              <row>
+                <entry>partname</entry>
+
+                <entry>Name of the part to remove</entry>
+              </row>
+
+              <row>
+                <entry><emphasis role="bold">Options</emphasis></entry>
+              </row>
+
+              <row>
+                <entry>--global-scope</entry>
+
+                <entry>The specified packagemap is sharable across multiple
+                targets</entry>
+              </row>
+
+              <row>
+                <entry>-v, --verbose</entry>
+
+                <entry>Output additional tracing information</entry>
+              </row>
+
+              <row>
+                <entry>-s, --server</entry>
+
+                <entry>The IP Address or hostname of ESP server running ECL
+                Watch services</entry>
+              </row>
+
+              <row>
+                <entry>--port</entry>
+
+                <entry>The ECL Watch services port (Default is 8010)</entry>
+              </row>
+
+              <row>
+                <entry>-ssl</entry>
+
+                <entry>Use SSL to secure the connection to the server.</entry>
+              </row>
+
+              <row>
+                <entry>-u, --username</entry>
+
+                <entry>The username (if necessary)</entry>
+              </row>
+
+              <row>
+                <entry>-pw, --password</entry>
+
+                <entry>The password (if necessary)</entry>
+              </row>
+            </tbody>
+          </tgroup>
+        </informaltable>
+      </sect2>
+
       <sect2 id="eclpackagevalidate" role="brk">
         <title>ecl packagemap validate</title>
 

docs/RoxieReference/RoxieRefMods/Packages.xml

@@ -303,17 +303,163 @@
   </sect1>
 
   <sect1>
-    <title>Working with Packages using the ecl command
-    line tool</title>
+    <title>Multi-Part Packagemaps</title>
+
+    <para>In version 6.0.0 or later, you can organize your packagemap file
+    mappings into a collection of files that define the files for some subset
+    of queries. These subsets are called parts and can be used to organize by
+    various groupings such as function, files, or developer, etc.</para>
+
+    <para>With a single part packagmap file like:</para>
+
+    <para><emphasis role="bold">master.pkg</emphasis></para>
+
+    <para><programlisting>&lt;RoxiePackages&gt;
+  &lt;Package id="my_base" preload="1"&gt;
+    &lt;SuperFile id="thor1::people::key::person_superkey"&gt;
+       &lt;SubFile value="thor1::key::people::person_file_today"/&gt;
+    &lt;/SuperFile&gt;
+  &lt;/Package&gt;
+  &lt;Package id="another_base" preload="1"&gt;
+    &lt;SuperFile id="thor1::people::key::contact_superkey"&gt;
+      &lt;SubFile value="thor1::key::people::contacts"/&gt;
+    &lt;/SuperFile&gt;
+  &lt;/Package&gt;
+  &lt;Package id="person_query" 2016HondaAccordLX-23041.xhtml preload="1"&gt;
+    &lt;Base id="my_base"/&gt;
+  &lt;/Package&gt;
+  &lt;Package id="contacts_query" preload="1"&gt;
+    &lt;Base id="another_base"/&gt;
+  &lt;/Package&gt;
+&lt;/RoxiePackages&gt;
+</programlisting><parameter>To add the packagmap to the target and activate
+    it, you would use:</parameter></para>
+
+    <para><programlisting>ecl packagemap add &lt;target&gt; master.pkg –activate</programlisting></para>
+
+    <para>To update the mappings, you would edit the packagemap file (or a
+    copy) add and activate the new one, and then delete the old one.</para>
+
+    <para>Using <emphasis role="bold">parts</emphasis>, package management can
+    be performed in a more granular fashion. You can break the mappings up
+    into individual part files.</para>
+
+    <para><emphasis role="bold">contacts.pkg</emphasis></para>
+
+    <para><programlisting>&lt;RoxiePackages&gt;
+  &lt;Package id="contact_base" preload="1"&gt;
+    &lt;SuperFile id="thor1::people::key::contacts_superkey"&gt;
+      &lt;SubFile value="thor1::key::people::contacts_file_today"/&gt;
+    &lt;/SuperFile&gt;
+  &lt;/Package&gt;
+  &lt;Package id="contact_query" preload="1"&gt;
+    &lt;Base id="contact_base"/&gt;
+  &lt;/Package&gt;
+  &lt;Package id="contact_references_query" preload="1"&gt;
+    &lt;Base id="contact_base"/&gt;
+  &lt;/Package&gt;
+&lt;/RoxiePackages&gt;
+</programlisting></para>
+
+    <para><emphasis role="bold">persons.pkg</emphasis></para>
+
+    <para><programlisting>&lt;RoxiePackages&gt;
+  &lt;Package id="person_base" preload="1"&gt;
+    &lt;SuperFile id="thor1::people::key::person_superkey"&gt;
+      &lt;SubFile value="thor1::key::people::person_file_today"/&gt;
+    &lt;/SuperFile&gt;
+  &lt;/Package&gt;
+  &lt;Package id="person_query" preload="1"&gt;
+    &lt;Base id="person_base"/&gt;
+  &lt;/Package&gt;
+&lt;/RoxiePackages&gt;
+</programlisting></para>
+
+    <para>This allows you to add them separately:</para>
+
+    <programlisting>ecl packagemap add-part &lt;target&gt; target_pmid contacts.pkg
+ecl packagemap add-part &lt;target&gt; target_pmid persons.pkg
+</programlisting>
+
+    <para>And then activate the entire packagemap (which consists of all parts
+    added to the given target_pmid):</para>
+
+    <programlisting>ecl packagemap activate &lt;target&gt; target_pmid</programlisting>
+
+    <para>You can then change individual parts, and update them in the
+    packagemap:</para>
+
+    <programlisting>ecl packagemap add-part &lt;target&gt; &lt;target_pmid&gt; persons.pkg --delete-previous</programlisting>
+
+    <para>Notice that target_pmid is the identifier for the collection of all
+    the packagmap parts. That identifier can be used to activate or delete the
+    entire collection (packagemap).</para>
+
+    <para>You can use "get-part" to see what the named part looks like in Dali
+    for the given identifier:</para>
+
+    <programlisting>ecl packagemap get-part &lt;target&gt; &lt;packagemap&gt; contents.pkg</programlisting>
+
+    <para>You could potentially retrieve the part that way, edit it, and
+    replace it as shown above. You can use <emphasis
+    role="bold">remove-part</emphasis> to remove a part explicitly from a
+    packagemap.</para>
+
+    <para>You can still treat the entire packagemap as a single file, although
+    its organization has changed a bit to reflect its parts:</para>
+
+    <para><emphasis role="bold">multipart.pkg</emphasis></para>
+
+    <para><programlisting>&lt;PackageMaps multipart="1"&gt;
+  &lt;Part id="contacts.pkg"&gt;
+    &lt;Package id="contact_base" preload="1"&gt;
+      &lt;SuperFile id="thor1::people::key::contacts_superkey"&gt;
+        &lt;SubFile value="thor1::key::people::contacts_file_today"/&gt;
+      &lt;/SuperFile&gt;
+    &lt;/Package&gt;
+    &lt;Package id="contact_query" preload="1"&gt;
+      &lt;Base id="contact_base"/&gt;
+    &lt;/Package&gt;
+    &lt;Package id="contact_references_query" preload="1"&gt;
+      &lt;Base id="contact_base"/&gt;
+    &lt;/Package&gt;
+  &lt;/Part&gt;
+  &lt;Part id="persons.pkg"&gt;
+    &lt;Package id="person_base" preload="1"&gt;
+      &lt;SuperFile id="thor1::people::key::person_superkey"&gt;
+        &lt;SubFile value="thor1::key::people::person_file_today"/&gt;
+      &lt;/SuperFile&gt;
+    &lt;/Package&gt;
+    &lt;Package id="person_query" preload="1"&gt;
+      &lt;Base id="person_base"/&gt;
+    &lt;/Package&gt;
+  &lt;/Part&gt;
+&lt;/PackageMaps&gt;
+</programlisting></para>
+
+    <para>Notice how the content is now divided into &lt;Part/&gt; parts. But
+    you can still use</para>
+
+    <para><programlisting>ecl packagemap add &lt;target&gt; master.pkg</programlisting>on
+    that multipart packagemap and then continue by managing its individual
+    parts. In this case:</para>
+
+    <programlisting>ecl packagemap add &lt;target&gt; multipart.pkg
+ecl packagemap add-part &lt;target&gt; multipart.pkg addresses.pkg 
+ecl packagemap activate &lt;target&gt; multipart.pkg
+</programlisting>
+
+    <para>This would be useful, for example, when copying an entire packagemap
+    from one target to another.</para>
+  </sect1>
+
+  <sect1>
+    <title>Working with Packages using the ecl command line tool</title>
 
     <para>This section contains details about the actions and options used to
     work with packages. The ECL command line tool is fully documented in the
     Client Tools Manual.</para>
 
-    <para></para>
-
-    <para></para>
-
     <xi:include href="../../HPCCClientTools/CT_Mods/CT_ECL_CLI.xml"
                 xpointer="eclpackageadd"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
@@ -339,6 +485,18 @@
                 xmlns:xi="http://www.w3.org/2001/XInclude" />
 
     <xi:include href="../../HPCCClientTools/CT_Mods/CT_ECL_CLI.xml"
+                xpointer="eclpackagemapaddpart"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+    <xi:include href="../../HPCCClientTools/CT_Mods/CT_ECL_CLI.xml"
+                xpointer="eclpackagemapgetpart"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+    <xi:include href="../../HPCCClientTools/CT_Mods/CT_ECL_CLI.xml"
+                xpointer="eclpackagemapremovepart"
+                xmlns:xi="http://www.w3.org/2001/XInclude" />
+
+    <xi:include href="../../HPCCClientTools/CT_Mods/CT_ECL_CLI.xml"
                 xpointer="eclpackagevalidate"
                 xmlns:xi="http://www.w3.org/2001/XInclude" />