Merge remote-tracking branch 'origin/candidate-9.10.x'

Signed-off-by: Jake Smith <jake.smith@lexisnexisrisk.com>
hpcc-systems · Jan 23, 2025 · 5757172 · 5757172
2 parents 44a639d + c31e6cc
commit 5757172
Show file tree

Hide file tree

Showing 68 changed files with 727 additions and 259 deletions.
diff --git a/.github/workflows/build-assets.yml b/.github/workflows/build-assets.yml
@@ -87,6 +87,10 @@ jobs:
     strategy:
       matrix:
         include:
+          - os: ubuntu-24.04
+          - os: ubuntu-24.04
+            name: k8s
+            container: true
           - os: ubuntu-22.04
           - os: ubuntu-22.04
             name: k8s
@@ -100,6 +104,10 @@ jobs:
             name: k8s
             container: true
           - os: rockylinux-8
+          - os: ubuntu-24.04
+            name: LN k8s
+            ln: true
+            container: true
           - os: ubuntu-22.04
             name: LN k8s
             ln: true
@@ -109,9 +117,15 @@ jobs:
             name: LN k8s
             ln: true
             container: true
+          - os: ubuntu-24.04
+            name: LN
+            ln: true
           - os: ubuntu-22.04
             name: LN
             ln: true
+          - os: ubuntu-24.04
+            name: Enterprise
+            ee: true
           - os: ubuntu-22.04
             name: Enterprise
             ee: true

diff --git a/cmake_modules/vcpkg.cmake b/cmake_modules/vcpkg.cmake
@@ -6,7 +6,6 @@ set(CMAKE_TOOLCHAIN_FILE ${HPCC_SOURCE_DIR}/vcpkg/scripts/buildsystems/vcpkg.cma
 set(VCPKG_ROOT ${HPCC_SOURCE_DIR}/vcpkg)
 set(VCPKG_INSTALLED_DIR "${VCPKG_FILES_DIR}/vcpkg_installed")
 set(VCPKG_INSTALL_OPTIONS "--x-abi-tools-use-exact-versions;--downloads-root=${VCPKG_FILES_DIR}/vcpkg_downloads;--x-buildtrees-root=${VCPKG_FILES_DIR}/vcpkg_buildtrees;--x-packages-root=${VCPKG_FILES_DIR}/vcpkg_packages")
-set(VCPKG_VERBOSE OFF)
 
 if(WIN32)
     set(VCPKG_HOST_TRIPLET "x64-windows" CACHE STRING "host triplet")
@@ -24,6 +23,15 @@ elseif(UNIX)
     set(VCPKG_TARGET_TRIPLET "x64-linux-dynamic" CACHE STRING "target triplet")
 endif()
 
+message("-- vcpkg settings:")
+message("---- VCPKG_FILES_DIR: ${VCPKG_FILES_DIR}")
+message("---- CMAKE_TOOLCHAIN_FILE: ${CMAKE_TOOLCHAIN_FILE}")
+message("---- VCPKG_ROOT: ${VCPKG_ROOT}")
+message("---- VCPKG_INSTALLED_DIR: ${VCPKG_INSTALLED_DIR}")
+message("---- VCPKG_INSTALL_OPTIONS: ${VCPKG_INSTALL_OPTIONS}")
+message("---- VCPKG_HOST_TRIPLET: ${VCPKG_HOST_TRIPLET}")
+message("---- VCPKG_TARGET_TRIPLET: ${VCPKG_TARGET_TRIPLET}")
+
 #  Create a catalog of the vcpkg dependencies ---
 file(GLOB VCPKG_PACKAGES ${VCPKG_FILES_DIR}/vcpkg_packages/*/CONTROL)
 list(APPEND VCPKG_PACKAGE_LIST "-----------------\n")

diff --git a/common/thorhelper/thorcommon.cpp b/common/thorhelper/thorcommon.cpp
@@ -1801,8 +1801,6 @@ void ActivityTimeAccumulator::addStatistics(IStatisticGatherer & builder) const
         if (blockedCycles)
             builder.addStatistic(StTimeBlocked, cycle_to_nanosec(blockedCycles));
     }
-    if (lookAheadCycles)
-        builder.addStatistic(StTimeLookAhead, (unsigned __int64)cycle_to_nanosec(lookAheadCycles));
 }
 
 void ActivityTimeAccumulator::addStatistics(CRuntimeStatisticCollection & merged) const
@@ -1816,8 +1814,6 @@ void ActivityTimeAccumulator::addStatistics(CRuntimeStatisticCollection & merged
         if (blockedCycles)
             merged.mergeStatistic(StTimeBlocked, cycle_to_nanosec(blockedCycles));
     }
-    if (lookAheadCycles)
-        merged.mergeStatistic(StTimeLookAhead, (unsigned __int64)cycle_to_nanosec(lookAheadCycles));
 }
 
 void ActivityTimeAccumulator::merge(const ActivityTimeAccumulator & other)

diff --git a/common/workunit/workunit.cpp b/common/workunit/workunit.cpp
@@ -14985,3 +14985,12 @@ void recordTraceDebugOptions(IWorkUnit * target, const IProperties * source)
         }
     }
 }
+
+WuidPattern::WuidPattern(const char* _pattern)
+    : pattern(_pattern)
+{
+    if (!pattern.isEmpty())
+        pattern.trim();
+    if (!pattern.isEmpty() && islower(pattern.charAt(0)))
+        pattern.setCharAt(0, toupper(pattern.charAt(0)));
+}
diff --git a/common/workunit/workunit.hpp b/common/workunit/workunit.hpp
@@ -1843,4 +1843,17 @@ class WORKUNIT_API StatisticsAggregator : public CInterface
     const StatisticsMapping & mapping;
 };
 
+class WORKUNIT_API WuidPattern
+{
+private:
+    StringBuffer pattern;
+public:
+    WuidPattern(const char* _pattern);
+    inline bool isEmpty() const { return pattern.isEmpty(); }
+    inline const char* str() const { return pattern.str(); }
+    inline operator const char* () const { return pattern.str(); }
+    inline operator const StringBuffer& () const { return pattern; }
+    inline operator StringBuffer& () { return pattern; }
+};
+
 #endif
diff --git a/dali/base/dautils.cpp b/dali/base/dautils.cpp
@@ -3675,6 +3675,11 @@ static CConfigUpdateHook directIOUpdateHook;
 static CriticalSection dafileSrvNodeCS;
 static Owned<INode> tlsDirectIONode, nonTlsDirectIONode;
 
+unsigned getPreferredDaFsServerPort()
+{
+    return getPreferredDafsClientPort(true);
+}
+
 void remapGroupsToDafilesrv(IPropertyTree *file, bool foreign, bool secure)
 {
     Owned<IPropertyTreeIterator> iter = file->getElements("Cluster");
@@ -3683,7 +3688,7 @@ void remapGroupsToDafilesrv(IPropertyTree *file, bool foreign, bool secure)
         IPropertyTree &cluster = iter->query();
         const char *planeName = cluster.queryProp("@name");
         Owned<IStoragePlane> plane = getDataStoragePlane(planeName, true);
-        if ((0 == plane->queryHosts().size()) && isAbsolutePath(plane->queryPrefix())) // if hosts group, or url, don't touch
+        if (isAbsolutePath(plane->queryPrefix())) // if url (i.e. not absolute prefix path) don't touch
         {
             if (isContainerized())
             {

diff --git a/dali/base/dautils.hpp b/dali/base/dautils.hpp
@@ -583,7 +583,7 @@ inline unsigned calcStripeNumber(unsigned partNum, const char *lfnName, unsigned
 }
 interface INamedGroupStore;
 extern da_decl void remapGroupsToDafilesrv(IPropertyTree *file, bool foreign, bool secure);
-
+extern da_decl unsigned getPreferredDaFsServerPort();
 #ifdef NULL_DALIUSER_STACKTRACE
 extern da_decl void logNullUser(IUserDescriptor *userDesc);
 #else

diff --git a/dali/sasha/saarch.cpp b/dali/sasha/saarch.cpp
@@ -1166,7 +1166,7 @@ class CDFUWorkUnitArchiver: public CBranchArchiver
             getWorkUnitCreateTime(wuid,time);
         }
         virtual ~cDFUWUBranchItem() {}
-        bool isempty() { return (wuid[0]!='D')||iserr; }
+        bool isempty() { return (toupper(wuid[0])!='D')||iserr; }
         bool qualifies() 
         { 
             if (isprotected)

diff --git a/dockerfiles/vcpkg/centos-7.dockerfile b/dockerfiles/vcpkg/centos-7.dockerfile
@@ -7,4 +7,8 @@ RUN yum install -y \
 
 ENTRYPOINT ["/bin/bash", "--login", "-c"]
 
+RUN yum install -y \
+    rpm-build && \
+    yum -y clean all && rm -rf /var/cache 
+
 CMD ["/bin/bash"]
diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile
@@ -62,7 +62,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/b
     chmod +x ./kubectl && \
     mv ./kubectl /usr/local/bin
 
-RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.0_amd64.deb/download && \
+RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.1_amd64.deb/download && \
     dpkg -i download && \
     rm download
 

diff --git a/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile b/dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile
@@ -62,7 +62,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/b
     chmod +x ./kubectl && \
     mv ./kubectl /usr/local/bin
 
-RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.0_amd64.deb/download && \
+RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.1_amd64.deb/download && \
     dpkg -i download && \
     rm download
 

diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml
@@ -509,7 +509,7 @@
 
               <entry>components</entry>
 
-              <entry></entry>
+              <entry/>
             </row>
           </tbody>
         </tgroup>
@@ -1332,9 +1332,9 @@ thor:
 
         <informaltable colsep="1" frame="all" rowsep="1">
           <tgroup cols="2">
-            <colspec colwidth="125.55pt" />
+            <colspec colwidth="125.55pt"/>
 
-            <colspec />
+            <colspec/>
 
             <tbody>
               <row>

diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml
@@ -70,8 +70,39 @@
       choose to integrate HPCC Systems logging data with any existing logging
       solutions, or to implement another one specifically for HPCC Systems
       data. Starting with HPCC Systems version 8.4, we provide a lightweight,
-      yet complete log-processing solution for your convenience. The following
-      sections will look at a couple of the possible solutions.</para>
+      yet complete log-processing solution for convenience. Subsequent
+      sections will look at a couple other possible log-processing
+      solutions.</para>
+
+      <sect3 id="LogP_Depends">
+        <title>Log Dependant Applications</title>
+
+        <para>Currently there is a utility delivered with a containerized HPCC
+        Systems deployment which is dependant on having a properly configured
+        log-processing solution for optimal results.</para>
+
+        <sect4 id="CNTR_ZAP">
+          <title>The Z.A.P. Utility</title>
+
+          <para>The Zipped Analysis Package (Z.A.P.) utility collects system
+          information and encapsulates it into a shareable package. This
+          utility packages up information to send for further analysis. ZAP
+          reports contain several artifacts related to a given workunit, to
+          aid in debugging.</para>
+
+          <para>The Component logs are one of most important artifacts
+          expected to be included in the report. In containerized deployments
+          logging is handled differently from bare metal. The log fetching is
+          dependent on a back-end log processor being properly configured and
+          available and the HPCC LogAccess feature configured to bind to the
+          log processor. If those two dependencies are not met, the
+          containerized cluster logs are not included in the ZAP report. Those
+          ZAP reports will then be incomplete. To ensure inclusion of the logs
+          in the ZAP report you must have log access configured properly. See
+          the <link linkend="Containerized_Logging">Containerized
+          Logging</link> sections for more information.</para>
+        </sect4>
+      </sect3>
     </sect2>
   </sect1>
 

diff --git a/docs/EN_US/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml b/docs/EN_US/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml
@@ -721,6 +721,146 @@
           workunits in the event of a loss. In addition it would affect every
           other Thor/Roxie cluster in the same environment if you lose this
           node.</para>
+
+          <sect4 id="DaliRestore">
+            <title>Restoring Dali from backup</title>
+
+            <para>If configured correctly, Dali creates a backup or mirror
+            copy to a secondary location on another physical server.
+            (Bare-metal only).</para>
+
+            <para>Systems can be configured with their own scheduled backup to
+            create a snapshot of the primary store files to a custom location.
+            The same steps apply when using a snapshot copy of a backup set as
+            when using the mirror copy. In other words, this technique applies
+            to either bare-metal or k8s deployments.</para>
+
+            <para>The Dali meta files are comprised of:</para>
+
+            <orderedlist>
+              <listitem>
+                <para><emphasis role="bold">store.&lt;NNNN&gt;</emphasis>
+                (e.g., store.36). This file is a reference to the current Dali
+                meta file edition. There should never be more than one of
+                these files. The NNNN is used to determine the current base
+                and delta files in use.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">dalisds&lt;NNNN&gt;.xml</emphasis>
+                (e.g., dalisds36.xml). This is the main Dali meta info file,
+                containing all logical file, workunit, and state information.
+                Sasha (or Dali on save) periodically creates new versions
+                (with incrementally rising NNNN’s). It will keep the last T
+                copies (default 10) based on the configuration option
+                “keepStores”.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">daliinc&lt;NNNN&gt;.xml</emphasis>
+                (e.g., daliinc36.xml). This is the delta transaction log. Dali
+                continuously writes to this file, recording all changes that
+                are made to any meta data. It is used to playback changes and
+                apply them to the base meta info from the
+                dalisds&lt;NNNN&gt;xml file.</para>
+
+                <para>Specifically, when Sasha creates a new store version, it
+                loads the base file (e.g., dalisds36.xml), then loads and
+                applies the delta file (e.g., daliinc36.xml). Sasha then has
+                its own independent representation of the current state and
+                saves a new base file (e.g., dalisds(NNNN+1).xml).</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">dalidet&lt;NNNN&gt;.xml</emphasis>
+                (e.g., dalidet36.xml). This file is created at the point that
+                Sasha starts the process of creating a new base file. At which
+                point it atomically renames the delta transaction file to a
+                ‘det’ file (short for 'detached'). For example, it renames
+                daliinc36.xml to dalidet36.xml. Dali then continues to write
+                new transactions to daliinc36.xml.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis
+                role="bold">dalisds_&lt;MMMM&gt;.bv2</emphasis> files. These
+                files are in effect part of the main store (part of
+                dalisdsNNNN.xml). They are single large values that were
+                deemed too big to keep in Dali memory, and written to disk
+                separately instead (and are loaded on demand).</para>
+              </listitem>
+            </orderedlist>
+
+            <para>If Dali is shutdown cleanly and saves its files as expected,
+            the daliinc*.xml and dalidet*.xml files are not needed, since it
+            saves the entire state of the store directly from internal memory,
+            and on startup, there is no daliincNNNN.xml or dalidetNNNN.xml
+            related to the new version.</para>
+
+            <para>These transaction delta files are only used by Sasha when
+            creating new versions of the base store or if Dali has been
+            stopped abruptly (e.g., machine rebooted). If Dali restarts after
+            an unclean exit, there will be a daliincNNN.xml (and possibly a
+            dalidetNNNN.xml file if Sasha was actively creating a new version
+            at the time). In those cases, Dali will load these files in
+            addition to the base file.</para>
+
+            <para>By default Dali’s main data store directory is
+            /var/lib/HPCCSystems/hpcc-data/dali/ . In other words, all meta
+            data is written to and read from this location.</para>
+
+            <para>When restoring from a backup: <orderedlist>
+                <listitem>
+                  <para>Make sure Dali is not running</para>
+                </listitem>
+
+                <listitem>
+                  <para>Make sure the /var/lib/HPCCSystems/hpcc-data/dali
+                  folder is empty.</para>
+                </listitem>
+
+                <listitem>
+                  <para>Copy all pertinent backup file into the
+                  /var/lib/HPCCSystems/hpcc-data/dali folder:</para>
+
+                  <itemizedlist>
+                    <listitem>
+                      <para>One store.NNNN file</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>One dalisdsNNNN.xml file</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>&lt;=1 daliincNNNN.xml file (only if
+                      present)</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>&lt;=1 dalidetNNNN.xml file (only if
+                      present)</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>All dalisds_MMMM.bv2 files.</para>
+                    </listitem>
+                  </itemizedlist>
+                </listitem>
+              </orderedlist>Other/older dalisds/daliinc/dalidet editions could
+            be copied, but the above are the only ones that will be used. In
+            other words, only the NNNN version based on the single store.NNNN
+            file will be loaded.</para>
+
+            <para>The automatic back to a mirror location is bare-metal only.
+            In a cloud deployment, it is assumed that the storage choices
+            provided by the cloud provider are providing redundancy, such as
+            multi-zone replication.</para>
+
+            <para>In either case, and/or if a manual strategy has been used to
+            copy Dali’s files on a schedule, the process of restoring from a
+            backup should be the same.</para>
+          </sect4>
         </sect3>
 
         <sect3 id="SysAdm_BkUp_Sasha">