Merge remote-tracking branch 'origin/candidate-9.8.x' into candidate-…

…9.10.x

dockerfiles/vcpkg/platform-core-ubuntu-22.04.dockerfile

@@ -62,7 +62,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/b
     chmod +x ./kubectl && \
     mv ./kubectl /usr/local/bin
 
-RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.0_amd64.deb/download && \
+RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.1_amd64.deb/download && \
     dpkg -i download && \
     rm download
 

dockerfiles/vcpkg/platform-core-ubuntu-22.04/Dockerfile

@@ -62,7 +62,7 @@ RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.29.7/b
     chmod +x ./kubectl && \
     mv ./kubectl /usr/local/bin
 
-RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.0_amd64.deb/download && \
+RUN curl -LO https://packagecloud.io/github/git-lfs/packages/ubuntu/jammy/git-lfs_3.6.1_amd64.deb/download && \
     dpkg -i download && \
     rm download
 

docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml

@@ -509,7 +509,7 @@
 
               <entry>components</entry>
 
-              <entry></entry>
+              <entry/>
             </row>
           </tbody>
         </tgroup>
@@ -1332,9 +1332,9 @@ thor:
 
         <informaltable colsep="1" frame="all" rowsep="1">
           <tgroup cols="2">
-            <colspec colwidth="125.55pt" />
+            <colspec colwidth="125.55pt"/>
 
-            <colspec />
+            <colspec/>
 
             <tbody>
               <row>

docs/EN_US/ContainerizedHPCC/ContainerizedMods/ContainerLogging.xml

@@ -70,8 +70,39 @@
       choose to integrate HPCC Systems logging data with any existing logging
       solutions, or to implement another one specifically for HPCC Systems
       data. Starting with HPCC Systems version 8.4, we provide a lightweight,
-      yet complete log-processing solution for your convenience. The following
-      sections will look at a couple of the possible solutions.</para>
+      yet complete log-processing solution for convenience. Subsequent
+      sections will look at a couple other possible log-processing
+      solutions.</para>
+
+      <sect3 id="LogP_Depends">
+        <title>Log Dependant Applications</title>
+
+        <para>Currently there is a utility delivered with a containerized HPCC
+        Systems deployment which is dependant on having a properly configured
+        log-processing solution for optimal results.</para>
+
+        <sect4 id="CNTR_ZAP">
+          <title>The Z.A.P. Utility</title>
+
+          <para>The Zipped Analysis Package (Z.A.P.) utility collects system
+          information and encapsulates it into a shareable package. This
+          utility packages up information to send for further analysis. ZAP
+          reports contain several artifacts related to a given workunit, to
+          aid in debugging.</para>
+
+          <para>The Component logs are one of most important artifacts
+          expected to be included in the report. In containerized deployments
+          logging is handled differently from bare metal. The log fetching is
+          dependent on a back-end log processor being properly configured and
+          available and the HPCC LogAccess feature configured to bind to the
+          log processor. If those two dependencies are not met, the
+          containerized cluster logs are not included in the ZAP report. Those
+          ZAP reports will then be incomplete. To ensure inclusion of the logs
+          in the ZAP report you must have log access configured properly. See
+          the <link linkend="Containerized_Logging">Containerized
+          Logging</link> sections for more information.</para>
+        </sect4>
+      </sect3>
     </sect2>
   </sect1>
 

docs/EN_US/HPCCSystemAdmin/HPCCSystemAdministratorsGuide.xml

@@ -721,6 +721,146 @@
           workunits in the event of a loss. In addition it would affect every
           other Thor/Roxie cluster in the same environment if you lose this
           node.</para>
+
+          <sect4 id="DaliRestore">
+            <title>Restoring Dali from backup</title>
+
+            <para>If configured correctly, Dali creates a backup or mirror
+            copy to a secondary location on another physical server.
+            (Bare-metal only).</para>
+
+            <para>Systems can be configured with their own scheduled backup to
+            create a snapshot of the primary store files to a custom location.
+            The same steps apply when using a snapshot copy of a backup set as
+            when using the mirror copy. In other words, this technique applies
+            to either bare-metal or k8s deployments.</para>
+
+            <para>The Dali meta files are comprised of:</para>
+
+            <orderedlist>
+              <listitem>
+                <para><emphasis role="bold">store.&lt;NNNN&gt;</emphasis>
+                (e.g., store.36). This file is a reference to the current Dali
+                meta file edition. There should never be more than one of
+                these files. The NNNN is used to determine the current base
+                and delta files in use.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">dalisds&lt;NNNN&gt;.xml</emphasis>
+                (e.g., dalisds36.xml). This is the main Dali meta info file,
+                containing all logical file, workunit, and state information.
+                Sasha (or Dali on save) periodically creates new versions
+                (with incrementally rising NNNN’s). It will keep the last T
+                copies (default 10) based on the configuration option
+                “keepStores”.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">daliinc&lt;NNNN&gt;.xml</emphasis>
+                (e.g., daliinc36.xml). This is the delta transaction log. Dali
+                continuously writes to this file, recording all changes that
+                are made to any meta data. It is used to playback changes and
+                apply them to the base meta info from the
+                dalisds&lt;NNNN&gt;xml file.</para>
+
+                <para>Specifically, when Sasha creates a new store version, it
+                loads the base file (e.g., dalisds36.xml), then loads and
+                applies the delta file (e.g., daliinc36.xml). Sasha then has
+                its own independent representation of the current state and
+                saves a new base file (e.g., dalisds(NNNN+1).xml).</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis role="bold">dalidet&lt;NNNN&gt;.xml</emphasis>
+                (e.g., dalidet36.xml). This file is created at the point that
+                Sasha starts the process of creating a new base file. At which
+                point it atomically renames the delta transaction file to a
+                ‘det’ file (short for 'detached'). For example, it renames
+                daliinc36.xml to dalidet36.xml. Dali then continues to write
+                new transactions to daliinc36.xml.</para>
+              </listitem>
+
+              <listitem>
+                <para><emphasis
+                role="bold">dalisds_&lt;MMMM&gt;.bv2</emphasis> files. These
+                files are in effect part of the main store (part of
+                dalisdsNNNN.xml). They are single large values that were
+                deemed too big to keep in Dali memory, and written to disk
+                separately instead (and are loaded on demand).</para>
+              </listitem>
+            </orderedlist>
+
+            <para>If Dali is shutdown cleanly and saves its files as expected,
+            the daliinc*.xml and dalidet*.xml files are not needed, since it
+            saves the entire state of the store directly from internal memory,
+            and on startup, there is no daliincNNNN.xml or dalidetNNNN.xml
+            related to the new version.</para>
+
+            <para>These transaction delta files are only used by Sasha when
+            creating new versions of the base store or if Dali has been
+            stopped abruptly (e.g., machine rebooted). If Dali restarts after
+            an unclean exit, there will be a daliincNNN.xml (and possibly a
+            dalidetNNNN.xml file if Sasha was actively creating a new version
+            at the time). In those cases, Dali will load these files in
+            addition to the base file.</para>
+
+            <para>By default Dali’s main data store directory is
+            /var/lib/HPCCSystems/hpcc-data/dali/ . In other words, all meta
+            data is written to and read from this location.</para>
+
+            <para>When restoring from a backup: <orderedlist>
+                <listitem>
+                  <para>Make sure Dali is not running</para>
+                </listitem>
+
+                <listitem>
+                  <para>Make sure the /var/lib/HPCCSystems/hpcc-data/dali
+                  folder is empty.</para>
+                </listitem>
+
+                <listitem>
+                  <para>Copy all pertinent backup file into the
+                  /var/lib/HPCCSystems/hpcc-data/dali folder:</para>
+
+                  <itemizedlist>
+                    <listitem>
+                      <para>One store.NNNN file</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>One dalisdsNNNN.xml file</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>&lt;=1 daliincNNNN.xml file (only if
+                      present)</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>&lt;=1 dalidetNNNN.xml file (only if
+                      present)</para>
+                    </listitem>
+
+                    <listitem>
+                      <para>All dalisds_MMMM.bv2 files.</para>
+                    </listitem>
+                  </itemizedlist>
+                </listitem>
+              </orderedlist>Other/older dalisds/daliinc/dalidet editions could
+            be copied, but the above are the only ones that will be used. In
+            other words, only the NNNN version based on the single store.NNNN
+            file will be loaded.</para>
+
+            <para>The automatic back to a mirror location is bare-metal only.
+            In a cloud deployment, it is assumed that the storage choices
+            provided by the cloud provider are providing redundancy, such as
+            multi-zone replication.</para>
+
+            <para>In either case, and/or if a manual strategy has been used to
+            copy Dali’s files on a schedule, the process of restoring from a
+            backup should be the same.</para>
+          </sect4>
         </sect3>
 
         <sect3 id="SysAdm_BkUp_Sasha">

esp/services/ws_workunits/ws_workunitsHelpers.cpp

@@ -4544,7 +4544,8 @@ void CWsWuFileHelper::createWUZAPFile(IEspContext& context, IConstWorkUnit* cwu,
     if (request.includeThorSlaveLog.isEmpty() || strieq(request.includeThorSlaveLog.str(), "on"))
         createThorSlaveLogfile(cwu, winfo, tempDirName);
 #else
-    readWULogToFiles(cwu, winfo, tempDirName, request);
+    if (request.includeRelatedLogs || request.includePerComponentLogs)
+        readWULogToFiles(cwu, winfo, tempDirName, request);
 #endif
 
     //Write out to ZIP file

esp/services/ws_workunits/ws_workunitsService.cpp

@@ -1718,6 +1718,7 @@ void doWUQueryBySingleWU(IEspContext &context, IConstWorkUnit *cw, IEspWUQueryRe
     resp.setWorkunits(results);
     resp.setPageSize(1);
     resp.setCount(1);
+    resp.setNumWUs(1);
     PROGLOG("getWUInfo: %s", cw->queryWuid());
 }
 
@@ -1754,6 +1755,7 @@ void doWUQueryByFile(IEspContext &context, const char *logicalFile, IEspWUQueryR
     resp.setFirst(false);
     resp.setPageSize(1);
     resp.setCount(1);
+    resp.setNumWUs(1);
 }
 
 bool addWUQueryFilter(WUSortField *filters, unsigned short &count, MemoryBuffer &buff, const char *name, WUSortField value)
@@ -2602,6 +2604,8 @@ bool CWsWorkunitsEx::onWUQuery(IEspContext &context, IEspWUQueryRequest & req, I
     {
         WuidPattern pattern(req.getWuid());
 
+        resp.setNumWUs(0);
+
         if (req.getType() && strieq(req.getType(), "archived workunits"))
             doWUQueryFromArchive(context, sashaServerIp.get(), sashaServerPort, *archivedWuCache, awusCacheMinutes, req, resp);
         else if(notEmpty(pattern) && looksLikeAWuid(pattern, 'W'))

esp/src/src-react/components/controls/Grid.tsx

@@ -1,5 +1,5 @@
 import * as React from "react";
-import { DetailsList, DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IRenderFunction, IDetailsRowProps, SelectionMode, ConstrainMode, ISelection, ScrollablePane, Sticky } from "@fluentui/react";
+import { DetailsListLayoutMode, Dropdown, IColumn as _IColumn, ICommandBarItemProps, IDetailsHeaderProps, IDetailsListStyles, mergeStyleSets, Selection, Stack, TooltipHost, TooltipOverflowMode, IRenderFunction, IDetailsRowProps, SelectionMode, ConstrainMode, ISelection, ScrollablePane, ShimmeredDetailsList, Sticky } from "@fluentui/react";
 import { Pagination } from "@fluentui/react-experiments/lib/Pagination";
 import { useConst } from "@fluentui/react-hooks";
 import { BaseStore, Memory, QueryRequest, QuerySortItem } from "src/store/Memory";
@@ -227,6 +227,7 @@ const FluentStoreGrid: React.FunctionComponent<FluentStoreGridProps> = ({
     const memoizedColumns = useDeepMemo(() => columns, [], [columns]);
     const [sorted, setSorted] = React.useState<QuerySortItem>(sort);
     const [items, setItems] = React.useState<any[]>([]);
+    const [loaded, setLoaded] = React.useState(false);
     const [columnWidths] = useNonReactiveEphemeralPageStore("columnWidths");
 
     const selectionHandler = useConst(() => {
@@ -248,6 +249,7 @@ const FluentStoreGrid: React.FunctionComponent<FluentStoreGridProps> = ({
 
     const refreshTable = useDeepCallback((clearSelection = false) => {
         if (isNaN(start) || isNaN(count)) return;
+        setLoaded(false);
         if (clearSelection) {
             selectionHandler.setItems([], true);
         }
@@ -257,6 +259,7 @@ const FluentStoreGrid: React.FunctionComponent<FluentStoreGridProps> = ({
         });
         storeQuery.then(items => {
             const selectedIndices = selectionHandler.getSelectedIndices();
+            setLoaded(true);
             setItems(items);
             selectedIndices.forEach(index => selectionHandler.setIndexSelected(index, true, false));
         });
@@ -319,8 +322,9 @@ const FluentStoreGrid: React.FunctionComponent<FluentStoreGridProps> = ({
 
     return <div style={{ position: "relative", height: "100%" }}>
         <ScrollablePane>
-            <DetailsList
+            <ShimmeredDetailsList
                 compact={true}
+                enableShimmer={!loaded}
                 items={items}
                 columns={fluentColumns}
                 layoutMode={DetailsListLayoutMode.fixedColumns}

fs/dafsclient/rmtfile.cpp

@@ -1421,6 +1421,9 @@ class CRemoteFileIO : public CInterfaceOf<IFileIO>
 
     size32_t read(offset_t pos, size32_t len, void * data)
     {
+        if (0 == len)
+            return 0;
+        dbgassertex(data);
         size32_t got;
         MemoryBuffer replyBuffer;
         CCycleTimer timer;

roxie/ccd/ccdlistener.cpp

@@ -1566,12 +1566,8 @@ class RoxieProtocolMsgContext : implements IHpccProtocolMsgContext, public CInte
 
         ensureContextLogger();
 
-        const char * spanQueryName = !isEmptyString(queryName) ? queryName : "run_query";
-        StringBuffer spanName(querySetName);
-        if (spanName.length())
-            spanName.append('/');
-        spanName.append(spanQueryName);
-        requestSpan.setown(queryTraceManager().createServerSpan(spanName, allHeaders, flags));
+        requestSpan.setown(queryTraceManager().createServerSpan(!isEmptyString(queryName) ? queryName : "run_query", allHeaders, flags));
+        requestSpan->setSpanAttribute("queryset.name", querySetName);
         logctx->setActiveSpan(requestSpan);
 
         const char * globalId = requestSpan->queryGlobalId();

system/jlib/jlog.hpp

@@ -608,7 +608,7 @@ class jlib_decl LogMsgTraceInfo
 
     const char * queryTraceID() const
     {
-        if (span)
+        if (span && span->isValid())
         {
             const char * traceId = span->queryTraceId();
             if (traceId)
@@ -619,7 +619,7 @@ class jlib_decl LogMsgTraceInfo
 
     const char * querySpanID() const
     {
-        if (span)
+        if (span && span->isValid())
         {
             const char * spanId = span->querySpanId();
             if (spanId)

system/jlib/jtrace.cpp

@@ -770,6 +770,15 @@ class CSpan : public CInterfaceOf<ISpan>
         return span ? span->IsRecording() : false;
     }
 
+    virtual bool isValid() const override
+    {
+        if (span == nullptr)
+            return false;
+
+        auto spanCtx = span->GetContext();
+        return spanCtx.IsValid();
+    }
+
     virtual void setSpanStatusSuccess(bool spanSucceeded, const char * statusMessage)
     {
         if (span != nullptr)
@@ -923,6 +932,7 @@ class CNullSpan final : public CInterfaceOf<ISpan>
     virtual void toString(StringBuffer & out) const override {}
     virtual void getLogPrefix(StringBuffer & out) const override {}
     virtual bool isRecording() const { return false; }
+    virtual bool isValid() const { return false; }
 
     virtual void recordException(IException * e, bool spanFailed, bool escapedScope) override {}
     virtual void recordError(const SpanError & error) override {};

system/jlib/jtrace.hpp

@@ -133,6 +133,7 @@ struct SpanTimeStamp
 
 interface ISpan : extends IInterface
 {
+    virtual bool isValid() const = 0;
     virtual void setSpanAttribute(const char * key, const char * val) = 0;
     virtual void setSpanAttribute(const char *name, __uint64 value) = 0;
     virtual void setSpanAttributes(const IProperties * attributes) = 0;

testing/regress/ecl/despray.ecl

@@ -30,7 +30,7 @@ import $.setup;
 import Std.File AS FileServices;
 
 
-dropzonePathTemp := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+dropzonePathTemp := FileServices.GetDefaultDropZone() : STORED('dropzonePath');
 dropzonePath := dropzonePathTemp + IF(dropzonePathTemp[LENGTH(dropzonePathTemp)]='/', '', '/');
 prefix := setup.Files(false, false).QueryFilePrefix;
 

testing/regress/ecl/spray_dir_test.ecl

@@ -39,7 +39,7 @@ import ^ as root;
 engine := thorlib.platform();
 prefix := '~regress::' + engine + '::' + WORKUNIT + '::';
 
-dropzonePathTemp := '/var/lib/HPCCSystems/mydropzone/' : STORED('dropzonePath');
+dropzonePathTemp := FileServices.GetDefaultDropZone() : STORED('dropzonePath');
 dropzonePath := dropzonePathTemp + IF(dropzonePathTemp[LENGTH(dropzonePathTemp)]='/', '', '/');
 
 unsigned VERBOSE := 0;