feat: Perform bundles extraction in stream (#2387)

lib/app-utils.js

@@ -1,11 +1,13 @@
 import _ from 'lodash';
 import path from 'path';
-import {plist, fs, util, tempDir, zip} from 'appium/support';
+import {plist, fs, util, tempDir, zip, timing} from 'appium/support';
 import log from './logger.js';
 import {LRUCache} from 'lru-cache';
 import os from 'node:os';
 import {exec} from 'teen_process';
 import B from 'bluebird';
+import {spawn} from 'node:child_process';
+import assert from 'node:assert';
 
 const STRINGSDICT_RESOURCE = '.stringsdict';
 const STRINGS_RESOURCE = '.strings';
@@ -22,6 +24,9 @@ const SAFARI_OPTS_ALIASES_MAP = /** @type {const} */ ({
   safariIgnoreFraudWarning: [['WarnAboutFraudulentWebsites'], (x) => Number(!x)],
   safariOpenLinksInBackground: [['OpenLinksInBackground'], (x) => Number(Boolean(x))],
 });
+const MAX_ARCHIVE_SCAN_DEPTH = 1;
+export const SUPPORTED_EXTENSIONS = [IPA_EXT, APP_EXT];
+const MACOS_RESOURCE_FOLDER = '__MACOSX';
 
 
 /**
@@ -261,27 +266,110 @@ export async function isAppBundle(appPath) {
 }
 
 /**
- * Extract the given archive and looks for items with given extensions in it
+ * @typedef {Object} UnzipInfo
+ * @property {string} rootDir
+ * @property {number} archiveSize
+ */
+
+/**
+ * Unzips a ZIP archive on the local file system.
  *
  * @param {string} archivePath Full path to a .zip archive
- * @param {Array<string>} appExtensions List of matching item extensions
- * @returns {Promise<[string, string[]]>} Tuple, where the first element points to
- * a temporary folder root where the archive has been extracted and the second item
- * contains a list of relative paths to matched items
+ * @returns {Promise<UnzipInfo>} temporary folder root where the archive has been extracted
  */
-export async function findApps(archivePath, appExtensions) {
+export async function unzipFile(archivePath) {
   const useSystemUnzipEnv = process.env.APPIUM_PREFER_SYSTEM_UNZIP;
   const useSystemUnzip =
     _.isEmpty(useSystemUnzipEnv) || !['0', 'false'].includes(_.toLower(useSystemUnzipEnv));
   const tmpRoot = await tempDir.openDir();
-  await zip.extractAllTo(archivePath, tmpRoot, {useSystemUnzip});
+  try {
+    await zip.extractAllTo(archivePath, tmpRoot, {useSystemUnzip});
+  } catch (e) {
+    await fs.rimraf(tmpRoot);
+    throw e;
+  }
+  return {
+    rootDir: tmpRoot,
+    archiveSize: (await fs.stat(archivePath)).size,
+  };
+}
+
+/**
+ * Unzips a ZIP archive from a stream.
+ * Uses bdstar tool for this purpose.
+ * This allows to optimize the time needed to prepare the app under test
+ * to MAX(download, unzip) instead of SUM(download, unzip)
+ *
+ * @param {import('node:stream').Readable} zipStream
+ * @returns {Promise<UnzipInfo>}
+ */
+export async function unzipStream(zipStream) {
+  const tmpRoot = await tempDir.openDir();
+  const bsdtarProcess = spawn(await fs.which('bsdtar'), [
+    '-x',
+    '--exclude', MACOS_RESOURCE_FOLDER,
+    '--exclude', `${MACOS_RESOURCE_FOLDER}/*`,
+    '-',
+  ], {
+    cwd: tmpRoot,
+  });
+  let archiveSize = 0;
+  bsdtarProcess.stderr.on('data', (chunk) => {
+    const stderr = chunk.toString();
+    if (_.trim(stderr)) {
+      log.warn(stderr);
+    }
+  });
+  zipStream.on('data', (chunk) => {
+    archiveSize += _.size(chunk);
+  });
+  zipStream.pipe(bsdtarProcess.stdin);
+  try {
+    await new B((resolve, reject) => {
+      zipStream.once('error', reject);
+      bsdtarProcess.once('exit', (code, signal) => {
+        zipStream.unpipe(bsdtarProcess.stdin);
+        log.debug(`bsdtar process exited with code ${code}, signal ${signal}`);
+        if (code === 0) {
+          resolve();
+        } else {
+          reject(new Error('Is it a valid ZIP archive?'));
+        }
+      });
+      bsdtarProcess.once('error', (e) => {
+        zipStream.unpipe(bsdtarProcess.stdin);
+        reject(e);
+      });
+    });
+  } catch (err) {
+    bsdtarProcess.kill(9);
+    await fs.rimraf(tmpRoot);
+    throw new Error(`The response data cannot be unzipped: ${err.message}`);
+  } finally {
+    bsdtarProcess.removeAllListeners();
+    zipStream.removeAllListeners();
+  }
+  return {
+    rootDir: tmpRoot,
+    archiveSize,
+  };
+}
+
+/**
+ * Looks for items with given extensions in the given folder
+ *
+ * @param {string} appPath Full path to an app bundle
+ * @param {Array<string>} appExtensions List of matching item extensions
+ * @returns {Promise<string[]>} List of relative paths to matched items
+ */
+async function findApps(appPath, appExtensions) {
   const globPattern = `**/*.+(${appExtensions.map((ext) => ext.replace(/^\./, '')).join('|')})`;
   const sortedBundleItems = (
     await fs.glob(globPattern, {
-      cwd: tmpRoot,
+      cwd: appPath,
     })
   ).sort((a, b) => a.split(path.sep).length - b.split(path.sep).length);
-  return [tmpRoot, sortedBundleItems];
+  return sortedBundleItems;
 }
 
 /**
@@ -318,3 +406,148 @@ export function buildSafariPreferences(opts) {
   }
   return safariSettings;
 }
+
+/**
+ * Unzip the given archive and find a matching .app bundle in it
+ *
+ * @this {import('./driver').XCUITestDriver}
+ * @param {string|import('node:stream').Readable} appPathOrZipStream The path to the archive.
+ * @param {number} depth [0] the current nesting depth. App bundles whose nesting level
+ * is greater than 1 are not supported.
+ * @returns {Promise<string>} Full path to the first matching .app bundle..
+ * @throws If no matching .app bundles were found in the provided archive.
+ */
+async function unzipApp(appPathOrZipStream, depth = 0) {
+  const errMsg = `The archive '${this.opts.app}' did not have any matching ${APP_EXT} or ${IPA_EXT} ` +
+    `bundles. Please make sure the provided package is valid and contains at least one matching ` +
+    `application bundle which is not nested.`;
+  if (depth > MAX_ARCHIVE_SCAN_DEPTH) {
+    throw new Error(errMsg);
+  }
+
+  const timer = new timing.Timer().start();
+  /** @type {string} */
+  let rootDir;
+  /** @type {number} */
+  let archiveSize;
+  try {
+    if (_.isString(appPathOrZipStream)) {
+      ({rootDir, archiveSize} = await unzipFile(appPathOrZipStream));
+    } else {
+      if (depth > 0) {
+        assert.fail('Streaming unzip cannot be invoked for nested archive items');
+      }
+      ({rootDir, archiveSize} = await unzipStream(appPathOrZipStream));
+    }
+  } catch (e) {
+    this.log.debug(e.stack);
+    throw new Error(
+      `Cannot prepare the application at '${this.opts.app}' for testing. Original error: ${e.message}`
+    );
+  }
+  const secondsElapsed = timer.getDuration().asSeconds;
+  this.log.info(
+    `The app '${this.opts.app}' (${util.toReadableSizeString(archiveSize)}) ` +
+      `has been ${_.isString(appPathOrZipStream) ? 'extracted' : 'downloaded and extracted'} ` +
+      `to '${rootDir}' in ${secondsElapsed.toFixed(3)}s`
+  );
+  // it does not make much sense to approximate the speed for short downloads
+  if (secondsElapsed >= 1) {
+    const bytesPerSec = Math.floor(archiveSize / secondsElapsed);
+    this.log.debug(`Approximate decompression speed: ${util.toReadableSizeString(bytesPerSec)}/s`);
+  }
+
+  const matchedPaths = await findApps(rootDir, SUPPORTED_EXTENSIONS);
+  if (_.isEmpty(matchedPaths)) {
+    this.log.debug(`'${path.basename(rootDir)}' has no bundles`);
+  } else {
+    this.log.debug(
+      `Found ${util.pluralize('bundle', matchedPaths.length, true)} in ` +
+        `'${path.basename(rootDir)}': ${matchedPaths}`,
+    );
+  }
+  try {
+    for (const matchedPath of matchedPaths) {
+      const fullPath = path.join(rootDir, matchedPath);
+      if (await isAppBundle(fullPath)) {
+        const supportedPlatforms = await fetchSupportedAppPlatforms(fullPath);
+        if (this.isSimulator() && !supportedPlatforms.some((p) => _.includes(p, 'Simulator'))) {
+          this.log.info(
+            `'${matchedPath}' does not have Simulator devices in the list of supported platforms ` +
+              `(${supportedPlatforms.join(',')}). Skipping it`,
+          );
+          continue;
+        }
+        if (this.isRealDevice() && !supportedPlatforms.some((p) => _.includes(p, 'OS'))) {
+          this.log.info(
+            `'${matchedPath}' does not have real devices in the list of supported platforms ` +
+              `(${supportedPlatforms.join(',')}). Skipping it`,
+          );
+          continue;
+        }
+        this.log.info(
+          `'${matchedPath}' is the resulting application bundle selected from '${rootDir}'`,
+        );
+        return await isolateAppBundle(fullPath);
+      } else if (_.endsWith(_.toLower(fullPath), IPA_EXT) && (await fs.stat(fullPath)).isFile()) {
+        try {
+          return await unzipApp.bind(this)(fullPath, depth + 1);
+        } catch (e) {
+          this.log.warn(`Skipping processing of '${matchedPath}': ${e.message}`);
+        }
+      }
+    }
+  } finally {
+    await fs.rimraf(rootDir);
+  }
+  throw new Error(errMsg);
+}
+
+/**
+ * @this {import('./driver').XCUITestDriver}
+ * @param {import('@appium/types').DownloadAppOptions} opts
+ * @returns {Promise<string>}
+ */
+export async function onDownloadApp({stream}) {
+  return await unzipApp.bind(this)(stream);
+}
+
+/**
+ * @this {import('./driver').XCUITestDriver}
+ * @param {import('@appium/types').PostProcessOptions} opts
+ * @returns {Promise<import('@appium/types').PostProcessResult|false>}
+ */
+export async function onPostConfigureApp({cachedAppInfo, isUrl, appPath}) {
+  // Pick the previously cached entry if its integrity has been preserved
+  /** @type {import('@appium/types').CachedAppInfo|undefined} */
+  const appInfo = _.isPlainObject(cachedAppInfo) ? cachedAppInfo : undefined;
+  const cachedPath = appInfo ? /** @type {string} */ (appInfo.fullPath) : undefined;
+  if (
+    // If cache is present
+    appInfo && cachedPath
+    // And if the path exists
+    && await fs.exists(cachedPath)
+    // And if hash matches to the cached one if this is a file
+    // Or count of files >= of the cached one if this is a folder
+    && (
+      ((await fs.stat(cachedPath)).isFile()
+        && await fs.hash(cachedPath) === /** @type {any} */ (appInfo.integrity)?.file)
+      || (await fs.glob('**/*', {cwd: cachedPath})).length >= /** @type {any} */ (
+        appInfo.integrity
+      )?.folder
+    )
+  ) {
+    this.log.info(`Using '${cachedPath}' which was cached from '${appPath}'`);
+    return {appPath: cachedPath};
+  }
+
+  const isBundleAlreadyUnpacked = await isAppBundle(/** @type {string} */(appPath));
+  // Only local .app bundles that are available in-place should not be cached
+  if (!isUrl && isBundleAlreadyUnpacked) {
+    return false;
+  }
+  // Cache the app while unpacking the bundle if necessary
+  return {
+    appPath: isBundleAlreadyUnpacked ? appPath : await unzipApp.bind(this)(/** @type {string} */(appPath))
+  };
+}