From b04cebd99418b0e6d55d3c1813700779248e6541 Mon Sep 17 00:00:00 2001 From: Mykola Mokhnach Date: Tue, 23 Apr 2024 07:38:34 +0200 Subject: [PATCH] feat: Perform bundles extraction in stream (#2387) --- lib/app-utils.js | 253 +++++++++++++++++++++++++++++++++-- lib/driver.js | 110 +-------------- package.json | 2 +- test/unit/app-utils-specs.js | 91 +++++++++++++ 4 files changed, 340 insertions(+), 116 deletions(-) create mode 100644 test/unit/app-utils-specs.js diff --git a/lib/app-utils.js b/lib/app-utils.js index dda29ce53..f6dc02843 100644 --- a/lib/app-utils.js +++ b/lib/app-utils.js @@ -1,11 +1,13 @@ import _ from 'lodash'; import path from 'path'; -import {plist, fs, util, tempDir, zip} from 'appium/support'; +import {plist, fs, util, tempDir, zip, timing} from 'appium/support'; import log from './logger.js'; import {LRUCache} from 'lru-cache'; import os from 'node:os'; import {exec} from 'teen_process'; import B from 'bluebird'; +import {spawn} from 'node:child_process'; +import assert from 'node:assert'; const STRINGSDICT_RESOURCE = '.stringsdict'; const STRINGS_RESOURCE = '.strings'; @@ -22,6 +24,9 @@ const SAFARI_OPTS_ALIASES_MAP = /** @type {const} */ ({ safariIgnoreFraudWarning: [['WarnAboutFraudulentWebsites'], (x) => Number(!x)], safariOpenLinksInBackground: [['OpenLinksInBackground'], (x) => Number(Boolean(x))], }); +const MAX_ARCHIVE_SCAN_DEPTH = 1; +export const SUPPORTED_EXTENSIONS = [IPA_EXT, APP_EXT]; +const MACOS_RESOURCE_FOLDER = '__MACOSX'; /** @@ -261,27 +266,110 @@ export async function isAppBundle(appPath) { } /** - * Extract the given archive and looks for items with given extensions in it + * @typedef {Object} UnzipInfo + * @property {string} rootDir + * @property {number} archiveSize + */ + +/** + * Unzips a ZIP archive on the local file system. * * @param {string} archivePath Full path to a .zip archive - * @param {Array} appExtensions List of matching item extensions - * @returns {Promise<[string, string[]]>} Tuple, where the first element points to - * a temporary folder root where the archive has been extracted and the second item - * contains a list of relative paths to matched items + * @returns {Promise} temporary folder root where the archive has been extracted */ -export async function findApps(archivePath, appExtensions) { +export async function unzipFile(archivePath) { const useSystemUnzipEnv = process.env.APPIUM_PREFER_SYSTEM_UNZIP; const useSystemUnzip = _.isEmpty(useSystemUnzipEnv) || !['0', 'false'].includes(_.toLower(useSystemUnzipEnv)); const tmpRoot = await tempDir.openDir(); - await zip.extractAllTo(archivePath, tmpRoot, {useSystemUnzip}); + try { + await zip.extractAllTo(archivePath, tmpRoot, {useSystemUnzip}); + } catch (e) { + await fs.rimraf(tmpRoot); + throw e; + } + return { + rootDir: tmpRoot, + archiveSize: (await fs.stat(archivePath)).size, + }; +} + +/** + * Unzips a ZIP archive from a stream. + * Uses bdstar tool for this purpose. + * This allows to optimize the time needed to prepare the app under test + * to MAX(download, unzip) instead of SUM(download, unzip) + * + * @param {import('node:stream').Readable} zipStream + * @returns {Promise} + */ +export async function unzipStream(zipStream) { + const tmpRoot = await tempDir.openDir(); + const bsdtarProcess = spawn(await fs.which('bsdtar'), [ + '-x', + '--exclude', MACOS_RESOURCE_FOLDER, + '--exclude', `${MACOS_RESOURCE_FOLDER}/*`, + '-', + ], { + cwd: tmpRoot, + }); + let archiveSize = 0; + bsdtarProcess.stderr.on('data', (chunk) => { + const stderr = chunk.toString(); + if (_.trim(stderr)) { + log.warn(stderr); + } + }); + zipStream.on('data', (chunk) => { + archiveSize += _.size(chunk); + }); + zipStream.pipe(bsdtarProcess.stdin); + try { + await new B((resolve, reject) => { + zipStream.once('error', reject); + bsdtarProcess.once('exit', (code, signal) => { + zipStream.unpipe(bsdtarProcess.stdin); + log.debug(`bsdtar process exited with code ${code}, signal ${signal}`); + if (code === 0) { + resolve(); + } else { + reject(new Error('Is it a valid ZIP archive?')); + } + }); + bsdtarProcess.once('error', (e) => { + zipStream.unpipe(bsdtarProcess.stdin); + reject(e); + }); + }); + } catch (err) { + bsdtarProcess.kill(9); + await fs.rimraf(tmpRoot); + throw new Error(`The response data cannot be unzipped: ${err.message}`); + } finally { + bsdtarProcess.removeAllListeners(); + zipStream.removeAllListeners(); + } + return { + rootDir: tmpRoot, + archiveSize, + }; +} + +/** + * Looks for items with given extensions in the given folder + * + * @param {string} appPath Full path to an app bundle + * @param {Array} appExtensions List of matching item extensions + * @returns {Promise} List of relative paths to matched items + */ +async function findApps(appPath, appExtensions) { const globPattern = `**/*.+(${appExtensions.map((ext) => ext.replace(/^\./, '')).join('|')})`; const sortedBundleItems = ( await fs.glob(globPattern, { - cwd: tmpRoot, + cwd: appPath, }) ).sort((a, b) => a.split(path.sep).length - b.split(path.sep).length); - return [tmpRoot, sortedBundleItems]; + return sortedBundleItems; } /** @@ -318,3 +406,148 @@ export function buildSafariPreferences(opts) { } return safariSettings; } + +/** + * Unzip the given archive and find a matching .app bundle in it + * + * @this {import('./driver').XCUITestDriver} + * @param {string|import('node:stream').Readable} appPathOrZipStream The path to the archive. + * @param {number} depth [0] the current nesting depth. App bundles whose nesting level + * is greater than 1 are not supported. + * @returns {Promise} Full path to the first matching .app bundle.. + * @throws If no matching .app bundles were found in the provided archive. + */ +async function unzipApp(appPathOrZipStream, depth = 0) { + const errMsg = `The archive '${this.opts.app}' did not have any matching ${APP_EXT} or ${IPA_EXT} ` + + `bundles. Please make sure the provided package is valid and contains at least one matching ` + + `application bundle which is not nested.`; + if (depth > MAX_ARCHIVE_SCAN_DEPTH) { + throw new Error(errMsg); + } + + const timer = new timing.Timer().start(); + /** @type {string} */ + let rootDir; + /** @type {number} */ + let archiveSize; + try { + if (_.isString(appPathOrZipStream)) { + ({rootDir, archiveSize} = await unzipFile(appPathOrZipStream)); + } else { + if (depth > 0) { + assert.fail('Streaming unzip cannot be invoked for nested archive items'); + } + ({rootDir, archiveSize} = await unzipStream(appPathOrZipStream)); + } + } catch (e) { + this.log.debug(e.stack); + throw new Error( + `Cannot prepare the application at '${this.opts.app}' for testing. Original error: ${e.message}` + ); + } + const secondsElapsed = timer.getDuration().asSeconds; + this.log.info( + `The app '${this.opts.app}' (${util.toReadableSizeString(archiveSize)}) ` + + `has been ${_.isString(appPathOrZipStream) ? 'extracted' : 'downloaded and extracted'} ` + + `to '${rootDir}' in ${secondsElapsed.toFixed(3)}s` + ); + // it does not make much sense to approximate the speed for short downloads + if (secondsElapsed >= 1) { + const bytesPerSec = Math.floor(archiveSize / secondsElapsed); + this.log.debug(`Approximate decompression speed: ${util.toReadableSizeString(bytesPerSec)}/s`); + } + + const matchedPaths = await findApps(rootDir, SUPPORTED_EXTENSIONS); + if (_.isEmpty(matchedPaths)) { + this.log.debug(`'${path.basename(rootDir)}' has no bundles`); + } else { + this.log.debug( + `Found ${util.pluralize('bundle', matchedPaths.length, true)} in ` + + `'${path.basename(rootDir)}': ${matchedPaths}`, + ); + } + try { + for (const matchedPath of matchedPaths) { + const fullPath = path.join(rootDir, matchedPath); + if (await isAppBundle(fullPath)) { + const supportedPlatforms = await fetchSupportedAppPlatforms(fullPath); + if (this.isSimulator() && !supportedPlatforms.some((p) => _.includes(p, 'Simulator'))) { + this.log.info( + `'${matchedPath}' does not have Simulator devices in the list of supported platforms ` + + `(${supportedPlatforms.join(',')}). Skipping it`, + ); + continue; + } + if (this.isRealDevice() && !supportedPlatforms.some((p) => _.includes(p, 'OS'))) { + this.log.info( + `'${matchedPath}' does not have real devices in the list of supported platforms ` + + `(${supportedPlatforms.join(',')}). Skipping it`, + ); + continue; + } + this.log.info( + `'${matchedPath}' is the resulting application bundle selected from '${rootDir}'`, + ); + return await isolateAppBundle(fullPath); + } else if (_.endsWith(_.toLower(fullPath), IPA_EXT) && (await fs.stat(fullPath)).isFile()) { + try { + return await unzipApp.bind(this)(fullPath, depth + 1); + } catch (e) { + this.log.warn(`Skipping processing of '${matchedPath}': ${e.message}`); + } + } + } + } finally { + await fs.rimraf(rootDir); + } + throw new Error(errMsg); +} + +/** + * @this {import('./driver').XCUITestDriver} + * @param {import('@appium/types').DownloadAppOptions} opts + * @returns {Promise} + */ +export async function onDownloadApp({stream}) { + return await unzipApp.bind(this)(stream); +} + +/** + * @this {import('./driver').XCUITestDriver} + * @param {import('@appium/types').PostProcessOptions} opts + * @returns {Promise} + */ +export async function onPostConfigureApp({cachedAppInfo, isUrl, appPath}) { + // Pick the previously cached entry if its integrity has been preserved + /** @type {import('@appium/types').CachedAppInfo|undefined} */ + const appInfo = _.isPlainObject(cachedAppInfo) ? cachedAppInfo : undefined; + const cachedPath = appInfo ? /** @type {string} */ (appInfo.fullPath) : undefined; + if ( + // If cache is present + appInfo && cachedPath + // And if the path exists + && await fs.exists(cachedPath) + // And if hash matches to the cached one if this is a file + // Or count of files >= of the cached one if this is a folder + && ( + ((await fs.stat(cachedPath)).isFile() + && await fs.hash(cachedPath) === /** @type {any} */ (appInfo.integrity)?.file) + || (await fs.glob('**/*', {cwd: cachedPath})).length >= /** @type {any} */ ( + appInfo.integrity + )?.folder + ) + ) { + this.log.info(`Using '${cachedPath}' which was cached from '${appPath}'`); + return {appPath: cachedPath}; + } + + const isBundleAlreadyUnpacked = await isAppBundle(/** @type {string} */(appPath)); + // Only local .app bundles that are available in-place should not be cached + if (!isUrl && isBundleAlreadyUnpacked) { + return false; + } + // Cache the app while unpacking the bundle if necessary + return { + appPath: isBundleAlreadyUnpacked ? appPath : await unzipApp.bind(this)(/** @type {string} */(appPath)) + }; +} diff --git a/lib/driver.js b/lib/driver.js index c0d8633e9..e439a3295 100644 --- a/lib/driver.js +++ b/lib/driver.js @@ -12,15 +12,12 @@ import EventEmitter from 'node:events'; import path from 'node:path'; import url from 'node:url'; import { - APP_EXT, - IPA_EXT, + SUPPORTED_EXTENSIONS, SAFARI_BUNDLE_ID, extractBundleId, extractBundleVersion, - fetchSupportedAppPlatforms, - findApps, - isAppBundle, - isolateAppBundle, + onPostConfigureApp, + onDownloadApp, verifyApplicationPlatform, } from './app-utils'; import commands from './commands'; @@ -70,8 +67,6 @@ import { const SHUTDOWN_OTHER_FEAT_NAME = 'shutdown_other_sims'; const CUSTOMIZE_RESULT_BUNDLE_PATH = 'customize_result_bundle_path'; -const SUPPORTED_EXTENSIONS = [IPA_EXT, APP_EXT]; -const MAX_ARCHIVE_SCAN_DEPTH = 1; const defaultServerCaps = { webStorageEnabled: false, locationContextEnabled: false, @@ -1065,107 +1060,12 @@ export class XCUITestDriver extends BaseDriver { } this.opts.app = await this.helpers.configureApp(this.opts.app, { - onPostProcess: this.onPostConfigureApp.bind(this), + onPostProcess: onPostConfigureApp.bind(this), + onDownload: onDownloadApp.bind(this), supportedExtensions: SUPPORTED_EXTENSIONS, }); } - /** - * Unzip the given archive and find a matching .app bundle in it - * - * @param {string} appPath The path to the archive. - * @param {number} depth [0] the current nesting depth. App bundles whose nesting level - * is greater than 1 are not supported. - * @returns {Promise} Full path to the first matching .app bundle.. - * @throws If no matching .app bundles were found in the provided archive. - */ - async unzipApp(appPath, depth = 0) { - if (depth > MAX_ARCHIVE_SCAN_DEPTH) { - throw new Error('Nesting of package bundles is not supported'); - } - const [rootDir, matchedPaths] = await findApps(appPath, SUPPORTED_EXTENSIONS); - if (_.isEmpty(matchedPaths)) { - this.log.debug(`'${path.basename(appPath)}' has no bundles`); - } else { - this.log.debug( - `Found ${util.pluralize('bundle', matchedPaths.length, true)} in ` + - `'${path.basename(appPath)}': ${matchedPaths}`, - ); - } - try { - for (const matchedPath of matchedPaths) { - const fullPath = path.join(rootDir, matchedPath); - if (await isAppBundle(fullPath)) { - const supportedPlatforms = await fetchSupportedAppPlatforms(fullPath); - if (this.isSimulator() && !supportedPlatforms.some((p) => _.includes(p, 'Simulator'))) { - this.log.info( - `'${matchedPath}' does not have Simulator devices in the list of supported platforms ` + - `(${supportedPlatforms.join(',')}). Skipping it`, - ); - continue; - } - if (this.isRealDevice() && !supportedPlatforms.some((p) => _.includes(p, 'OS'))) { - this.log.info( - `'${matchedPath}' does not have real devices in the list of supported platforms ` + - `(${supportedPlatforms.join(',')}). Skipping it`, - ); - continue; - } - this.log.info( - `'${matchedPath}' is the resulting application bundle selected from '${appPath}'`, - ); - return await isolateAppBundle(fullPath); - } else if (_.endsWith(_.toLower(fullPath), IPA_EXT) && (await fs.stat(fullPath)).isFile()) { - try { - return await this.unzipApp(fullPath, depth + 1); - } catch (e) { - this.log.warn(`Skipping processing of '${matchedPath}': ${e.message}`); - } - } - } - } finally { - await fs.rimraf(rootDir); - } - throw new Error( - `${this.opts.app} did not have any matching ${APP_EXT} or ${IPA_EXT} ` + - `bundles. Please make sure the provided package is valid and contains at least one matching ` + - `application bundle which is not nested.`, - ); - } - - async onPostConfigureApp({cachedAppInfo, isUrl, appPath}) { - // Pick the previously cached entry if its integrity has been preserved - if ( - _.isPlainObject(cachedAppInfo) && - (await fs.stat(appPath)).isFile() && - (await fs.hash(appPath)) === cachedAppInfo.packageHash && - (await fs.exists(cachedAppInfo.fullPath)) && - ( - await fs.glob('**/*', { - cwd: cachedAppInfo.fullPath, - }) - ).length === cachedAppInfo.integrity.folder - ) { - this.log.info(`Using '${cachedAppInfo.fullPath}' which was cached from '${appPath}'`); - return {appPath: cachedAppInfo.fullPath}; - } - - // Only local .app bundles that are available in-place should not be cached - if (await isAppBundle(appPath)) { - return false; - } - - // Extract the app bundle and cache it - try { - return {appPath: await this.unzipApp(appPath)}; - } finally { - // Cleanup previously downloaded archive - if (isUrl) { - await fs.rimraf(appPath); - } - } - } - async determineDevice() { // in the one case where we create a sim, we will set this state this.lifecycleData.createSim = false; diff --git a/package.json b/package.json index 591081fe9..70b6d8c74 100644 --- a/package.json +++ b/package.json @@ -131,7 +131,7 @@ "singleQuote": true }, "peerDependencies": { - "appium": "^2.4.1" + "appium": "^2.5.4" }, "devDependencies": { "@appium/docutils": "^1.0.2", diff --git a/test/unit/app-utils-specs.js b/test/unit/app-utils-specs.js new file mode 100644 index 000000000..fbbe49338 --- /dev/null +++ b/test/unit/app-utils-specs.js @@ -0,0 +1,91 @@ +import { + unzipStream, + unzipFile, +} from '../../lib/app-utils'; +import chai from 'chai'; +import chaiAsPromised from 'chai-as-promised'; +import { fs, tempDir, zip } from 'appium/support'; +import path from 'node:path'; + +chai.should(); +chai.use(chaiAsPromised); + +describe('app-utils', function () { + describe('unzipStream', function () { + it('should unzip from stream', async function () { + try { + await fs.which('bsdtar'); + } catch (e) { + return; + } + + const tmpDir = await tempDir.openDir(); + let appRoot; + let srcStream; + try { + const tmpSrc = path.join(tmpDir, 'temp.zip'); + await zip.toArchive(tmpSrc, { + cwd: path.resolve(__dirname, '..', 'assets', 'biometric.app'), + }); + srcStream = fs.createReadStream(tmpSrc); + ({rootDir: appRoot} = await unzipStream(srcStream)); + await fs.exists(path.resolve(appRoot, 'Info.plist')).should.eventually.be.true; + } finally { + await fs.rimraf(tmpDir); + if (appRoot) { + await fs.rimraf(appRoot); + } + } + }); + + it('should fail for invalid archives', async function () { + try { + await fs.which('bsdtar'); + } catch (e) { + return; + } + + const tmpDir = await tempDir.openDir(); + let srcStream; + try { + const tmpSrc = path.join(tmpDir, 'Info.plist'); + await fs.copyFile(path.resolve(__dirname, '..', 'assets', 'biometric.app', 'Info.plist'), tmpSrc); + srcStream = fs.createReadStream(tmpSrc); + await unzipStream(srcStream).should.be.rejected; + } finally { + await fs.rimraf(tmpDir); + } + }); + }); + + describe('unzipFile', function () { + it('should unzip from file', async function () { + const tmpDir = await tempDir.openDir(); + let appRoot; + try { + const tmpSrc = path.join(tmpDir, 'temp.zip'); + await zip.toArchive(tmpSrc, { + cwd: path.resolve(__dirname, '..', 'assets', 'biometric.app'), + }); + ({rootDir: appRoot} = await unzipFile(tmpSrc)); + await fs.exists(path.resolve(appRoot, 'Info.plist')).should.eventually.be.true; + } finally { + await fs.rimraf(tmpDir); + if (appRoot) { + await fs.rimraf(appRoot); + } + } + }); + + it('should fail for invalid archives', async function () { + const tmpDir = await tempDir.openDir(); + try { + const tmpSrc = path.join(tmpDir, 'Info.plist'); + await fs.copyFile(path.resolve(__dirname, '..', 'assets', 'biometric.app', 'Info.plist'), tmpSrc); + await unzipFile(tmpSrc).should.be.rejected; + } finally { + await fs.rimraf(tmpDir); + } + }); + }); +});