From fb1c7df99e18dd677535af9cbe1c297a0bc06470 Mon Sep 17 00:00:00 2001 From: jona <93538252+wannkunstbeikor@users.noreply.github.com> Date: Fri, 27 Oct 2023 22:22:32 +0200 Subject: [PATCH] [Sdk] Fixed issue with superbundlemanifest bundles --- FrostySdk/Managers/AssetManager.cs | 118 +++++++++--------- FrostySdk/Managers/FileSystemManager.cs | 54 ++++---- .../Managers/Loaders/ManifestAssetLoader.cs | 37 +++--- FrostySdk/ProfilesLibrary.cs | 9 +- 4 files changed, 107 insertions(+), 111 deletions(-) diff --git a/FrostySdk/Managers/AssetManager.cs b/FrostySdk/Managers/AssetManager.cs index 39514f457..f18bd3efe 100644 --- a/FrostySdk/Managers/AssetManager.cs +++ b/FrostySdk/Managers/AssetManager.cs @@ -21,17 +21,17 @@ namespace Frosty.Sdk.Managers; public static class AssetManager { public static bool IsInitialized { get; private set; } - + public static ILogger? Logger { get; set; } private static readonly Dictionary s_bundleMapping = new(); - + private static readonly Dictionary s_ebxNameHashMapping = new(); private static readonly Dictionary s_ebxGuidMapping = new(); - + private static readonly Dictionary s_resNameHashMapping = new(); private static readonly Dictionary s_resRidMapping = new(); - + private static readonly Dictionary s_chunkGuidMapping = new(); /// @@ -54,7 +54,7 @@ public static bool Initialize(PatchResult? patchResult = null) { return true; } - + if (!FileSystemManager.IsInitialized || !ResourceManager.IsInitialized) { return false; @@ -68,34 +68,34 @@ public static bool Initialize(PatchResult? patchResult = null) if (FileSystemManager.BundleFormat == BundleFormat.Dynamic2018 || FileSystemManager.BundleFormat == BundleFormat.SuperBundleManifest) { Logger?.Report("Sdk", "Loading FileInfos from catalogs"); - + timer.Start(); ResourceManager.LoadInstallChunks(); timer.Stop(); - + Logger?.Report("Sdk", $"Loaded FileInfos from catalogs in {timer.Elapsed.TotalSeconds} seconds"); } - + IAssetLoader assetLoader = GetAssetLoader(); - + Logger?.Report("Sdk", "Loading Assets from SuperBundles"); - + timer.Restart(); assetLoader.Load(); timer.Stop(); - + Logger?.Report("Sdk", $"Loaded Assets from SuperBundles in {timer.Elapsed.TotalSeconds} seconds"); - + ResourceManager.CLearInstallChunks(); - + Logger?.Report("Sdk", "Indexing Ebx"); - + timer.Restart(); DoEbxIndexing(); timer.Stop(); - + Logger?.Report("Sdk", $"Indexed ebx in {timer.Elapsed}"); - + WriteCache(); if (prePatchEbx.Count > 0 || prePatchRes.Count > 0 || prePatchChunks.Count > 0) @@ -120,7 +120,7 @@ public static bool Initialize(PatchResult? patchResult = null) patchResult.Added.Ebx.Add(ebxAssetEntry.Name); } } - + // modified/added res foreach (ResAssetEntry resAssetEntry in s_resNameHashMapping.Values) { @@ -139,7 +139,7 @@ public static bool Initialize(PatchResult? patchResult = null) patchResult.Added.Res.Add(resAssetEntry.Name); } } - + // modified/added chunks foreach (ChunkAssetEntry chunkAssetEntry in s_chunkGuidMapping.Values) { @@ -164,28 +164,28 @@ public static bool Initialize(PatchResult? patchResult = null) { patchResult.Removed.Ebx.Add(ebxAssetEntry.Name); } - + // removed res foreach (ResAssetEntry resAssetEntry in prePatchRes) { patchResult.Removed.Res.Add(resAssetEntry.Name); } - + // removed chunks foreach (ChunkAssetEntry chunkAssetEntry in prePatchChunks) { patchResult.Removed.Chunks.Add(chunkAssetEntry.Id); } } - + prePatchEbx.Clear(); prePatchRes.Clear(); prePatchChunks.Clear(); } } - + Logger?.Report("Sdk", "Finished initializing"); - + IsInitialized = true; return true; } @@ -205,7 +205,7 @@ public static bool Initialize(PatchResult? patchResult = null) } #endregion - + #region -- AssetEntry -- #region -- Ebx -- @@ -220,7 +220,7 @@ public static bool Initialize(PatchResult? patchResult = null) int nameHash = Utils.Utils.HashString(name, true); return s_ebxNameHashMapping.TryGetValue(nameHash, out EbxAssetEntry? value) ? value : null; } - + /// /// Gets the by . /// @@ -232,7 +232,7 @@ public static bool Initialize(PatchResult? patchResult = null) } #endregion - + #region -- Res -- /// @@ -245,7 +245,7 @@ public static bool Initialize(PatchResult? patchResult = null) int nameHash = Utils.Utils.HashString(name, true); return s_resNameHashMapping.TryGetValue(nameHash, out ResAssetEntry? value) ? value : null; } - + /// /// Gets the by Rid. /// @@ -259,7 +259,7 @@ public static bool Initialize(PatchResult? patchResult = null) #endregion #region -- Chunk -- - + /// /// Gets the by Id. /// @@ -277,7 +277,7 @@ public static bool Initialize(PatchResult? patchResult = null) #endregion #region -- GetAsset -- - + public static EbxAsset GetEbx(EbxAssetEntry entry) { //using (EbxReader reader = EbxReader.CreateReader(GetAsset(entry))) @@ -295,7 +295,7 @@ public static T GetResAs(ResAssetEntry entry) T retVal = new(); retVal.Set(entry); retVal.Deserialize(stream); - + return retVal; } } @@ -311,7 +311,7 @@ public static Block GetRawAsset(AssetEntry entry) } #endregion - + public static IEnumerable EnumerateEbxAssetEntries() { foreach (EbxAssetEntry entry in s_ebxNameHashMapping.Values) @@ -387,7 +387,7 @@ internal static void AddChunk(ChunkAssetEntry entry, int bundleId) existing.FileInfos.UnionWith(entry.FileInfos); existing.Bundles.Add(bundleId); - + if (existing.LogicalSize == 0) { // this chunk was first added as a superbundle chunk, so add logical offset/size and sha1 @@ -416,7 +416,7 @@ internal static void AddSuperBundleChunk(ChunkAssetEntry entry) { // add existing Bundles entry.Bundles.UnionWith(existing.Bundles); - + entry.FileInfos.UnionWith(existing.FileInfos); // add logicalOffset/Size, since those are only stored in bundles @@ -444,7 +444,7 @@ private static void DoEbxIndexing() { // TODO: implement GetAsset() return; - + if (s_ebxGuidMapping.Count > 0) { return; @@ -514,12 +514,12 @@ private static bool ReadCache(out List prePatchEbx, out List(); prePatchRes = new List(); prePatchChunks = new List(); - + if (!File.Exists($"{FileSystemManager.CacheName}.cache")) { return false; } - + bool isPatched = false; using (DataStream stream = new(new FileStream($"{FileSystemManager.CacheName}.cache", FileMode.Open, FileAccess.Read))) @@ -547,7 +547,7 @@ private static bool ReadCache(out List prePatchEbx, out List prePatchEbx, out List prePatchEbx, out List prePatchEbx, out List s_installChunks = new(); private static readonly Dictionary s_sbIcMapping = new(); private static readonly List s_casFiles = new(); - + private static Type? s_deobfuscator; private static readonly Dictionary> s_memoryFs = new(); @@ -67,7 +67,7 @@ public static bool Initialize(string basePath) { Sources.RemoveAt(0); } - + if (Directory.Exists($"{BasePath}/Update")) { foreach (string dlc in Directory.EnumerateDirectories($"{BasePath}/Update")) @@ -79,7 +79,7 @@ public static bool Initialize(string basePath) } else { - Sources.Insert(0, new FileSystemSource($"Update/{subPath}/Data", FileSystemSource.Type.DLC)); + Sources.Insert(0, new FileSystemSource($"Update/{subPath}/Data", FileSystemSource.Type.DLC)); } } } @@ -88,7 +88,7 @@ public static bool Initialize(string basePath) { return false; } - + IsInitialized = true; return true; } @@ -137,25 +137,25 @@ public static string ResolvePath(bool isPatch, string filename) return string.Empty; } - + public static string GetFilePath(CasFileIdentifier casFileIdentifier) { InstallChunkInfo installChunkInfo = s_installChunks[s_persistentIndexMapping[casFileIdentifier.InstallChunkIndex]]; if (casFileIdentifier.IsPatch) { - FileSystemSource.Patch.ResolvePath( + return FileSystemSource.Patch.ResolvePath( $"{installChunkInfo.InstallBundle}/cas_{casFileIdentifier.CasIndex:D2}.cas"); } return FileSystemSource.Base.ResolvePath( $"{installChunkInfo.InstallBundle}/cas_{casFileIdentifier.CasIndex:D2}.cas"); } - + public static string GetFilePath(int casIndex) { return s_casFiles[casIndex]; } - + public static IDeobfuscator? CreateDeobfuscator() => s_deobfuscator != null ? (IDeobfuscator?)Activator.CreateInstance(s_deobfuscator) : null; public static IEnumerable EnumerateSuperBundles() @@ -165,7 +165,7 @@ public static IEnumerable EnumerateSuperBundles() yield return sbInfo; } } - + public static IEnumerable EnumerateInstallChunks() { foreach (InstallChunkInfo installChunkInfo in s_installChunks) @@ -178,7 +178,7 @@ public static InstallChunkInfo GetInstallChunkInfo(int index) { return s_installChunks[s_persistentIndexMapping[index]]; } - + public static InstallChunkInfo GetInstallChunkInfo(Guid id) { return s_installChunks.FirstOrDefault(ic => ic.Id == id) ?? throw new KeyNotFoundException(); @@ -194,10 +194,10 @@ public static SuperBundleInstallChunk GetSuperBundleInstallChunk(string sbIcName { return s_sbIcMapping[Utils.Utils.HashString(sbIcName, true)]; } - + public static bool HasFileInMemoryFs(string name) => s_memoryFs.ContainsKey(name); public static Block GetFileFromMemoryFs(string name) => s_memoryFs[name]; - + private static bool LoadInitFs(string name, bool isBase = false) { ParseGamePlatform(name.Remove(0, 7)); @@ -209,13 +209,13 @@ private static bool LoadInitFs(string name, bool isBase = false) } else { - path = ResolvePath(name); + path = ResolvePath(name); } if (string.IsNullOrEmpty(path)) { return false; } - + DbObject? initFs = DbObject.Deserialize(path); if (initFs is null) @@ -229,16 +229,16 @@ private static bool LoadInitFs(string name, bool isBase = false) { return false; } - + if (!KeyManager.HasKey("InitFsKey")) { return false; } - + using (BlockStream stream = new(new Block(initFs.AsDict().AsBlob("encrypted")))) { stream.Decrypt(KeyManager.GetKey("InitFsKey"), PaddingMode.PKCS7); - + initFs = DbObject.Deserialize(stream); if (initFs is null) @@ -260,7 +260,7 @@ private static bool LoadInitFs(string name, bool isBase = false) } s_memoryFs.TryAdd(fileName, new Block(file.AsBlob("payload"))); } - + return true; } @@ -297,7 +297,7 @@ private static void ParseGamePlatform(string platform) throw new NotImplementedException($"GamePlatform not implemented: {platform}"); } } - + private static bool ProcessLayouts() { string baseLayoutPath = ResolvePath(false, "layout.toc"); @@ -307,7 +307,7 @@ private static bool ProcessLayouts() { return false; } - + // Process base layout.toc DbObjectDict? baseLayout = DbObject.Deserialize(baseLayoutPath)?.AsDict(); @@ -321,7 +321,7 @@ private static bool ProcessLayouts() { BundleFormat = BundleFormat.Kelvin; } - + foreach (DbObject superBundle in baseLayout.AsDict().AsList("superBundles")) { string name = superBundle.AsDict().AsString("name"); @@ -337,7 +337,7 @@ private static bool ProcessLayouts() { return false; } - + foreach (DbObject superBundle in patchLayout.AsList("superBundles")) { // Merge super bundles @@ -359,7 +359,7 @@ private static bool ProcessLayouts() { BundleFormat = BundleFormat.SuperBundleManifest; } - + if (!LoadInitFs(patchLayout.AsList("fs")[0].AsString())) { return false; @@ -381,7 +381,7 @@ private static bool ProcessLayouts() { BundleFormat = BundleFormat.SuperBundleManifest; } - + if (!LoadInitFs(baseLayout.AsList("fs")[0].AsString())) { return false; @@ -398,7 +398,7 @@ private static bool ProcessInstallChunks(DbObjectDict? installManifest) // Older games dont have an InstallManifest, they have one InstallChunk in the data/patch folder InstallChunkInfo ic = new(); foreach (SuperBundleInfo sb in s_superBundleMapping.Values) - { + { ic.SuperBundles.Add(sb.Name); SuperBundleInstallChunk sbIc = new(sb, ic, InstallChunkType.Default); @@ -415,7 +415,7 @@ private static bool ProcessInstallChunks(DbObjectDict? installManifest) { ParseGamePlatform(platform); } - + // check for platform, else we get it from the initFs foreach (DbObject installChunk in installManifest.AsList("installChunks")) { @@ -503,7 +503,7 @@ private static bool ProcessInstallChunks(DbObjectDict? installManifest) { BundleFormat = (BundleFormat)installManifest.AsDict("settings").AsLong("bundleFormat", (long)BundleFormat.Dynamic2018); } - + foreach (SuperBundleInfo sb in s_superBundleMapping.Values) { if (sb.InstallChunks.Count == 0) diff --git a/FrostySdk/Managers/Loaders/ManifestAssetLoader.cs b/FrostySdk/Managers/Loaders/ManifestAssetLoader.cs index df9e462c4..1fa2a7492 100644 --- a/FrostySdk/Managers/Loaders/ManifestAssetLoader.cs +++ b/FrostySdk/Managers/Loaders/ManifestAssetLoader.cs @@ -19,13 +19,13 @@ public void Load() // all of the bundles and chunks of all SuperBundles are put into the manifest // afaik u cant reconstruct the SuperBundles, so this might make things a bit ugly // They also have catalog files which entries are not used, but they still make a sanity check for the offsets and indices in the file - + DbObjectDict manifest = FileSystemManager.SuperBundleManifest!; - + CasFileIdentifier file = CasFileIdentifier.FromManifestFileIdentifier(manifest.AsUInt("file")); - + string path = FileSystemManager.GetFilePath(file); - + using (BlockStream stream = BlockStream.FromFile(path, manifest.AsUInt("offset"), manifest.AsInt("size"))) { uint resourceInfoCount = stream.ReadUInt32(); @@ -33,7 +33,7 @@ public void Load() uint chunkCount = stream.ReadUInt32(); (CasFileIdentifier, uint, long)[] files = new (CasFileIdentifier, uint, long)[resourceInfoCount]; - + // resource infos for (int i = 0; i < resourceInfoCount; i++) { @@ -42,26 +42,26 @@ public void Load() } Dictionary> mapping = new(); - + // bundles for (int i = 0; i < bundleCount; i++) { int nameHash = stream.ReadInt32(); int startIndex = stream.ReadInt32(); int resourceCount = stream.ReadInt32(); - + // unknown, always 0 stream.Position += sizeof(ulong); - + (CasFileIdentifier, uint, long) resourceInfo = files[startIndex]; - // we use the installChunk of the bundle to get a superBundle and SuperBundleInstallChunk + // we use the installChunk of the bundle to get a superBundle and SuperBundleInstallChunk InstallChunkInfo ic = FileSystemManager.GetInstallChunkInfo(resourceInfo.Item1.InstallChunkIndex); string superbundle = ic.SuperBundles.FirstOrDefault() ?? string.Empty; Debug.Assert(!string.IsNullOrEmpty(superbundle), "no super bundle found for install chunk"); // hack we just assume there are no splitSuperBundles SuperBundleInstallChunk sbIc = FileSystemManager.GetSuperBundleInstallChunk(superbundle); - + BinaryBundle bundleMeta; using (BlockStream bundleStream = BlockStream.FromFile( FileSystemManager.GetFilePath(resourceInfo.Item1), resourceInfo.Item2, @@ -71,13 +71,12 @@ public void Load() } // get name since they are hashed - bool needToGetName = !ProfilesLibrary.SharedBundles.TryGetValue(nameHash, out string? name); - foreach (EbxAssetEntry ebx in bundleMeta.EbxList) + if (!ProfilesLibrary.SharedBundles.TryGetValue(nameHash, out string? name)) { - if (needToGetName) + foreach (EbxAssetEntry ebx in bundleMeta.EbxList) { // blueprint and sublevel bundles always have an ebx with the same name - string potentialName = $"{FileSystemManager.GamePlatform}/{ebx.Name}"; + string potentialName = ebx.Name.StartsWith(FileSystemManager.GamePlatform.ToString(), StringComparison.OrdinalIgnoreCase) ? ebx.Name : $"{FileSystemManager.GamePlatform}/{ebx.Name}"; int hash = Utils.Utils.HashString(potentialName, true); if (nameHash == hash) { @@ -86,17 +85,15 @@ public void Load() } } } - - Debug.Assert(!string.IsNullOrEmpty(name), "couldn't resolve bundle name"); - // if we couldn't get a name just use the nameHash + // if we couldn't get a name just use the nameHash for now when indexing ebx the ui stuff will assign those if (string.IsNullOrEmpty(name)) { name = nameHash.ToString("X8"); } BundleInfo bundle = AssetManager.AddBundle(name, sbIc); - + // load the assets // we use the file infos from the catalogs, since its easier even if they are not used by the game foreach (EbxAssetEntry ebx in bundleMeta.EbxList) @@ -106,7 +103,7 @@ public void Load() { ebx.FileInfos.UnionWith(fileInfos); } - + AssetManager.AddEbx(ebx, bundle.Id); } @@ -132,7 +129,7 @@ public void Load() AssetManager.AddChunk(chunk, bundle.Id); } } - + // chunks for (int i = 0; i < chunkCount; i++) { diff --git a/FrostySdk/ProfilesLibrary.cs b/FrostySdk/ProfilesLibrary.cs index 56367799c..27085cdd5 100644 --- a/FrostySdk/ProfilesLibrary.cs +++ b/FrostySdk/ProfilesLibrary.cs @@ -5,14 +5,13 @@ using System.Text.Json; using Frosty.Sdk.Profiles; using Frosty.Sdk.IO.Compression; -using static Frosty.Sdk.Utils.Utils; namespace Frosty.Sdk; public static class ProfilesLibrary { public static bool IsInitialized { get; private set; } - + public static string ProfileName => s_effectiveProfile?.Name ?? string.Empty; public static string DisplayName => s_effectiveProfile?.DisplayName ?? string.Empty; public static string InternalName => s_effectiveProfile?.InternalName?? string.Empty; @@ -27,7 +26,7 @@ public static class ProfilesLibrary public static bool MustAddChunks => s_effectiveProfile?.MustAddChunks ?? false; public static bool EnableExecution => s_effectiveProfile?.EnableExecution ?? false; public static bool HasAntiCheat => s_effectiveProfile?.HasAntiCheat ?? false; - + public static CompressionType EbxCompression => (CompressionType)(s_effectiveProfile?.EbxCompression ?? 0); public static CompressionType ResCompression => (CompressionType)(s_effectiveProfile?.ResCompression ?? 0); public static CompressionType ChunkCompression => (CompressionType)(s_effectiveProfile?.ChunkCompression ?? 0); @@ -47,7 +46,7 @@ public static class ProfilesLibrary private static Profile? s_effectiveProfile; private static bool s_profilesLoaded; private static readonly List s_profiles = new(); - + public static void Initialize() { if (Directory.Exists("Profiles")) @@ -84,7 +83,7 @@ public static bool Initialize(string profileKey) { foreach (string bundle in s_effectiveProfile.SharedBundles) { - SharedBundles.Add(HashString(bundle), bundle); + SharedBundles.Add(Utils.Utils.HashString(bundle, true), bundle); } IsInitialized = true;