diff --git a/FrostySdk/Interfaces/IFileInfo.cs b/FrostySdk/Interfaces/IFileInfo.cs index f0f598606..f06ab1ca9 100644 --- a/FrostySdk/Interfaces/IFileInfo.cs +++ b/FrostySdk/Interfaces/IFileInfo.cs @@ -9,10 +9,10 @@ namespace Frosty.Sdk.Interfaces; public interface IFileInfo { public bool IsComplete(); - + public Block GetRawData(); - - public Block GetData(int inOriginalSize = 0); + + public Block GetData(int inOriginalSize); protected void SerializeInternal(DataStream stream); @@ -26,6 +26,9 @@ public static void Serialize(DataStream stream, IFileInfo fileInfo) case KelvinFileInfo: stream.WriteByte(1); break; + case NonCasFileInfo: + stream.WriteByte(2); + break; default: throw new NotImplementedException(); } @@ -41,6 +44,8 @@ public static IFileInfo Deserialize(DataStream stream) return CasFileInfo.DeserializeInternal(stream); case 1: return KelvinFileInfo.DeserializeInternal(stream); + case 2: + return NonCasFileInfo.DeserializeInternal(stream); default: throw new InvalidDataException(); } diff --git a/FrostySdk/Managers/Infos/FileInfos/NonCasFileInfo.cs b/FrostySdk/Managers/Infos/FileInfos/NonCasFileInfo.cs index 831719d6f..fda9a28aa 100644 --- a/FrostySdk/Managers/Infos/FileInfos/NonCasFileInfo.cs +++ b/FrostySdk/Managers/Infos/FileInfos/NonCasFileInfo.cs @@ -1,3 +1,4 @@ +using System.IO; using Frosty.Sdk.Interfaces; using Frosty.Sdk.IO; using Frosty.Sdk.Utils; @@ -6,28 +7,56 @@ namespace Frosty.Sdk.Managers.Infos.FileInfos; public class NonCasFileInfo : IFileInfo { - public NonCasFileInfo(string inSuperBundleName, uint offset, uint size, uint logicalOffset = 0) + private string m_superBundleName; + private uint m_offset; + private uint m_size; + private uint m_logicalOffset; + + public NonCasFileInfo(string inSuperBundleName, uint inOffset, uint inSize, uint inLogicalOffset = 0) { - + m_superBundleName = inSuperBundleName; + m_offset = inOffset; + m_size = inSize; + m_logicalOffset = inLogicalOffset; } - + public bool IsComplete() { - throw new System.NotImplementedException(); + return m_logicalOffset == 0; } public Block GetRawData() { - throw new System.NotImplementedException(); + using (FileStream stream = new(m_superBundleName, FileMode.Open, FileAccess.Read)) + { + stream.Position = m_offset; + + Block retVal = new((int)m_size); + + stream.ReadExactly(retVal); + return retVal; + } } - public Block GetData(int inOriginalSize = 0) + public Block GetData(int inOriginalSize) { - throw new System.NotImplementedException(); + using (BlockStream stream = BlockStream.FromFile(m_superBundleName, m_offset, (int)m_size)) + { + return Cas.DecompressData(stream, inOriginalSize); + } } public void SerializeInternal(DataStream stream) { - throw new System.NotImplementedException(); + stream.WriteNullTerminatedString(m_superBundleName); + stream.WriteUInt32(m_offset); + stream.WriteUInt32(m_size); + stream.WriteUInt32(m_logicalOffset); + } + + public static NonCasFileInfo DeserializeInternal(DataStream stream) + { + return new NonCasFileInfo(stream.ReadNullTerminatedString(), stream.ReadUInt32(), stream.ReadUInt32(), + stream.ReadUInt32()); } } \ No newline at end of file diff --git a/FrostySdk/Managers/Loaders/Dynamic2018AssetLoader.cs b/FrostySdk/Managers/Loaders/Dynamic2018AssetLoader.cs index 0d6d123f3..5abb28029 100644 --- a/FrostySdk/Managers/Loaders/Dynamic2018AssetLoader.cs +++ b/FrostySdk/Managers/Loaders/Dynamic2018AssetLoader.cs @@ -26,7 +26,7 @@ public BundleHelper(string inName, long inOffset, long inSize) Size = inSize; } } - + public void Load() { foreach (SuperBundleInfo sbInfo in FileSystemManager.EnumerateSuperBundles()) @@ -47,6 +47,8 @@ public void Load() found = true; break; } + + break; } if (!found) @@ -73,14 +75,14 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk Debug.Assert(false, "We should not be here"); return Code.NotFound; } - + // check for format flags bool isCas = toc.AsBoolean("cas"); bool isDas = toc.AsBoolean("das"); // path to sb file string sbPath = path.Replace(".toc", ".sb"); - + // load bundles if (toc.ContainsKey("bundles")) { @@ -88,8 +90,8 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk DataStream sbStream = BlockStream.FromFile(sbPath, false); DataStream? baseSbStream = null; Dictionary? baseBundleMapping = null; - - // is its a das superBundle it stores the bundle values in lists + + // is its a das superBundle it stores the bundle values in lists if (isDas) { DbObjectDict bundles = toc.AsDict("bundles"); @@ -114,7 +116,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk long offset = bundleObj.AsLong("offset"); long size = bundleObj.AsLong("size"); - + // legacy flags used until fb 2014.4.11 // cas + delta -> casPatchType for bundle members // noncas + delta -> patched bundle and bundle members @@ -126,7 +128,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk { baseSbStream ??= BlockStream.FromFile(FileSystemManager.ResolvePath(false, $"{inSbIc.Name}.sb"), false); - + LoadBundle(baseSbStream, offset, size, ref bundle, !isCas); } else if (!isCas && isDelta) @@ -135,7 +137,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk // we need to load the base toc to get the corresponding base bundle baseBundleMapping ??= LoadBaseBundles(FileSystemManager.ResolvePath(false, $"{inSbIc.Name}.toc")); - if (baseBundleMapping.TryGetValue(Utils.Utils.HashString(bundle.Name), out BundleHelper helper)) + if (baseBundleMapping.TryGetValue(Utils.Utils.HashString(bundle.Name, true), out BundleHelper helper)) { baseSbStream ??= BlockStream.FromFile(FileSystemManager.ResolvePath(false, $"{inSbIc.Name}.sb"), false); @@ -149,6 +151,9 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk } } } + + sbStream.Dispose(); + baseSbStream?.Dispose(); } // load chunks @@ -166,7 +171,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk baseChunks.Add(obj.AsDict().AsGuid("id")); } } - + foreach (DbObject obj in toc.AsList("chunks")) { DbObjectDict chunkObj = obj.AsDict(); @@ -187,7 +192,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk } bool b = baseChunks.Remove(entry.Id); - + AssetManager.AddSuperBundleChunk(entry); if (entry.LogicalSize == 0) @@ -196,7 +201,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk // entry.OriginalSize = entry.FileInfo.GetOriginalSize(); } } - + Debug.Assert(baseChunks.Count == 0); } @@ -204,7 +209,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk { /* these are never actually used, tho the newer games check for them if (toc.ContainsKey("removedBundles")) - { + { } if (toc.ContainsKey("removedChunks")) @@ -213,7 +218,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk return Code.Continue; } - + return Code.Stop; } @@ -225,7 +230,7 @@ private Dictionary LoadBaseBundles(string inPath) { return retVal; } - + DbObjectDict? toc = DbObject.Deserialize(inPath)?.AsDict(); if (toc is null || !toc.ContainsKey("bundles")) @@ -236,7 +241,7 @@ private Dictionary LoadBaseBundles(string inPath) foreach (DbObject obj in toc.AsList("bundles")) { string name = obj.AsDict().AsString("id"); - retVal.Add(Utils.Utils.HashString(name), new BundleHelper(name, obj.AsDict().AsLong("offset"), obj.AsDict().AsLong("size"))); + retVal.Add(Utils.Utils.HashString(name, true), new BundleHelper(name, obj.AsDict().AsLong("offset"), obj.AsDict().AsLong("size"))); } return retVal; @@ -257,7 +262,7 @@ private void LoadDeltaBundle(DataStream deltaStream, long inDeltaOffset, long in } BinaryBundle bundleMeta = DeserializeDeltaBundle(deltaStream, baseStream); - + // TODO: get asset refs from sb file similar to this (https://github.com/GreyDynamics/Frostbite3_Editor/blob/develop/src/tk/greydynamics/Resource/Frostbite3/Cas/NonCasBundle.java) // or with a cache like before // this is just so u can load those games for now @@ -275,7 +280,7 @@ private void LoadDeltaBundle(DataStream deltaStream, long inDeltaOffset, long in { AssetManager.AddChunk(chunk, bundle.Id); } - + // disable for now since we dont read the data after the bundle // Debug.Assert(deltaStream.Position == inDeltaOffset + inDeltaSize, "Didnt read delta bundle correctly."); // Debug.Assert((baseStream?.Position ?? 0) == inBaseOffset + inBaseSize, "Didnt read base bundle correctly."); @@ -330,7 +335,7 @@ private static void LoadNonCasBundle(DataStream stream, BundleInfo bundle) AssetManager.AddChunk(chunk, bundle.Id); } } - + private static void LoadCasBundle(DataStream stream, BundleInfo bundle, bool isDelta) { DbObjectDict? bundleObj = DbObject.Deserialize(stream)?.AsDict(); @@ -362,7 +367,7 @@ private static void LoadCasBundle(DataStream stream, BundleInfo bundle, bool isD IEnumerable? fileInfos = ResourceManager.GetPatchFileInfos(entry.Sha1, deltaSha1, baseSha1); - + if (fileInfos is not null) { entry.FileInfos.UnionWith(fileInfos); @@ -445,7 +450,7 @@ private static void LoadCasBundle(DataStream stream, BundleInfo bundle, bool isD AssetManager.AddChunk(entry, bundle.Id); } } - + private BinaryBundle DeserializeDeltaBundle(DataStream deltaStream, DataStream? baseStream) { ulong magic = deltaStream.ReadUInt64(); @@ -456,17 +461,17 @@ private BinaryBundle DeserializeDeltaBundle(DataStream deltaStream, DataStream? uint bundleSize = deltaStream.ReadUInt32(Endian.Big); deltaStream.ReadUInt32(Endian.Big); // size of data after binary bundle - + long startOffset = deltaStream.Position; int patchedBundleSize = deltaStream.ReadInt32(Endian.Big); uint baseBundleSize = baseStream?.ReadUInt32(Endian.Big) ?? 0; long baseBundleOffset = baseStream?.Position ?? -1; - + using (BlockStream stream = new(patchedBundleSize + 4)) { stream.WriteInt32(patchedBundleSize, Endian.Big); - + while (deltaStream.Position < bundleSize + startOffset) { uint packed = deltaStream.ReadUInt32(Endian.Big); @@ -494,7 +499,7 @@ private BinaryBundle DeserializeDeltaBundle(DataStream deltaStream, DataStream? { baseStream.Position = baseBundleOffset + baseBundleSize; } - + stream.Position = 0; return BinaryBundle.Deserialize(stream); } diff --git a/FrostySdk/Managers/Loaders/KelvinAssetLoader.cs b/FrostySdk/Managers/Loaders/KelvinAssetLoader.cs index 1ece7ba8a..c014a98ce 100644 --- a/FrostySdk/Managers/Loaders/KelvinAssetLoader.cs +++ b/FrostySdk/Managers/Loaders/KelvinAssetLoader.cs @@ -27,7 +27,7 @@ public FileIdentifier(int inFileIndex, uint inOffset, uint inSize) Size = inSize; } } - + public void Load() { foreach (SuperBundleInfo sbInfo in FileSystemManager.EnumerateSuperBundles()) @@ -48,6 +48,8 @@ public void Load() found = true; break; } + + break; } if (!found) @@ -57,7 +59,7 @@ public void Load() } } } - + private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk inSbIc) { if (!inSource.TryResolvePath($"{inSbIc.Name}.toc", out string? path)) @@ -70,29 +72,29 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk uint magic = stream.ReadUInt32(); uint bundlesOffset = stream.ReadUInt32(); uint chunksOffset = stream.ReadUInt32(); - + if (magic == 0xC3E5D5C3) { stream.Decrypt(KeyManager.GetKey("BundleEncryptionKey"), PaddingMode.None); } - + if (bundlesOffset != 0xFFFFFFFF) { stream.Position = bundlesOffset; - + int bundleCount = stream.ReadInt32(); - + // bundle hashmap stream.Position += sizeof(int) * bundleCount; - + for (int i = 0; i < bundleCount; i++) { uint bundleOffset = stream.ReadUInt32(); - + long curPos = stream.Position; - + stream.Position = bundleOffset; - + string name = ReadString(stream, stream.ReadInt32()); List files = new(); @@ -101,32 +103,32 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk int file = stream.ReadInt32(); uint fileOffset = stream.ReadUInt32(); uint fileSize = stream.ReadUInt32(); - + files.Add(new FileIdentifier(file & 0x7FFFFFFF, fileOffset, fileSize)); if ((file & 0x80000000) == 0) { break; } } - + stream.Position = curPos; - + if (inSbIc.BundleMapping.ContainsKey(name)) { continue; } - + BundleInfo bundle = AssetManager.AddBundle(name, inSbIc); - + int index = 0; FileIdentifier resourceInfo = files[index]; BlockStream dataStream = BlockStream.FromFile( FileSystemManager.ResolvePath(FileSystemManager.GetFilePath(resourceInfo.FileIndex)), resourceInfo.Offset, (int)resourceInfo.Size); - + BinaryBundle bundleMeta = BinaryBundle.Deserialize(dataStream); - + foreach (EbxAssetEntry ebx in bundleMeta.EbxList) { if (dataStream.Position == resourceInfo.Size) @@ -137,13 +139,13 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk FileSystemManager.ResolvePath(FileSystemManager.GetFilePath(resourceInfo.FileIndex)), resourceInfo.Offset, (int)resourceInfo.Size); } - + uint offset = (uint)dataStream.Position; uint size = (uint)Helper.GetSize(dataStream, ebx.OriginalSize); ebx.FileInfos.Add(new KelvinFileInfo(resourceInfo.FileIndex, resourceInfo.Offset + offset, size, 0)); - + AssetManager.AddEbx(ebx, bundle.Id); } foreach (ResAssetEntry res in bundleMeta.ResList) @@ -156,13 +158,13 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk FileSystemManager.ResolvePath(FileSystemManager.GetFilePath(resourceInfo.FileIndex)), resourceInfo.Offset, (int)resourceInfo.Size); } - + uint offset = (uint)dataStream.Position; uint size = (uint)Helper.GetSize(dataStream, res.OriginalSize); res.FileInfos.Add(new KelvinFileInfo(resourceInfo.FileIndex, resourceInfo.Offset + offset, size, 0)); - + AssetManager.AddRes(res, bundle.Id); } foreach (ChunkAssetEntry chunk in bundleMeta.ChunkList) @@ -182,40 +184,40 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk chunk.FileInfos.Add(new KelvinFileInfo(resourceInfo.FileIndex, resourceInfo.Offset + offset, size, chunk.LogicalOffset)); - + AssetManager.AddChunk(chunk, bundle.Id); } - + dataStream.Dispose(); } } - + if (chunksOffset != 0xFFFFFFFF) { stream.Position = chunksOffset; int chunksCount = stream.ReadInt32(); - + // hashmap stream.Position += sizeof(int) * chunksCount; - + for (int i = 0; i < chunksCount; i++) { int offset = stream.ReadInt32(); - + long pos = stream.Position; stream.Position = offset; - + Guid guid = stream.ReadGuid(); int fileIndex = stream.ReadInt32(); uint dataOffset = stream.ReadUInt32(); uint dataSize = stream.ReadUInt32(); - + ChunkAssetEntry chunk = new(guid, Sha1.Zero, 0, 0, Utils.Utils.HashString(inSbIc.Name, true)); - + chunk.FileInfos.Add(new KelvinFileInfo(fileIndex, dataOffset, dataSize, 0)); - + AssetManager.AddSuperBundleChunk(chunk); - + stream.Position = pos; } } @@ -223,7 +225,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk return Code.Stop; } - + private string ReadString(DataStream reader, int offset) { long curPos = reader.Position; diff --git a/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs b/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs index 7a5a3471c..a68dd0375 100644 --- a/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs +++ b/FrostySdk/Managers/Loaders/Manifest2019AssetLoader.cs @@ -31,7 +31,7 @@ private enum Flags private readonly HashSet m_removedChunks = new(); private readonly HashSet m_removedBundles = new(); - + public void Load() { foreach (SuperBundleInfo sbInfo in FileSystemManager.EnumerateSuperBundles()) @@ -54,6 +54,8 @@ public void Load() found = true; break; } + + break; } if (!found) @@ -70,7 +72,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk { return Code.NotFound; } - + using (BlockStream stream = BlockStream.FromFile(path, true)) { stream.Position += sizeof(uint); // bundleHashMapOffset @@ -109,7 +111,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk { // stream for loading from sb file DataStream? sbStream = null; - + if (flags.HasFlag(Flags.HasCompressedNames)) { stream.Position = namesOffset; @@ -118,7 +120,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk stream.Position = tableOffset; huffmanDecoder.ReadHuffmanTable(stream, tableCount, Endian.Big); } - + stream.Position = bundleDataOffset; for (int i = 0; i < bundlesCount; i++) @@ -149,7 +151,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk } BundleInfo bundle = AssetManager.AddBundle(name, inSbIc); - + // load bundle byte bundleLoadFlag = (byte)(bundleSize >> 30); bundleSize &= 0x3FFFFFFFU; @@ -175,7 +177,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk stream.Position = chunkDataOffset; Block chunkData = new(dataCount); stream.ReadExactly(chunkData.ToBlock()); - + stream.Position = chunkGuidOffset; Span b = stackalloc byte[16]; for (int i = 0; i < chunksCount; i++) @@ -200,7 +202,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk } byte fileIdentifierFlag = (byte)(index >> 24); - + index &= 0x00FFFFFF; CasFileIdentifier casFileIdentifier; @@ -226,7 +228,7 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk AssetManager.AddSuperBundleChunk(chunk); } - + chunkData.Dispose(); } @@ -238,13 +240,13 @@ private Code LoadSuperBundle(FileSystemSource inSource, SuperBundleInstallChunk return Code.Stop; } - + private void LoadBundle(DataStream stream, long inOffset, uint inSize, ref BundleInfo bundle) { long curPos = stream.Position; stream.Position = inOffset; - + int bundleOffset = stream.ReadInt32(Endian.Big); int bundleSize = stream.ReadInt32(Endian.Big); uint locationOffset = stream.ReadUInt32(Endian.Big); @@ -265,13 +267,13 @@ private void LoadBundle(DataStream stream, long inOffset, uint inSize, ref Bundl Block fileIdentifierFlags = new Block(totalCount); stream.ReadExactly(fileIdentifierFlags); - + // the flags should be the last thing in the bundle Debug.Assert(stream.Position == inOffset + inSize, "Didnt read bundle correctly."); CasFileIdentifier file = default; int currentIndex = 0; - + // load the bundle meta BinaryBundle bundleMeta; if (inlineBundle) @@ -279,7 +281,7 @@ private void LoadBundle(DataStream stream, long inOffset, uint inSize, ref Bundl stream.Position = inOffset + bundleOffset; bundleMeta = BinaryBundle.Deserialize(stream); Debug.Assert(stream.Position == inOffset + bundleOffset + bundleSize, "We did not read the bundle meta completely"); - + // go to the start of the data stream.Position = inOffset + dataOffset; } @@ -297,41 +299,41 @@ private void LoadBundle(DataStream stream, long inOffset, uint inSize, ref Bundl using (BlockStream bundleStream = BlockStream.FromFile(path, offset, size)) { bundleMeta = BinaryBundle.Deserialize(bundleStream); - + Debug.Assert(bundleStream.Position == bundleStream.Length, "We did not read the bundle meta completely"); } } - + // load assets from bundle foreach (EbxAssetEntry ebx in bundleMeta.EbxList) { file = ReadCasFileIdentifier(stream, fileIdentifierFlags[currentIndex++], file); - + ebx.FileInfos.Add(new CasFileInfo(file, stream.ReadUInt32(Endian.Big), stream.ReadUInt32(Endian.Big), 0)); - + AssetManager.AddEbx(ebx, bundle.Id); } foreach (ResAssetEntry res in bundleMeta.ResList) { file = ReadCasFileIdentifier(stream, fileIdentifierFlags[currentIndex++], file); - + res.FileInfos.Add(new CasFileInfo(file, stream.ReadUInt32(Endian.Big), stream.ReadUInt32(Endian.Big), 0)); - + AssetManager.AddRes(res, bundle.Id); } foreach (ChunkAssetEntry chunk in bundleMeta.ChunkList) { file = ReadCasFileIdentifier(stream, fileIdentifierFlags[currentIndex++], file); - + chunk.FileInfos.Add(new CasFileInfo(file, stream.ReadUInt32(Endian.Big), stream.ReadUInt32(Endian.Big), chunk.LogicalOffset)); - + AssetManager.AddChunk(chunk, bundle.Id); } fileIdentifierFlags.Dispose(); - + stream.Position = curPos; }