From 7f04304b3b463ad1ed46bb72b4d6662c8798b16f Mon Sep 17 00:00:00 2001 From: Justin Wei Date: Tue, 20 Sep 2022 08:02:35 +0000 Subject: [PATCH 1/2] Merge branch 'jw/fix-usage' into 'develop' Fix RemoveUninstantiables's parsing of usage.txt to deobfuscate See merge request mobile_perf_infra/Redex!158 --- libredex/ConfigFiles.cpp | 5 + libredex/ConfigFiles.h | 2 + .../RemoveUninstantiablesPass.cpp | 166 +++++++++++++++++- .../RemoveUninstantiablesPass.h | 42 +++++ test/unit/RemoveUninstantiablesTest.cpp | 125 ++++++++++++- 5 files changed, 333 insertions(+), 7 deletions(-) diff --git a/libredex/ConfigFiles.cpp b/libredex/ConfigFiles.cpp index c8dfdda19a..626620f91c 100644 --- a/libredex/ConfigFiles.cpp +++ b/libredex/ConfigFiles.cpp @@ -49,6 +49,11 @@ ConfigFiles::ConfigFiles(const Json::Value& config, const std::string& outdir) m_instruction_size_bitwidth_limit = instruction_size_bitwidth_limit; } +ConfigFiles::ConfigFiles(const Json::Value& config, std::istream& proguard_input) : ConfigFiles(config, "") { + m_proguard_map = std::make_unique(proguard_input); + m_method_profiles = std::make_unique(*m_proguard_map); +} + ConfigFiles::ConfigFiles(const Json::Value& config) : ConfigFiles(config, "") {} ConfigFiles::~ConfigFiles() { diff --git a/libredex/ConfigFiles.h b/libredex/ConfigFiles.h index 29f1da6b06..eeaf237711 100644 --- a/libredex/ConfigFiles.h +++ b/libredex/ConfigFiles.h @@ -46,6 +46,8 @@ class MethodProfiles; */ struct ConfigFiles { explicit ConfigFiles(const Json::Value& config); + // For test purposes to inject a ProguardMap without a stand-alone file + explicit ConfigFiles(const Json::Value& config, std::istream& proguard_input); ConfigFiles(const Json::Value& config, const std::string& outdir); ~ConfigFiles(); diff --git a/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp b/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp index 52eccb397d..9195dd34ae 100644 --- a/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp +++ b/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp @@ -22,6 +22,10 @@ #include "Walkers.h" #include +#include + +std::string RemoveUninstantiablesPass::m_proguard_usage_name = ""; +std::istream* RemoveUninstantiablesPass::test_only_usage_file_input = nullptr; namespace { @@ -280,6 +284,126 @@ void RemoveUninstantiablesPass::Stats::report(PassManager& mgr) const { #undef REPORT } +// Map deobfuscated type name in internal format (La/b/MyClass;) to the obfuscated DexType*. +std::unordered_map make_deobfuscated_map(const std::unordered_set& obfuscated_types) { + std::unordered_map deobfuscated_to_type; + for (auto obfuscated_type : obfuscated_types) { + auto obfuscated_cls = type_class(obfuscated_type); + auto deobfuscated_name = obfuscated_cls->get_deobfuscated_name().str(); + TRACE( + RMUNINST, 4, "Mapping deobfuscated name to obfuscated type: '%s' -> '%s'", + deobfuscated_name.c_str(), SHOW(obfuscated_type)); + if (deobfuscated_to_type.count(deobfuscated_name) > 0) { + always_assert_log( + deobfuscated_to_type.count(deobfuscated_name), + "Multiple uninstantiable types with same deobfuscated name! '%s' == '%s' && '%s'", + deobfuscated_name.c_str(), + SHOW(obfuscated_type), + SHOW(deobfuscated_to_type.at(deobfuscated_name)) + ); + } + deobfuscated_to_type.emplace(deobfuscated_name, obfuscated_type); + } + return deobfuscated_to_type; +} + +// Handles a single line of usage.txt proguard file from `-printusage`. +// Focuses on looking for removed constructors for instantiability. +// +// Typical usage.txt contents: +// ---- +// androidx.core.view.ViewKt$postOnAnimationDelayed$runnable$1 +// androidx.core.view.ViewParentCompat: +// private static int[] sTempNestedScrollConsumed +// 41:41:private ViewParentCompat() +// ---- +// A classname on its own line means the whole class was removed and can be +// ignored. +// If the classname is followed by a ":", then what follows is a 4-space +// indented list of parts that were removed. The list continues until we find a +// de-indent which means we once again have a class name. +// We specifically look for removed constructors by doing a name check for a +// removed method that is named the name of the class. All names are +// unobfuscated (pre-proguard) and need to be obfuscated to find relevant redex +// classes. +void UsageHandler::handle_usage_line( + const std::string& line, + const std::unordered_map& deobfuscated_to_uninstantiable_type, + std::unordered_set& uninstantiable_types) { + TRACE(RMUNINST, 5, "usage.txt line: %s", line.c_str()); + // The current line is a class + if (line.find(" ") != 0) { + int dot_index = line.find_last_of('.'); + int end_index = line.find_first_of(':'); + cls_name = line.substr(dot_index + 1, end_index - dot_index - 1); + // usage lines use external naming (a.b.MyClass) and we need to translate + // to internal naming (La/b/MyClass;) to match type names + type_name = convert_type(line.substr(0, end_index)); + usage_type = DexType::get_type(type_name); + + TRACE(RMUNINST, 5, "usage.txt line type (deobfuscated) name: '%s'", type_name.c_str()); + if (usage_type != nullptr) { + TRACE(RMUNINST, 5, "usage.txt line type (deobfuscated): '%s'", SHOW(usage_type)); + } else { + TRACE(RMUNINST, 5, "usage.txt line type (deobfuscated) does not exist!"); + } + + if (deobfuscated_to_uninstantiable_type.count(type_name) <= 0) { + TRACE(RMUNINST, 5, "usage.txt type has no obfuscated version: %s", type_name.c_str()); + cls_type = usage_type; + } else { + cls_type = deobfuscated_to_uninstantiable_type.at(type_name); + TRACE(RMUNINST, 5, "usage.txt obfuscated type: '%s' -> '%s'", SHOW(cls_type), type_name.c_str()); + } + if (uninstantiable_types.count(cls_type)) { + has_ctor = false; + } else { + has_ctor = true; + } + } else { + if (has_ctor) { + return; + } + // The current line is a constructor method + if (line.find(cls_name + "(") != std::string::npos) { + has_ctor = true; + if (uninstantiable_types.count(cls_type)) { + TRACE( + RMUNINST, 4, + "Keeping class due to proguard-removed constructor: %s", + SHOW(cls_type)); + count++; + } else { + TRACE( + RMUNINST, 4, "Keep class not present! Skipping: %s", + SHOW(cls_type)); + } + uninstantiable_types.erase(cls_type); + } + } +} + +// Prune the list of possibly uninstantiable types by looking at the +// constructor that Proguard deleted. +void RemoveUninstantiablesPass::readUsage(std::istream& usage_file, std::unordered_set& uninstantiable_types) { + // uninstantiable_types may have obfuscated names. Deobfuscate for usage.txt. + std::unordered_map deobfuscated_to_uninstantiable_type = + make_deobfuscated_map(uninstantiable_types); + TRACE( + RMUNINST, 1, "Mapped %lu deobfuscated<->obfuscated types", + deobfuscated_to_uninstantiable_type.size()); + + UsageHandler uh; + std::string line; + while(getline(usage_file, line)) { + uh.handle_usage_line(line, deobfuscated_to_uninstantiable_type, uninstantiable_types); + } + TRACE( + RMUNINST, 1, + "Removed %u types from uninstantiable_types by reading proguard file", + uh.count); +} + // Computes set of uninstantiable types, also looking at the type system to // find non-external (and non-native)... // - interfaces that are not annotations, are not root (or unrenameable) and @@ -317,13 +441,24 @@ RemoveUninstantiablesPass::compute_scoped_uninstantiable_types( instantiable_classes.insert(cls); } }); + + if(RemoveUninstantiablesPass::m_proguard_usage_name.compare("") != 0) { + std::ifstream infile; + infile.open(RemoveUninstantiablesPass::m_proguard_usage_name.data()); + always_assert(infile.is_open()); + readUsage(infile, uninstantiable_types); + infile.close(); + } else if (RemoveUninstantiablesPass::test_only_usage_file_input != nullptr) { + readUsage(*RemoveUninstantiablesPass::test_only_usage_file_input, uninstantiable_types); + } + // Next, we prune the list of possibly uninstantiable types by looking at // what instantiable classes implement and extend. std::unordered_set visited; - std::function visit; + std::function visit; visit = [&](const DexClass* cls) { if (cls == nullptr || !visited.insert(cls).second) { - return false; + return; } if (instantiable_children) { (*instantiable_children)[cls->get_super_class()].insert(cls->get_type()); @@ -332,13 +467,17 @@ RemoveUninstantiablesPass::compute_scoped_uninstantiable_types( for (auto interface : *cls->get_interfaces()) { visit(type_class(interface)); } - return true; + visit(type_class(cls->get_super_class())); }; + + size_t count = uninstantiable_types.size(); for (auto cls : instantiable_classes) { - while (visit(cls)) { - cls = type_class(cls->get_super_class()); - } + visit(cls); } + + TRACE( + RMUNINST, 1, "Removed %lu classes by pruning parents/interfaces", + count - uninstantiable_types.size()); uninstantiable_types.insert(type::java_lang_Void()); return uninstantiable_types; } @@ -445,6 +584,21 @@ void RemoveUninstantiablesPass::run_pass(DexStoresVector& stores, instantiable_children; std::unordered_set scoped_uninstantiable_types = compute_scoped_uninstantiable_types(scope, &instantiable_children); + TRACE(RMUNINST, 2, "Total instantiable types: %lu", instantiable_children.size()); + TRACE(RMUNINST, 2, "Total uninstantiable types: %lu", scoped_uninstantiable_types.size()); + for (auto utype : scoped_uninstantiable_types) { + TRACE( + RMUNINST, 5, "uninstantiable class: %s -> %s", + SHOW(utype), type_class(utype)->get_deobfuscated_name_or_empty().c_str()); + } + for (auto itype : instantiable_children) { + if (itype.first == nullptr) { + continue; + } + TRACE( + RMUNINST, 5, "instantible class: %s -> %s", + SHOW(itype.first), type_class(itype.first)->get_deobfuscated_name_or_empty().c_str()); + } OverriddenVirtualScopesAnalysis overridden_virtual_scopes_analysis( scope, scoped_uninstantiable_types, instantiable_children); // We perform structural changes, i.e. whether a method has a body and diff --git a/opt/remove-uninstantiables/RemoveUninstantiablesPass.h b/opt/remove-uninstantiables/RemoveUninstantiablesPass.h index 19661b2049..dd91e7a152 100644 --- a/opt/remove-uninstantiables/RemoveUninstantiablesPass.h +++ b/opt/remove-uninstantiables/RemoveUninstantiablesPass.h @@ -14,6 +14,34 @@ namespace cfg { class ControlFlowGraph; } // namespace cfg +std::unordered_map make_deobfuscated_map(const std::unordered_set& obfuscated_types); + +class UsageHandler { + public: + std::string cls_name; + std::string type_name; + DexType* cls_type = nullptr; + DexType* usage_type = nullptr; + int count = 0; + bool has_ctor = false; + + void reset() { + cls_name = ""; + type_name = ""; + cls_type = nullptr; + usage_type = nullptr; + count = 0; + has_ctor = false; + }; + + UsageHandler() {}; + + void handle_usage_line( + const std::string& line, + const std::unordered_map& deobfuscated_uninstantiable_type, + std::unordered_set& uninstantiable_types); +}; + /// Looks for mentions of classes that have no constructors and use the fact /// they can't be instantiated to simplify those mentions: /// @@ -33,6 +61,10 @@ class ControlFlowGraph; /// `null`. class RemoveUninstantiablesPass : public Pass { public: + UsageHandler uh; + // Testing injector to mimic m_proguard_usage_name + static std::istream* test_only_usage_file_input; + RemoveUninstantiablesPass() : Pass("RemoveUninstantiablesPass") {} /// Counts of references to uninstantiable classes removed. @@ -60,6 +92,8 @@ class RemoveUninstantiablesPass : public Pass { std::unordered_map>* instantiable_children = nullptr); + static void readUsage(std::istream& usage_file, std::unordered_set& uninstantiable_types); + /// Look for mentions of uninstantiable classes in \p cfg and modify them /// in-place. static Stats replace_uninstantiable_refs( @@ -73,4 +107,12 @@ class RemoveUninstantiablesPass : public Pass { static Stats replace_all_with_throw(cfg::ControlFlowGraph& cfg); void run_pass(DexStoresVector&, ConfigFiles&, PassManager&) override; + + void bind_config() override { + // The Proguard usage.txt file name. Need to be added by gradle plugin. + bind("proguard_usage_name", "", m_proguard_usage_name); + } + + protected: + static std::string m_proguard_usage_name; }; diff --git a/test/unit/RemoveUninstantiablesTest.cpp b/test/unit/RemoveUninstantiablesTest.cpp index 4bfebb4469..f0bd16a352 100644 --- a/test/unit/RemoveUninstantiablesTest.cpp +++ b/test/unit/RemoveUninstantiablesTest.cpp @@ -4,7 +4,7 @@ * This source code is licensed under the MIT license found in the * LICENSE file in the root directory of this source tree. */ - +#include #include #include "Creators.h" @@ -762,4 +762,127 @@ TEST_F(RemoveUninstantiablesTest, RunPassInstantiableChildrenDefined) { EXPECT_EQ(0, rm_uninst->metrics.at("get_uninstantiables")); } +TEST_F(RemoveUninstantiablesTest, UsageHandlerMixedObfuscationUninstantiables_UsageInstantiablesRemoved) { + auto* obfuscated_instantiable = def_class("La/b/c$d;"); + auto* obfuscated_uninstantiable = def_class("Le/f/g;"); + auto* unobfuscated_instantiable = def_class("Lunobfuscated/but/instantiable;"); + auto* unobfuscated_uninstantiable = def_class("Lunobfuscated/and/uninstantiable;"); + + std::unordered_set uninstantiable_types; + uninstantiable_types.emplace(obfuscated_instantiable->get_type()); + uninstantiable_types.emplace(obfuscated_uninstantiable->get_type()); + uninstantiable_types.emplace(unobfuscated_instantiable->get_type()); + uninstantiable_types.emplace(unobfuscated_uninstantiable->get_type()); + + std::unordered_map deobfuscated_uninstantiable_type; + deobfuscated_uninstantiable_type.emplace("Ldeobfuscated/instantiable$name;", obfuscated_instantiable->get_type()); + deobfuscated_uninstantiable_type.emplace("Ldeobfuscated/uninstantiable$name;", obfuscated_uninstantiable->get_type()); + + UsageHandler uh; + + EXPECT_EQ(4, uninstantiable_types.size()); + uh.handle_usage_line("deobfuscated.instantiable$name:", deobfuscated_uninstantiable_type, uninstantiable_types); + uh.handle_usage_line(" public instantiable$name()", deobfuscated_uninstantiable_type, uninstantiable_types); + uh.handle_usage_line("deobfuscated.uninstantiable$name:", deobfuscated_uninstantiable_type, uninstantiable_types); + uh.handle_usage_line(" public shouldNotMakeInstantiable()", deobfuscated_uninstantiable_type, uninstantiable_types); + uh.handle_usage_line("unobfuscated.but.instantiable:", deobfuscated_uninstantiable_type, uninstantiable_types); + uh.handle_usage_line(" public instantiable()", deobfuscated_uninstantiable_type, uninstantiable_types); + uh.handle_usage_line("unobfuscated.and.uninstantiable:", deobfuscated_uninstantiable_type, uninstantiable_types); + uh.handle_usage_line(" public shouldNotMakeInstantiable()", deobfuscated_uninstantiable_type, uninstantiable_types); + EXPECT_EQ(2, uninstantiable_types.size()); + EXPECT_THAT( + uninstantiable_types, + testing::UnorderedElementsAre( + obfuscated_uninstantiable->get_type(), + unobfuscated_uninstantiable->get_type())); +} + +TEST_F(RemoveUninstantiablesTest, MakeDeobfuscatedMapMixedObfuscation_OnlyDeobfuscatedInMapping) { + DexStoresVector dss{DexStore{"test_store"}}; + + const char* const abcd_init = R"( + (method (private) "La/b/c$d;.:()V" + ((load-param-object v0) + (return-void))))"; + + auto* obfuscated_class = def_class("La/b/c$d;"); + obfuscated_class->set_deobfuscated_name("deobfuscated.cls$name"); + auto* raw_class = def_class("La/b/c;"); + dss.back().add_classes({obfuscated_class, raw_class}); + + std::unordered_set uninstantiable_types; + uninstantiable_types.emplace(obfuscated_class->get_type()); + std::unordered_map deobfuscated_uninstantiable_type = + make_deobfuscated_map(uninstantiable_types); + EXPECT_EQ(1, deobfuscated_uninstantiable_type.size()); + EXPECT_TRUE(deobfuscated_uninstantiable_type.count("deobfuscated.cls$name")); + EXPECT_EQ(deobfuscated_uninstantiable_type.at("deobfuscated.cls$name"), obfuscated_class->get_type()); +} + +TEST_F(RemoveUninstantiablesTest, RunPassUsageObfuscatedDefaultConstructor_KeepsInstantiable) { + DexStoresVector dss{DexStore{"test_store"}}; + + const char* const abcd_something = R"( +(method (public) "La/b/c$d;.something:()V" + ((load-param-object v0) + (return-void)) +))"; + auto* abcd_class = def_class("La/b/c$d;", abcd_something); + auto expected_code = assembler::to_string(DexMethod::get_method("La/b/c$d;.something:()V")->as_def()->get_code()); + + dss.back().add_classes({abcd_class}); + + RemoveUninstantiablesPass pass; + + std::string usage_input_str = +R"(deobfuscated.cls$name: + public cls$name())"; + std::basic_istringstream usage_input(usage_input_str); + RemoveUninstantiablesPass::test_only_usage_file_input = &usage_input; + + std::string proguard_input_str = +R"(deobfuscated.cls$name -> a.b.c$d : + int field -> f)"; + std::basic_istringstream proguard_input(proguard_input_str); + + ConfigFiles c(Json::nullValue, proguard_input); + + PassManager pm({&pass}); + // applying deobfuscation happens in main.cpp which seems like bad design + for (auto& store : dss) { + apply_deobfuscated_names(store.get_dexen(), c.get_proguard_map()); + } + ASSERT_EQ(1, dss.back().get_dexen().size()); + pm.run_passes(dss, c); + ASSERT_EQ(1, dss.back().get_dexen().size()); + ASSERT_EQ(1, dss.back().get_dexen().at(0).size()); + + EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_name()->c_str(), testing::StrEq("La/b/c$d;")); + EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_deobfuscated_name().c_str(), testing::StrEq("Ldeobfuscated/cls$name;")); + + ASSERT_EQ(1, dss.back().get_dexen().at(0).at(0)->get_all_methods().size()); + EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_all_methods().at(0)->get_name()->c_str(), testing::StrEq("something")); + + ASSERT_TRUE(dss.back().get_dexen().at(0).at(0)->get_all_methods().at(0)->get_code()); + EXPECT_THAT(expected_code, assembler::to_string(dss.back().get_dexen().at(0).at(0)->get_all_methods().at(0)->get_code())); + EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_vmethods(), testing::ContainerEq(dss.back().get_dexen().at(0).at(0)->get_all_methods())); + + const auto& pass_infos = pm.get_pass_info(); + auto rm_uninst = + std::find_if(pass_infos.begin(), pass_infos.end(), [](const auto& pi) { + return pi.pass->name() == "RemoveUninstantiablesPass"; + }); + ASSERT_NE(rm_uninst, pass_infos.end()); + + EXPECT_EQ(0, rm_uninst->metrics.at("instance_ofs")); + EXPECT_EQ(0, rm_uninst->metrics.at("invokes")); + EXPECT_EQ(0, rm_uninst->metrics.at("field_accesses_on_uninstantiable")); + EXPECT_EQ(0, rm_uninst->metrics.at("throw_null_methods")); + EXPECT_EQ(0, rm_uninst->metrics.at("abstracted_classes")); + EXPECT_EQ(0, rm_uninst->metrics.at("abstracted_vmethods")); + EXPECT_EQ(0, rm_uninst->metrics.at("removed_vmethods")); + EXPECT_EQ(0, rm_uninst->metrics.at("get_uninstantiables")); + EXPECT_EQ(0, rm_uninst->metrics.at("check_casts")); +} + } // namespace From 6295d8163b88048a94c33b3b9c2c5c9d5ec6f9be Mon Sep 17 00:00:00 2001 From: "justin.wei" Date: Wed, 16 Nov 2022 21:51:06 -0800 Subject: [PATCH 2/2] Fix merge issues --- libredex/ConfigFiles.cpp | 4 ++-- libredex/DexHasher.cpp | 2 +- libredex/MethodProfiles.cpp | 3 ++- libredex/MethodProfiles.h | 5 ++++- .../RemoveUninstantiablesPass.cpp | 19 ++++++++++--------- .../RemoveUninstantiablesPass.h | 4 ++-- test/unit/RemoveUninstantiablesTest.cpp | 14 ++++++++------ 7 files changed, 29 insertions(+), 22 deletions(-) diff --git a/libredex/ConfigFiles.cpp b/libredex/ConfigFiles.cpp index 626620f91c..0f3808be5a 100644 --- a/libredex/ConfigFiles.cpp +++ b/libredex/ConfigFiles.cpp @@ -31,8 +31,8 @@ ConfigFiles::ConfigFiles(const Json::Value& config, const std::string& outdir) new ProguardMap(config.get("proguard_map", "").asString(), config.get("use_new_rename_map", 0).asBool())), m_printseeds(config.get("printseeds", "").asString()), - m_method_profiles(new method_profiles::MethodProfiles()), - m_secondary_method_profiles(new method_profiles::MethodProfiles()) { + m_method_profiles(new method_profiles::MethodProfiles(*m_proguard_map)), + m_secondary_method_profiles(new method_profiles::MethodProfiles(*m_proguard_map)) { m_coldstart_class_filename = config.get("coldstart_classes", "").asString(); if (m_coldstart_class_filename.empty()) { diff --git a/libredex/DexHasher.cpp b/libredex/DexHasher.cpp index f13ec72d7c..cda26dc705 100644 --- a/libredex/DexHasher.cpp +++ b/libredex/DexHasher.cpp @@ -231,7 +231,7 @@ void Impl::hash(const IRCode* c) { void Impl::hash(const cfg::ControlFlowGraph& cfg) { hash(cfg.get_registers_size()); - hash(cfg.entry_block()->id()); + hash((uint64_t)cfg.entry_block()->id()); std::unordered_map mie_ids; std::unordered_map pos_ids; for (auto b : cfg.blocks()) { diff --git a/libredex/MethodProfiles.cpp b/libredex/MethodProfiles.cpp index 7fca6e4aeb..42e083dd41 100644 --- a/libredex/MethodProfiles.cpp +++ b/libredex/MethodProfiles.cpp @@ -198,7 +198,8 @@ std::optional MethodProfiles::parse_main_internal( case NAME: // We move the string to a unique_ptr, so that its location is pinned, and // the string_views of the mdt are defined. - result.ref_str = std::make_unique(cell); + result.ref_str = std::make_unique( + m_pg_map.translate_method(std::string(cell))); result.mdt = dex_member_refs::parse_method(*result.ref_str); result.ref = DexMethod::get_method(*result.mdt); diff --git a/libredex/MethodProfiles.h b/libredex/MethodProfiles.h index 89b400a68d..24b3b73475 100644 --- a/libredex/MethodProfiles.h +++ b/libredex/MethodProfiles.h @@ -11,6 +11,7 @@ #include #include "DexClass.h" +#include "ProguardMap.h" #include "Timer.h" #include "Trace.h" @@ -60,7 +61,8 @@ const std::string COLD_START = "ColdStart"; class MethodProfiles { public: - MethodProfiles() {} + MethodProfiles(): m_pg_map(std::move(ProguardMap())) {} + MethodProfiles(const ProguardMap& pg_map): m_pg_map(pg_map) {} void initialize(const std::vector& csv_filenames) { m_initialized = true; @@ -136,6 +138,7 @@ class MethodProfiles { private: static AccumulatingTimer s_process_unresolved_lines_timer; + const ProguardMap& m_pg_map; AllInteractions m_method_stats; // Resolution may fail because of renaming or generated methods. Store the // unresolved lines here (per interaction) so we can update after passes run diff --git a/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp b/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp index 9195dd34ae..0c6f64ad78 100644 --- a/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp +++ b/opt/remove-uninstantiables/RemoveUninstantiablesPass.cpp @@ -16,6 +16,7 @@ #include "MethodDedup.h" #include "NullPointerExceptionUtil.h" #include "PassManager.h" +#include "ProguardMap.h" #include "Resolver.h" #include "Show.h" #include "Trace.h" @@ -285,26 +286,26 @@ void RemoveUninstantiablesPass::Stats::report(PassManager& mgr) const { } // Map deobfuscated type name in internal format (La/b/MyClass;) to the obfuscated DexType*. -std::unordered_map make_deobfuscated_map(const std::unordered_set& obfuscated_types) { - std::unordered_map deobfuscated_to_type; +std::unordered_map make_deobfuscated_map(const std::unordered_set& obfuscated_types) { + std::unordered_map deobfuscated_to_type; for (auto obfuscated_type : obfuscated_types) { auto obfuscated_cls = type_class(obfuscated_type); auto deobfuscated_name = obfuscated_cls->get_deobfuscated_name().str(); TRACE( RMUNINST, 4, "Mapping deobfuscated name to obfuscated type: '%s' -> '%s'", - deobfuscated_name.c_str(), SHOW(obfuscated_type)); + deobfuscated_name.data(), SHOW(obfuscated_type)); if (deobfuscated_to_type.count(deobfuscated_name) > 0) { always_assert_log( deobfuscated_to_type.count(deobfuscated_name), "Multiple uninstantiable types with same deobfuscated name! '%s' == '%s' && '%s'", - deobfuscated_name.c_str(), + deobfuscated_name.data(), SHOW(obfuscated_type), SHOW(deobfuscated_to_type.at(deobfuscated_name)) ); } deobfuscated_to_type.emplace(deobfuscated_name, obfuscated_type); } - return deobfuscated_to_type; + return std::move(deobfuscated_to_type); } // Handles a single line of usage.txt proguard file from `-printusage`. @@ -328,7 +329,7 @@ std::unordered_map make_deobfuscated_map(const std::unord // classes. void UsageHandler::handle_usage_line( const std::string& line, - const std::unordered_map& deobfuscated_to_uninstantiable_type, + const std::unordered_map& deobfuscated_to_uninstantiable_type, std::unordered_set& uninstantiable_types) { TRACE(RMUNINST, 5, "usage.txt line: %s", line.c_str()); // The current line is a class @@ -387,7 +388,7 @@ void UsageHandler::handle_usage_line( // constructor that Proguard deleted. void RemoveUninstantiablesPass::readUsage(std::istream& usage_file, std::unordered_set& uninstantiable_types) { // uninstantiable_types may have obfuscated names. Deobfuscate for usage.txt. - std::unordered_map deobfuscated_to_uninstantiable_type = + std::unordered_map deobfuscated_to_uninstantiable_type = make_deobfuscated_map(uninstantiable_types); TRACE( RMUNINST, 1, "Mapped %lu deobfuscated<->obfuscated types", @@ -589,7 +590,7 @@ void RemoveUninstantiablesPass::run_pass(DexStoresVector& stores, for (auto utype : scoped_uninstantiable_types) { TRACE( RMUNINST, 5, "uninstantiable class: %s -> %s", - SHOW(utype), type_class(utype)->get_deobfuscated_name_or_empty().c_str()); + SHOW(utype), type_class(utype)->get_deobfuscated_name_or_empty().data()); } for (auto itype : instantiable_children) { if (itype.first == nullptr) { @@ -597,7 +598,7 @@ void RemoveUninstantiablesPass::run_pass(DexStoresVector& stores, } TRACE( RMUNINST, 5, "instantible class: %s -> %s", - SHOW(itype.first), type_class(itype.first)->get_deobfuscated_name_or_empty().c_str()); + SHOW(itype.first), type_class(itype.first)->get_deobfuscated_name_or_empty().data()); } OverriddenVirtualScopesAnalysis overridden_virtual_scopes_analysis( scope, scoped_uninstantiable_types, instantiable_children); diff --git a/opt/remove-uninstantiables/RemoveUninstantiablesPass.h b/opt/remove-uninstantiables/RemoveUninstantiablesPass.h index dd91e7a152..2ebcd4673e 100644 --- a/opt/remove-uninstantiables/RemoveUninstantiablesPass.h +++ b/opt/remove-uninstantiables/RemoveUninstantiablesPass.h @@ -14,7 +14,7 @@ namespace cfg { class ControlFlowGraph; } // namespace cfg -std::unordered_map make_deobfuscated_map(const std::unordered_set& obfuscated_types); +std::unordered_map make_deobfuscated_map(const std::unordered_set& obfuscated_types); class UsageHandler { public: @@ -38,7 +38,7 @@ class UsageHandler { void handle_usage_line( const std::string& line, - const std::unordered_map& deobfuscated_uninstantiable_type, + const std::unordered_map& deobfuscated_uninstantiable_type, std::unordered_set& uninstantiable_types); }; diff --git a/test/unit/RemoveUninstantiablesTest.cpp b/test/unit/RemoveUninstantiablesTest.cpp index f0bd16a352..565f37e05a 100644 --- a/test/unit/RemoveUninstantiablesTest.cpp +++ b/test/unit/RemoveUninstantiablesTest.cpp @@ -12,6 +12,7 @@ #include "RedexTest.h" #include "RemoveUninstantiablesPass.h" #include "ScopeHelper.h" +#include "Trace.h" #include "VirtualScope.h" namespace { @@ -774,7 +775,7 @@ TEST_F(RemoveUninstantiablesTest, UsageHandlerMixedObfuscationUninstantiables_Us uninstantiable_types.emplace(unobfuscated_instantiable->get_type()); uninstantiable_types.emplace(unobfuscated_uninstantiable->get_type()); - std::unordered_map deobfuscated_uninstantiable_type; + std::unordered_map deobfuscated_uninstantiable_type; deobfuscated_uninstantiable_type.emplace("Ldeobfuscated/instantiable$name;", obfuscated_instantiable->get_type()); deobfuscated_uninstantiable_type.emplace("Ldeobfuscated/uninstantiable$name;", obfuscated_uninstantiable->get_type()); @@ -812,7 +813,7 @@ TEST_F(RemoveUninstantiablesTest, MakeDeobfuscatedMapMixedObfuscation_OnlyDeobfu std::unordered_set uninstantiable_types; uninstantiable_types.emplace(obfuscated_class->get_type()); - std::unordered_map deobfuscated_uninstantiable_type = + std::unordered_map deobfuscated_uninstantiable_type = make_deobfuscated_map(uninstantiable_types); EXPECT_EQ(1, deobfuscated_uninstantiable_type.size()); EXPECT_TRUE(deobfuscated_uninstantiable_type.count("deobfuscated.cls$name")); @@ -846,6 +847,7 @@ R"(deobfuscated.cls$name -> a.b.c$d : std::basic_istringstream proguard_input(proguard_input_str); ConfigFiles c(Json::nullValue, proguard_input); + c.parse_global_config(); PassManager pm({&pass}); // applying deobfuscation happens in main.cpp which seems like bad design @@ -856,15 +858,15 @@ R"(deobfuscated.cls$name -> a.b.c$d : pm.run_passes(dss, c); ASSERT_EQ(1, dss.back().get_dexen().size()); ASSERT_EQ(1, dss.back().get_dexen().at(0).size()); - + EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_name()->c_str(), testing::StrEq("La/b/c$d;")); EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_deobfuscated_name().c_str(), testing::StrEq("Ldeobfuscated/cls$name;")); - + ASSERT_EQ(1, dss.back().get_dexen().at(0).at(0)->get_all_methods().size()); EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_all_methods().at(0)->get_name()->c_str(), testing::StrEq("something")); - + ASSERT_TRUE(dss.back().get_dexen().at(0).at(0)->get_all_methods().at(0)->get_code()); - EXPECT_THAT(expected_code, assembler::to_string(dss.back().get_dexen().at(0).at(0)->get_all_methods().at(0)->get_code())); + EXPECT_THAT(expected_code, testing::StrEq(assembler::to_string(dss.back().get_dexen().at(0).at(0)->get_all_methods().at(0)->get_code()))); EXPECT_THAT(dss.back().get_dexen().at(0).at(0)->get_vmethods(), testing::ContainerEq(dss.back().get_dexen().at(0).at(0)->get_all_methods())); const auto& pass_infos = pm.get_pass_info();