Skip to content

Commit

Permalink
support lazy loading of compressed binary files
Browse files Browse the repository at this point in the history 
- add support for decompression of zip compressed files inside a zip
archive.
- add pre b_transport callback function to reg_router which can be used
to lazy load information to register memory before a register is accessed.

Signed-off-by: Mahmoud Kamel <quic_mkamel@quicinc.com>
  • Loading branch information
Mahmoud Kamel committed Sep 2, 2024
1 parent 6a6cc1f commit abc23ff
Show file tree
Hide file tree
Showing 2 changed files with 67 additions and 25 deletions.
43 changes: 36 additions & 7 deletions systemc-components/common/include/loader.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,8 +332,9 @@ class loader : public sc_core::sc_module
- archive_name: the name of the zip archive.
- file_name: name of the file to be extracted from archive_name.
*/
void zip_file_load(zip_t* p_archive, const std::string& archive_name, uint64_t addr, std::string file_name = "",
uint64_t file_offset = 0, uint64_t file_data_len = 0)
void zip_file_load(zip_t* p_archive, const std::string& archive_name, uint64_t addr,
const std::string& file_name = "", uint64_t file_offset = 0, uint64_t file_data_len = 0,
bool is_compressed = false, const std::string& uncompressed_file_name = "")
{
if (archive_name.empty()) SCP_FATAL(()) << "Missing zip archive name!";
zip_t* z_archive = p_archive;
Expand All @@ -357,9 +358,39 @@ class loader : public sc_core::sc_module
if (zip_stat_index(z_archive, 0, ZIP_FL_NOCASE, &z_stat) < 0)
SCP_FATAL(()) << "Can't get status os the file inside zip archive: " << archive_name;
}
zip_int64_t used_data_len = 0;
if (!is_compressed) {
std::vector<uint8_t> bin_info(z_stat.size);
used_data_len = zip_read_file(z_archive, archive_name, z_stat, bin_info, file_offset, file_data_len);
SCP_DEBUG(()) << "load data from zip archive " << archive_name << " to addr: 0x" << std::hex << addr
<< " len: 0x" << std::hex << used_data_len;
send(addr, reinterpret_cast<uint8_t*>(&bin_info[0]) + file_offset, used_data_len);
} else {
std::vector<uint8_t> compressed_file_data(z_stat.size);
zip_read_file(z_archive, file_name, z_stat, compressed_file_data, 0, z_stat.size);
zip_t* f_archive = zip_open_from_source(
zip_source_buffer_create(reinterpret_cast<uint8_t*>(&compressed_file_data[0]), z_stat.size, 0, nullptr),
ZIP_RDONLY, nullptr);
if (!f_archive) SCP_FATAL(()) << "Can't open zip archive: " << file_name;
zip_int64_t f_num_entries = zip_get_num_entries(f_archive, ZIP_FL_UNCHANGED);
if (f_num_entries != 1)
SCP_FATAL(()) << "Compressed file: " << file_name << " in: " << archive_name << " is corrupted!";
zip_stat_t f_stat;
if (zip_stat(f_archive, uncompressed_file_name.c_str(), ZIP_FL_NOCASE, &f_stat) < 0)
SCP_FATAL(()) << "Can't find any file named: " << uncompressed_file_name
<< " in the zip archive: " << file_name << " extracted from: " << archive_name;
std::vector<uint8_t> bin_info(f_stat.size);
used_data_len = zip_read_file(f_archive, file_name, f_stat, bin_info, file_offset, file_data_len);
SCP_DEBUG(()) << "load data from zip archive " << file_name << " to addr: 0x" << std::hex << addr
<< " len: 0x" << std::hex << used_data_len;
send(addr, reinterpret_cast<uint8_t*>(&bin_info[0]) + file_offset, used_data_len);
}
}

std::vector<uint8_t> bin_info(z_stat.size);
uint8_t* bin_info_t = &bin_info[0];
zip_int64_t zip_read_file(zip_t* z_archive, const std::string& archive_name, const zip_stat_t& z_stat,
std::vector<uint8_t>& data, uint64_t file_offset, uint64_t file_data_len)
{
uint8_t* bin_info_t = &data[0];
zip_file_t* fd = zip_fopen(z_archive, z_stat.name, ZIP_FL_NOCASE);
if (!fd) SCP_FATAL(()) << "Can't open file: " << z_stat.name << "in zip archive: " << archive_name;
zip_int64_t used_file_data_len = 0;
Expand All @@ -383,10 +414,8 @@ class loader : public sc_core::sc_module
bin_info_t += read_bytes_num;
rem -= read_bytes_num;
}
SCP_DEBUG(()) << "load data from zip archive " << archive_name << " to addr: 0x" << std::hex << addr
<< " len: 0x" << std::hex << used_file_data_len;
send(addr, reinterpret_cast<uint8_t*>(&bin_info[0]) + file_offset, used_file_data_len);
zip_fclose(fd);
return used_file_data_len;
}

void csv_load(std::string filename, uint64_t offset, std::string addr_str, std::string value_str, bool byte_swap)
Expand Down
49 changes: 31 additions & 18 deletions systemc-components/reg_router/include/reg_router.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
#include <vector>
#include <memory>
#include <map>
#include <functional>

#define ENABLE_MODULE_NAME_DBG_INFO 1
namespace gs {
Expand Down Expand Up @@ -68,8 +69,13 @@ class reg_router : public sc_core::sc_module, public gs::router_if<BUSWIDTH>
}

#ifdef ENABLE_MODULE_NAME_DBG_INFO
std::string mod_name = get_module_name(trans);
if (!mod_name.empty()) SCP_DEBUG(()) << "b_transport: " << mod_name;
uint64_t mod_addr = 0;
bool mod_found = false;
const char* mod_name = get_module_name(trans, mod_addr, mod_found);
if (mod_name)
SCP_DEBUG(()) << "b_transport: " << txn_command_str(trans) << " to Register at address: 0x" << std::hex
<< trans.get_address() << " which belongs to Module: [" << mod_name << "]";
if (m_pre_b_transport_callback) m_pre_b_transport_callback(mod_found, mod_addr);
#endif

bool found_cb = do_callbacks(trans, delay);
Expand Down Expand Up @@ -101,8 +107,13 @@ class reg_router : public sc_core::sc_module, public gs::router_if<BUSWIDTH>
}

#ifdef ENABLE_MODULE_NAME_DBG_INFO
std::string mod_name = get_module_name(trans);
if (!mod_name.empty()) SCP_DEBUG(()) << "transport_dbg: " << mod_name;
uint64_t mod_addr = 0;
bool mod_found = false;
const char* mod_name = get_module_name(trans, mod_addr, mod_found);
if (mod_name)
SCP_DEBUG(()) << "transport_dbg: " << txn_command_str(trans) << " to Register at address: 0x" << std::hex
<< trans.get_address() << " which belongs to Module: [" << mod_name << "]";
if (m_pre_b_transport_callback) m_pre_b_transport_callback(mod_found, mod_addr);
#endif

if (ti->use_offset) trans.set_address(addr - ti->address);
Expand Down Expand Up @@ -160,33 +171,32 @@ class reg_router : public sc_core::sc_module, public gs::router_if<BUSWIDTH>
}

#ifdef ENABLE_MODULE_NAME_DBG_INFO
std::string get_module_name(const tlm::tlm_generic_payload& trans)
const char* get_module_name(const tlm::tlm_generic_payload& trans, uint64_t& mod_addr, bool& mod_found)
{
if (mod_addr_name_map.empty()) {
return "";
return nullptr;
}
uint64_t addr = trans.get_address();
std::string cmd = txn_command_str(trans);
std::string mod_name = "";
std::stringstream ret;
auto search = mod_addr_name_map.equal_range(addr);
if (search.first != mod_addr_name_map.end() && search.first->first == addr) {
if ((addr - search.first->first) < search.first->second.first) {
mod_name = search.first->second.second;
mod_addr = search.first->first;
mod_found = true;
return search.first->second.second;
}
} else if (search.second != mod_addr_name_map.begin()) {
auto expected_node = std::prev(search.second);
if ((addr - expected_node->first) < expected_node->second.first) {
mod_name = expected_node->second.second;
mod_addr = expected_node->first;
mod_found = true;
return expected_node->second.second;
} else {
return "";
mod_found = false;
}
} else {
return "";
mod_found = false;
}
ret << cmd << " to Register at address: 0x" << std::hex << addr << " which belongs to Unimplemented Module: ["
<< mod_name << "]";
return ret.str();
return nullptr;
}
#endif

Expand Down Expand Up @@ -290,14 +300,16 @@ class reg_router : public sc_core::sc_module, public gs::router_if<BUSWIDTH>

public:
explicit reg_router(const sc_core::sc_module_name& nm,
const std::map<uint64_t, std::pair<uint64_t, std::string>>& p_mod_addr_name_map = {},
const std::map<uint64_t, std::pair<uint64_t, const char*>>& p_mod_addr_name_map = {},
const std::function<void(bool, uint64_t)>& pre_b_transport_callback = {},
cci::cci_broker_handle broker = cci::cci_get_broker())
: sc_core::sc_module(nm)
, initiator_socket("initiator_socket", [&](std::string s) -> void { register_boundto(s); })
, target_socket("target_socket")
, m_broker(broker)
, lazy_init("lazy_init", false, "Initialize the reg_router lazily (eg. during simulation rather than BEOL)")
, mod_addr_name_map(p_mod_addr_name_map)
, m_pre_b_transport_callback(pre_b_transport_callback)
{
SCP_DEBUG(()) << "reg_router constructed";

Expand All @@ -324,7 +336,8 @@ class reg_router : public sc_core::sc_module, public gs::router_if<BUSWIDTH>
private:
std::vector<target_info*> mem_targets;
std::map<sc_dt::uint64, target_info*> cb_targets;
std::map<uint64_t, std::pair<uint64_t, std::string>> mod_addr_name_map;
const std::map<uint64_t, std::pair<uint64_t, const char*>>& mod_addr_name_map;
std::function<void(bool, uint64_t)> m_pre_b_transport_callback;
bool initialized = false;
};
} // namespace gs
Expand Down

0 comments on commit abc23ff

Please sign in to comment.