/*
|
* Copyright (C) 2017 The Android Open Source Project
|
*
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
* you may not use this file except in compliance with the License.
|
* You may obtain a copy of the License at
|
*
|
* http://www.apache.org/licenses/LICENSE-2.0
|
*
|
* Unless required by applicable law or agreed to in writing, software
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
* See the License for the specific language governing permissions and
|
* limitations under the License.
|
*/
|
|
#include "compact_dex_writer.h"
|
|
#include "android-base/stringprintf.h"
|
#include "base/logging.h"
|
#include "base/time_utils.h"
|
#include "dex/compact_dex_file.h"
|
#include "dex/compact_offset_table.h"
|
#include "dexlayout.h"
|
|
namespace art {
|
|
CompactDexWriter::CompactDexWriter(DexLayout* dex_layout)
|
: DexWriter(dex_layout, /*compute_offsets=*/ true) {
|
CHECK(GetCompactDexLevel() != CompactDexLevel::kCompactDexLevelNone);
|
}
|
|
CompactDexLevel CompactDexWriter::GetCompactDexLevel() const {
|
return dex_layout_->GetOptions().compact_dex_level_;
|
}
|
|
CompactDexWriter::Container::Container(bool dedupe_code_items)
|
: code_item_dedupe_(dedupe_code_items, &data_section_),
|
data_item_dedupe_(/*enabled=*/ true, &data_section_) {}
|
|
uint32_t CompactDexWriter::WriteDebugInfoOffsetTable(Stream* stream) {
|
const uint32_t start_offset = stream->Tell();
|
// Debug offsets for method indexes. 0 means no debug info.
|
std::vector<uint32_t> debug_info_offsets(header_->MethodIds().Size(), 0u);
|
|
static constexpr InvokeType invoke_types[] = {
|
kDirect,
|
kVirtual
|
};
|
|
for (InvokeType invoke_type : invoke_types) {
|
for (auto& class_def : header_->ClassDefs()) {
|
// Skip classes that are not defined in this dex file.
|
dex_ir::ClassData* class_data = class_def->GetClassData();
|
if (class_data == nullptr) {
|
continue;
|
}
|
for (auto& method : *(invoke_type == InvokeType::kDirect
|
? class_data->DirectMethods()
|
: class_data->VirtualMethods())) {
|
const dex_ir::MethodId* method_id = method.GetMethodId();
|
dex_ir::CodeItem* code_item = method.GetCodeItem();
|
if (code_item != nullptr && code_item->DebugInfo() != nullptr) {
|
const uint32_t debug_info_offset = code_item->DebugInfo()->GetOffset();
|
const uint32_t method_idx = method_id->GetIndex();
|
if (debug_info_offsets[method_idx] != 0u) {
|
CHECK_EQ(debug_info_offset, debug_info_offsets[method_idx]);
|
}
|
debug_info_offsets[method_idx] = debug_info_offset;
|
}
|
}
|
}
|
}
|
|
std::vector<uint8_t> data;
|
debug_info_base_ = 0u;
|
debug_info_offsets_table_offset_ = 0u;
|
CompactOffsetTable::Build(debug_info_offsets,
|
&data,
|
&debug_info_base_,
|
&debug_info_offsets_table_offset_);
|
// Align the table and write it out.
|
stream->AlignTo(CompactOffsetTable::kAlignment);
|
debug_info_offsets_pos_ = stream->Tell();
|
stream->Write(data.data(), data.size());
|
|
// Verify that the whole table decodes as expected and measure average performance.
|
const bool kMeasureAndTestOutput = dex_layout_->GetOptions().verify_output_;
|
if (kMeasureAndTestOutput && !debug_info_offsets.empty()) {
|
uint64_t start_time = NanoTime();
|
stream->Begin();
|
CompactOffsetTable::Accessor accessor(stream->Begin() + debug_info_offsets_pos_,
|
debug_info_base_,
|
debug_info_offsets_table_offset_);
|
|
for (size_t i = 0; i < debug_info_offsets.size(); ++i) {
|
CHECK_EQ(accessor.GetOffset(i), debug_info_offsets[i]);
|
}
|
uint64_t end_time = NanoTime();
|
VLOG(dex) << "Average lookup time (ns) for debug info offsets: "
|
<< (end_time - start_time) / debug_info_offsets.size();
|
}
|
|
return stream->Tell() - start_offset;
|
}
|
|
CompactDexWriter::ScopedDataSectionItem::ScopedDataSectionItem(Stream* stream,
|
dex_ir::Item* item,
|
size_t alignment,
|
Deduper* deduper)
|
: stream_(stream),
|
item_(item),
|
alignment_(alignment),
|
deduper_(deduper),
|
start_offset_(stream->Tell()) {
|
stream_->AlignTo(alignment_);
|
}
|
|
CompactDexWriter::ScopedDataSectionItem::~ScopedDataSectionItem() {
|
// After having written, maybe dedupe the whole code item (excluding padding).
|
const uint32_t deduped_offset = deduper_->Dedupe(start_offset_,
|
stream_->Tell(),
|
item_->GetOffset());
|
// If we deduped, only use the deduped offset if the alignment matches the required alignment.
|
// Otherwise, return without deduping.
|
if (deduped_offset != Deduper::kDidNotDedupe && IsAlignedParam(deduped_offset, alignment_)) {
|
// Update the IR offset to the offset of the deduped item.
|
item_->SetOffset(deduped_offset);
|
// Clear the written data for the item so that the stream write doesn't abort in the future.
|
stream_->Clear(start_offset_, stream_->Tell() - start_offset_);
|
// Since we deduped, restore the offset to the original position.
|
stream_->Seek(start_offset_);
|
}
|
}
|
|
size_t CompactDexWriter::ScopedDataSectionItem::Written() const {
|
return stream_->Tell() - start_offset_;
|
}
|
|
void CompactDexWriter::WriteCodeItem(Stream* stream,
|
dex_ir::CodeItem* code_item,
|
bool reserve_only) {
|
DCHECK(code_item != nullptr);
|
DCHECK(!reserve_only) << "Not supported because of deduping.";
|
ScopedDataSectionItem data_item(stream,
|
code_item,
|
CompactDexFile::CodeItem::kAlignment,
|
code_item_dedupe_);
|
|
CompactDexFile::CodeItem disk_code_item;
|
|
uint16_t preheader_storage[CompactDexFile::CodeItem::kMaxPreHeaderSize] = {};
|
uint16_t* preheader_end = preheader_storage + CompactDexFile::CodeItem::kMaxPreHeaderSize;
|
const uint16_t* preheader = disk_code_item.Create(
|
code_item->RegistersSize(),
|
code_item->InsSize(),
|
code_item->OutsSize(),
|
code_item->TriesSize(),
|
code_item->InsnsSize(),
|
preheader_end);
|
const size_t preheader_bytes = (preheader_end - preheader) * sizeof(preheader[0]);
|
|
static constexpr size_t kPayloadInstructionRequiredAlignment = 4;
|
const uint32_t current_code_item_start = stream->Tell() + preheader_bytes;
|
if (!IsAlignedParam(current_code_item_start, kPayloadInstructionRequiredAlignment) ||
|
kIsDebugBuild) {
|
// If the preheader is going to make the code unaligned, consider adding 2 bytes of padding
|
// before if required.
|
IterationRange<DexInstructionIterator> instructions = code_item->Instructions();
|
SafeDexInstructionIterator it(instructions.begin(), instructions.end());
|
for (; !it.IsErrorState() && it < instructions.end(); ++it) {
|
// In case the instruction goes past the end of the code item, make sure to not process it.
|
if (std::next(it).IsErrorState()) {
|
break;
|
}
|
const Instruction::Code opcode = it->Opcode();
|
// Payload instructions possibly require special alignment for their data.
|
if (opcode == Instruction::FILL_ARRAY_DATA ||
|
opcode == Instruction::PACKED_SWITCH ||
|
opcode == Instruction::SPARSE_SWITCH) {
|
stream->Skip(
|
RoundUp(current_code_item_start, kPayloadInstructionRequiredAlignment) -
|
current_code_item_start);
|
break;
|
}
|
}
|
}
|
|
// Write preheader first.
|
stream->Write(reinterpret_cast<const uint8_t*>(preheader), preheader_bytes);
|
// Registered offset is after the preheader.
|
ProcessOffset(stream, code_item);
|
// Avoid using sizeof so that we don't write the fake instruction array at the end of the code
|
// item.
|
stream->Write(&disk_code_item, OFFSETOF_MEMBER(CompactDexFile::CodeItem, insns_));
|
// Write the instructions.
|
stream->Write(code_item->Insns(), code_item->InsnsSize() * sizeof(uint16_t));
|
// Write the post instruction data.
|
WriteCodeItemPostInstructionData(stream, code_item, reserve_only);
|
}
|
|
void CompactDexWriter::WriteDebugInfoItem(Stream* stream, dex_ir::DebugInfoItem* debug_info) {
|
ScopedDataSectionItem data_item(stream,
|
debug_info,
|
SectionAlignment(DexFile::kDexTypeDebugInfoItem),
|
data_item_dedupe_);
|
ProcessOffset(stream, debug_info);
|
stream->Write(debug_info->GetDebugInfo(), debug_info->GetDebugInfoSize());
|
}
|
|
|
CompactDexWriter::Deduper::Deduper(bool enabled, DexContainer::Section* section)
|
: enabled_(enabled),
|
dedupe_map_(/*__n=*/ 32,
|
HashedMemoryRange::HashEqual(section),
|
HashedMemoryRange::HashEqual(section)) {}
|
|
uint32_t CompactDexWriter::Deduper::Dedupe(uint32_t data_start,
|
uint32_t data_end,
|
uint32_t item_offset) {
|
if (!enabled_) {
|
return kDidNotDedupe;
|
}
|
HashedMemoryRange range {data_start, data_end - data_start};
|
auto existing = dedupe_map_.emplace(range, item_offset);
|
if (!existing.second) {
|
// Failed to insert means we deduped, return the existing item offset.
|
return existing.first->second;
|
}
|
return kDidNotDedupe;
|
}
|
|
void CompactDexWriter::SortDebugInfosByMethodIndex() {
|
static constexpr InvokeType invoke_types[] = {
|
kDirect,
|
kVirtual
|
};
|
std::map<const dex_ir::DebugInfoItem*, uint32_t> method_idx_map;
|
for (InvokeType invoke_type : invoke_types) {
|
for (auto& class_def : header_->ClassDefs()) {
|
// Skip classes that are not defined in this dex file.
|
dex_ir::ClassData* class_data = class_def->GetClassData();
|
if (class_data == nullptr) {
|
continue;
|
}
|
for (auto& method : *(invoke_type == InvokeType::kDirect
|
? class_data->DirectMethods()
|
: class_data->VirtualMethods())) {
|
const dex_ir::MethodId* method_id = method.GetMethodId();
|
dex_ir::CodeItem* code_item = method.GetCodeItem();
|
if (code_item != nullptr && code_item->DebugInfo() != nullptr) {
|
const dex_ir::DebugInfoItem* debug_item = code_item->DebugInfo();
|
method_idx_map.insert(std::make_pair(debug_item, method_id->GetIndex()));
|
}
|
}
|
}
|
}
|
std::sort(header_->DebugInfoItems().begin(),
|
header_->DebugInfoItems().end(),
|
[&](const std::unique_ptr<dex_ir::DebugInfoItem>& a,
|
const std::unique_ptr<dex_ir::DebugInfoItem>& b) {
|
auto it_a = method_idx_map.find(a.get());
|
auto it_b = method_idx_map.find(b.get());
|
uint32_t idx_a = it_a != method_idx_map.end() ? it_a->second : 0u;
|
uint32_t idx_b = it_b != method_idx_map.end() ? it_b->second : 0u;
|
return idx_a < idx_b;
|
});
|
}
|
|
void CompactDexWriter::WriteHeader(Stream* stream) {
|
CompactDexFile::Header header;
|
CompactDexFile::WriteMagic(&header.magic_[0]);
|
CompactDexFile::WriteCurrentVersion(&header.magic_[0]);
|
header.checksum_ = header_->Checksum();
|
std::copy_n(header_->Signature(), DexFile::kSha1DigestSize, header.signature_);
|
header.file_size_ = header_->FileSize();
|
// Since we are not necessarily outputting the same format as the input, avoid using the stored
|
// header size.
|
header.header_size_ = GetHeaderSize();
|
header.endian_tag_ = header_->EndianTag();
|
header.link_size_ = header_->LinkSize();
|
header.link_off_ = header_->LinkOffset();
|
header.map_off_ = header_->MapListOffset();
|
header.string_ids_size_ = header_->StringIds().Size();
|
header.string_ids_off_ = header_->StringIds().GetOffset();
|
header.type_ids_size_ = header_->TypeIds().Size();
|
header.type_ids_off_ = header_->TypeIds().GetOffset();
|
header.proto_ids_size_ = header_->ProtoIds().Size();
|
header.proto_ids_off_ = header_->ProtoIds().GetOffset();
|
header.field_ids_size_ = header_->FieldIds().Size();
|
header.field_ids_off_ = header_->FieldIds().GetOffset();
|
header.method_ids_size_ = header_->MethodIds().Size();
|
header.method_ids_off_ = header_->MethodIds().GetOffset();
|
header.class_defs_size_ = header_->ClassDefs().Size();
|
header.class_defs_off_ = header_->ClassDefs().GetOffset();
|
header.data_size_ = header_->DataSize();
|
header.data_off_ = header_->DataOffset();
|
header.owned_data_begin_ = owned_data_begin_;
|
header.owned_data_end_ = owned_data_end_;
|
|
// Compact dex specific flags.
|
header.debug_info_offsets_pos_ = debug_info_offsets_pos_;
|
header.debug_info_offsets_table_offset_ = debug_info_offsets_table_offset_;
|
header.debug_info_base_ = debug_info_base_;
|
header.feature_flags_ = 0u;
|
// In cases where apps are converted to cdex during install, maintain feature flags so that
|
// the verifier correctly verifies apps that aren't targetting default methods.
|
if (header_->SupportDefaultMethods()) {
|
header.feature_flags_ |= static_cast<uint32_t>(CompactDexFile::FeatureFlags::kDefaultMethods);
|
}
|
stream->Seek(0);
|
stream->Overwrite(reinterpret_cast<uint8_t*>(&header), sizeof(header));
|
}
|
|
size_t CompactDexWriter::GetHeaderSize() const {
|
return sizeof(CompactDexFile::Header);
|
}
|
|
void CompactDexWriter::WriteStringData(Stream* stream, dex_ir::StringData* string_data) {
|
ScopedDataSectionItem data_item(stream,
|
string_data,
|
SectionAlignment(DexFile::kDexTypeStringDataItem),
|
data_item_dedupe_);
|
ProcessOffset(stream, string_data);
|
stream->WriteUleb128(CountModifiedUtf8Chars(string_data->Data()));
|
stream->Write(string_data->Data(), strlen(string_data->Data()));
|
// Skip null terminator (already zeroed out, no need to write).
|
stream->Skip(1);
|
}
|
|
bool CompactDexWriter::CanGenerateCompactDex(std::string* error_msg) {
|
static constexpr InvokeType invoke_types[] = {
|
kDirect,
|
kVirtual
|
};
|
std::vector<bool> saw_method_id(header_->MethodIds().Size(), false);
|
std::vector<dex_ir::CodeItem*> method_id_code_item(header_->MethodIds().Size(), nullptr);
|
std::vector<dex_ir::DebugInfoItem*> method_id_debug_info(header_->MethodIds().Size(), nullptr);
|
for (InvokeType invoke_type : invoke_types) {
|
for (auto& class_def : header_->ClassDefs()) {
|
// Skip classes that are not defined in this dex file.
|
dex_ir::ClassData* class_data = class_def->GetClassData();
|
if (class_data == nullptr) {
|
continue;
|
}
|
for (auto& method : *(invoke_type == InvokeType::kDirect
|
? class_data->DirectMethods()
|
: class_data->VirtualMethods())) {
|
const uint32_t idx = method.GetMethodId()->GetIndex();
|
dex_ir::CodeItem* code_item = method.GetCodeItem();
|
dex_ir:: DebugInfoItem* debug_info_item = nullptr;
|
if (code_item != nullptr) {
|
debug_info_item = code_item->DebugInfo();
|
}
|
if (saw_method_id[idx]) {
|
if (method_id_code_item[idx] != code_item) {
|
*error_msg = android::base::StringPrintf("Conflicting code item for method id %u",
|
idx);
|
// Conflicting info, abort generation.
|
return false;
|
}
|
if (method_id_debug_info[idx] != debug_info_item) {
|
*error_msg = android::base::StringPrintf("Conflicting debug info for method id %u",
|
idx);
|
// Conflicting info, abort generation.
|
return false;
|
}
|
}
|
method_id_code_item[idx] = code_item;
|
method_id_debug_info[idx] = debug_info_item;
|
saw_method_id[idx] = true;
|
}
|
}
|
}
|
return true;
|
}
|
|
bool CompactDexWriter::Write(DexContainer* output, std::string* error_msg) {
|
DCHECK(error_msg != nullptr);
|
CHECK(compute_offsets_);
|
CHECK(output->IsCompactDexContainer());
|
|
if (!CanGenerateCompactDex(error_msg)) {
|
return false;
|
}
|
|
Container* const container = down_cast<Container*>(output);
|
// For now, use the same stream for both data and metadata.
|
Stream temp_main_stream(output->GetMainSection());
|
CHECK_EQ(output->GetMainSection()->Size(), 0u);
|
Stream temp_data_stream(output->GetDataSection());
|
Stream* main_stream = &temp_main_stream;
|
Stream* data_stream = &temp_data_stream;
|
|
// We want offset 0 to be reserved for null, seek to the data section alignment or the end of the
|
// section.
|
data_stream->Seek(std::max(
|
static_cast<uint32_t>(output->GetDataSection()->Size()),
|
kDataSectionAlignment));
|
code_item_dedupe_ = &container->code_item_dedupe_;
|
data_item_dedupe_ = &container->data_item_dedupe_;
|
|
// Starting offset is right after the header.
|
main_stream->Seek(GetHeaderSize());
|
|
// Based on: https://source.android.com/devices/tech/dalvik/dex-format
|
// Since the offsets may not be calculated already, the writing must be done in the correct order.
|
const uint32_t string_ids_offset = main_stream->Tell();
|
WriteStringIds(main_stream, /*reserve_only=*/ true);
|
WriteTypeIds(main_stream);
|
const uint32_t proto_ids_offset = main_stream->Tell();
|
WriteProtoIds(main_stream, /*reserve_only=*/ true);
|
WriteFieldIds(main_stream);
|
WriteMethodIds(main_stream);
|
const uint32_t class_defs_offset = main_stream->Tell();
|
WriteClassDefs(main_stream, /*reserve_only=*/ true);
|
const uint32_t call_site_ids_offset = main_stream->Tell();
|
WriteCallSiteIds(main_stream, /*reserve_only=*/ true);
|
WriteMethodHandles(main_stream);
|
|
if (compute_offsets_) {
|
// Data section.
|
data_stream->AlignTo(kDataSectionAlignment);
|
}
|
owned_data_begin_ = data_stream->Tell();
|
|
// Write code item first to minimize the space required for encoded methods.
|
// For cdex, the code items don't depend on the debug info.
|
WriteCodeItems(data_stream, /*reserve_only=*/ false);
|
|
// Sort the debug infos by method index order, this reduces size by ~0.1% by reducing the size of
|
// the debug info offset table.
|
SortDebugInfosByMethodIndex();
|
WriteDebugInfoItems(data_stream);
|
|
WriteEncodedArrays(data_stream);
|
WriteAnnotations(data_stream);
|
WriteAnnotationSets(data_stream);
|
WriteAnnotationSetRefs(data_stream);
|
WriteAnnotationsDirectories(data_stream);
|
WriteTypeLists(data_stream);
|
WriteClassDatas(data_stream);
|
WriteStringDatas(data_stream);
|
WriteHiddenapiClassData(data_stream);
|
|
// Write delayed id sections that depend on data sections.
|
{
|
Stream::ScopedSeek seek(main_stream, string_ids_offset);
|
WriteStringIds(main_stream, /*reserve_only=*/ false);
|
}
|
{
|
Stream::ScopedSeek seek(main_stream, proto_ids_offset);
|
WriteProtoIds(main_stream, /*reserve_only=*/ false);
|
}
|
{
|
Stream::ScopedSeek seek(main_stream, class_defs_offset);
|
WriteClassDefs(main_stream, /*reserve_only=*/ false);
|
}
|
{
|
Stream::ScopedSeek seek(main_stream, call_site_ids_offset);
|
WriteCallSiteIds(main_stream, /*reserve_only=*/ false);
|
}
|
|
// Write the map list.
|
if (compute_offsets_) {
|
data_stream->AlignTo(SectionAlignment(DexFile::kDexTypeMapList));
|
header_->SetMapListOffset(data_stream->Tell());
|
} else {
|
data_stream->Seek(header_->MapListOffset());
|
}
|
|
// Map items are included in the data section.
|
GenerateAndWriteMapItems(data_stream);
|
|
// Write link data if it exists.
|
const std::vector<uint8_t>& link_data = header_->LinkData();
|
if (link_data.size() > 0) {
|
CHECK_EQ(header_->LinkSize(), static_cast<uint32_t>(link_data.size()));
|
if (compute_offsets_) {
|
header_->SetLinkOffset(data_stream->Tell());
|
} else {
|
data_stream->Seek(header_->LinkOffset());
|
}
|
data_stream->Write(&link_data[0], link_data.size());
|
}
|
|
// Write debug info offset table last to make dex file verifier happy.
|
WriteDebugInfoOffsetTable(data_stream);
|
|
data_stream->AlignTo(kDataSectionAlignment);
|
owned_data_end_ = data_stream->Tell();
|
if (compute_offsets_) {
|
header_->SetDataSize(data_stream->Tell());
|
if (header_->DataSize() != 0) {
|
// Offset must be zero when the size is zero.
|
main_stream->AlignTo(kDataSectionAlignment);
|
// For now, default to saying the data is right after the main stream.
|
header_->SetDataOffset(main_stream->Tell());
|
} else {
|
header_->SetDataOffset(0u);
|
}
|
}
|
|
// Write header last.
|
if (compute_offsets_) {
|
header_->SetFileSize(main_stream->Tell());
|
}
|
WriteHeader(main_stream);
|
|
// Trim sections to make sure they are sized properly.
|
output->GetMainSection()->Resize(header_->FileSize());
|
output->GetDataSection()->Resize(data_stream->Tell());
|
|
if (dex_layout_->GetOptions().update_checksum_) {
|
// Compute the cdex section (also covers the used part of the data section).
|
header_->SetChecksum(CompactDexFile::CalculateChecksum(output->GetMainSection()->Begin(),
|
output->GetMainSection()->Size(),
|
output->GetDataSection()->Begin(),
|
output->GetDataSection()->Size()));
|
// Rewrite the header with the calculated checksum.
|
WriteHeader(main_stream);
|
}
|
|
// Clear the dedupe to prevent interdex code item deduping. This does not currently work well with
|
// dex2oat's class unloading. The issue is that verification encounters quickened opcodes after
|
// the first dex gets unloaded.
|
code_item_dedupe_->Clear();
|
|
return true;
|
}
|
|
std::unique_ptr<DexContainer> CompactDexWriter::CreateDexContainer() const {
|
return std::unique_ptr<DexContainer>(
|
new CompactDexWriter::Container(dex_layout_->GetOptions().dedupe_code_items_));
|
}
|
|
} // namespace art
|
