diff --git a/macho/input-files.cc b/macho/input-files.cc index b9816ff186..412b99dc47 100644 --- a/macho/input-files.cc +++ b/macho/input-files.cc @@ -166,7 +166,7 @@ split_regular_sections(Context &ctx, ObjectFile &file) { for (i64 i = 0; i < file.sections.size(); i++) if (InputSection *isec = file.sections[i].get()) - if (!isec->hdr.match("__TEXT", "__cstring")) + if (isec->hdr.type != S_CSTRING_LITERALS) vec[i].isec = isec; // Find all symbols whose type is N_SECT. @@ -226,12 +226,14 @@ template void ObjectFile::split_subsections_via_symbols(Context &ctx) { sym_to_subsec.resize(mach_syms.size()); - auto add = [&](InputSection &isec, u32 offset, u32 size, u8 p2align) { + auto add = [&](InputSection &isec, u32 offset, u32 size, u8 p2align, + bool is_cstring) { Subsection *subsec = new Subsection{ .isec = isec, .input_offset = offset, .input_size = size, .input_addr = (u32)(isec.hdr.addr + offset), + .is_cstring = is_cstring, .p2align = p2align, }; @@ -244,7 +246,7 @@ void ObjectFile::split_subsections_via_symbols(Context &ctx) { InputSection &isec = *info.isec; for (SplitRegion &r : info.regions) { if (!r.is_alt_entry) - add(isec, r.offset, r.size, isec.hdr.p2align); + add(isec, r.offset, r.size, isec.hdr.p2align, false); if (r.symidx != -1) sym_to_subsec[r.symidx] = subsections.back(); } @@ -252,14 +254,14 @@ void ObjectFile::split_subsections_via_symbols(Context &ctx) { // Split __cstring section. for (std::unique_ptr> &isec : sections) { - if (isec && isec->hdr.match("__TEXT", "__cstring")) { + if (isec && isec->hdr.type == S_CSTRING_LITERALS) { std::string_view str = isec->contents; size_t pos = 0; while (pos < str.size()) { size_t end = str.find('\0', pos); if (end == str.npos) - Fatal(ctx) << *this << " corruupted __TEXT,__cstring"; + Fatal(ctx) << *this << " corruupted cstring section: " << *isec; end = str.find_first_not_of('\0', end); if (end == str.npos) @@ -268,7 +270,7 @@ void ObjectFile::split_subsections_via_symbols(Context &ctx) { // A constant string in __cstring has no alignment info, so we // need to infer it. u8 p2align = std::min(isec->hdr.p2align, std::countr_zero(pos)); - add(*isec, pos, end - pos, p2align); + add(*isec, pos, end - pos, p2align, true); pos = end; } } @@ -422,7 +424,8 @@ void ObjectFile::parse_compact_unwind(Context &ctx, MachSection &hdr) { UnwindRecord &dst = unwind_records[idx]; auto error = [&] { - Fatal(ctx) << *this << ": __compact_unwind: unsupported relocation: " << i; + Fatal(ctx) << *this << ": __compact_unwind: unsupported relocation: " << i + << " " << *this->syms[r.idx]; }; if (r.is_pcrel || r.p2size != 3 || r.type) diff --git a/macho/main.cc b/macho/main.cc index c7d611728b..0dac77f5de 100644 --- a/macho/main.cc +++ b/macho/main.cc @@ -287,105 +287,6 @@ static void claim_unresolved_symbols(Context &ctx) { }); } -template -static void merge_cstring_sections(Context &ctx) { - Timer t(ctx, "merge_cstring_sections"); - - struct Entry { - Entry(Subsection *subsec) : owner(subsec) {} - - Entry(const Entry &other) : - owner(other.owner.load()), p2align(other.p2align.load()) {} - - std::atomic *> owner = nullptr; - std::atomic_uint8_t p2align = 0; - }; - - struct SubsecRef { - Subsection &subsec; - u64 hash = 0; - Entry *ent = nullptr; - }; - - std::vector> vec(ctx.objs.size()); - - // Estimate the number of unique strings. - HyperLogLog estimator; - - tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) { - ObjectFile *file = ctx.objs[i]; - HyperLogLog e; - - for (Subsection *subsec : file->subsections) { - if (&subsec->isec.osec == ctx.cstring) { - std::string_view str = subsec->get_contents(); - u64 h = hash_string(str); - vec[i].push_back({*subsec, h, nullptr}); - estimator.insert(h); - } - } - estimator.merge(e); - }); - - // Create a hash map large enough to hold all strings. - ConcurrentMap map(estimator.get_cardinality() * 3 / 2); - - // Insert all strings into the hash table. - tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) { - ObjectFile *file = ctx.objs[i]; - - for (i64 j = 0; j < vec[i].size(); j++) { - SubsecRef &ref = vec[i][j]; - std::string_view s = ref.subsec.get_contents(); - ref.ent = map.insert(s, ref.hash, {&ref.subsec}).first; - - Subsection *existing = ref.ent->owner; - while (existing->isec.file.priority < file->priority && - !ref.ent->owner.compare_exchange_weak(existing, &ref.subsec)); - - update_maximum(ref.ent->p2align, ref.subsec.p2align.load()); - } - }); - - // Decide who will become the owner for each subsection. - tbb::parallel_for((i64)0, (i64)ctx.objs.size(), [&](i64 i) { - for (i64 j = 0; j < vec[i].size(); j++) { - SubsecRef &ref = vec[i][j]; - if (ref.ent->owner != &ref.subsec) { - ref.subsec.is_coalesced = true; - ref.subsec.replacer = ref.ent->owner; - - static Counter counter("num_merged_strings"); - counter++; - } - } - }); - - // Merge strings - tbb::parallel_for_each(ctx.objs, [&](ObjectFile *file) { - for (std::unique_ptr> &isec : file->sections) - if (isec) - for (Relocation &r : isec->rels) - if (r.subsec && r.subsec->is_coalesced) - r.subsec = r.subsec->replacer; - }); - - auto replace = [&](InputFile *file) { - for (Symbol *sym : file->syms) - if (sym->subsec && sym->subsec->is_coalesced) - sym->subsec = sym->subsec->replacer; - }; - - tbb::parallel_for_each(ctx.objs, replace); - tbb::parallel_for_each(ctx.dylibs, replace); - - tbb::parallel_for_each(ctx.objs, [&](ObjectFile *file) { - std::erase_if(file->subsections, [](Subsection *subsec) { - return subsec->is_coalesced; - }); - }); -} - template static Chunk *find_section(Context &ctx, std::string_view segname, std::string_view sectname) { @@ -446,6 +347,109 @@ static void create_synthetic_chunks(Context &ctx) { sort(seg->chunks, compare_chunks); } +template +static void uniquify_cstrings(Context &ctx, OutputSection &osec) { + Timer t(ctx, "uniquify_cstrings"); + + struct Entry { + Entry(Subsection *subsec) : owner(subsec) {} + + Entry(const Entry &other) : + owner(other.owner.load()), p2align(other.p2align.load()) {} + + std::atomic *> owner = nullptr; + std::atomic_uint8_t p2align = 0; + }; + + struct SubsecRef { + Subsection *subsec = nullptr; + u64 hash = 0; + Entry *ent = nullptr; + }; + + std::vector vec(osec.members.size()); + + // Estimate the number of unique strings. + tbb::enumerable_thread_specific estimators; + + tbb::parallel_for((i64)0, (i64)osec.members.size(), [&](i64 i) { + Subsection *subsec = osec.members[i]; + if (subsec->is_cstring) { + u64 h = hash_string(subsec->get_contents()); + vec[i].subsec = subsec; + vec[i].hash = h; + estimators.local().insert(h); + } + }); + + HyperLogLog estimator; + for (HyperLogLog &e : estimators) + estimator.merge(e); + + // Create a hash map large enough to hold all strings. + ConcurrentMap map(estimator.get_cardinality() * 3 / 2); + + // Insert all strings into the hash table. + tbb::parallel_for_each(vec, [&](SubsecRef &ref) { + if (ref.subsec) { + std::string_view s = ref.subsec->get_contents(); + ref.ent = map.insert(s, ref.hash, {ref.subsec}).first; + + Subsection *existing = ref.ent->owner; + while (existing->isec.file.priority < ref.subsec->isec.file.priority && + !ref.ent->owner.compare_exchange_weak(existing, ref.subsec)); + + update_maximum(ref.ent->p2align, ref.subsec->p2align.load()); + } + }); + + // Decide who will become the owner for each subsection. + tbb::parallel_for_each(vec, [&](SubsecRef &ref) { + if (ref.subsec && ref.subsec != ref.ent->owner) { + ref.subsec->is_coalesced = true; + ref.subsec->replacer = ref.ent->owner; + } + }); + + static Counter counter("num_merged_strings"); + counter += std::erase_if(osec.members, [](Subsection *subsec) { + return subsec->is_coalesced; + }); +} + +template +static void merge_cstring_sections(Context &ctx) { + Timer t(ctx, "merge_cstring_sections"); + + for (Chunk *chunk : ctx.chunks) + if (chunk->is_output_section && chunk->hdr.type == S_CSTRING_LITERALS) + uniquify_cstrings(ctx, *(OutputSection *)chunk); + + // Rewrite relocations and symbols. + tbb::parallel_for_each(ctx.objs, [&](ObjectFile *file) { + for (std::unique_ptr> &isec : file->sections) + if (isec) + for (Relocation &r : isec->rels) + if (r.subsec && r.subsec->is_coalesced) + r.subsec = r.subsec->replacer; + }); + + std::vector *> files; + append(files, ctx.objs); + append(files, ctx.dylibs); + + for (InputFile *file: files) + for (Symbol *sym : file->syms) + if (sym && sym->subsec && sym->subsec->is_coalesced) + sym->subsec = sym->subsec->replacer; + + tbb::parallel_for_each(ctx.objs, [&](ObjectFile *file) { + std::erase_if(file->subsections, [](Subsection *subsec) { + return subsec->is_coalesced; + }); + }); +} + template static void scan_unwind_info(Context &ctx) { tbb::parallel_for_each(ctx.objs, [&](ObjectFile *file) { @@ -930,12 +934,11 @@ static int do_main(int argc, char **argv) { claim_unresolved_symbols(ctx); - merge_cstring_sections(ctx); - if (ctx.arg.dead_strip) dead_strip(ctx); create_synthetic_chunks(ctx); + merge_cstring_sections(ctx); for (ObjectFile *file : ctx.objs) file->check_duplicate_symbols(ctx); diff --git a/macho/mold.h b/macho/mold.h index 5215a9a1b7..8a6c794342 100644 --- a/macho/mold.h +++ b/macho/mold.h @@ -243,6 +243,7 @@ class Subsection { std::atomic_uint8_t p2align = 0; std::atomic_bool is_alive = true; + bool is_cstring : 1 = false; bool is_coalesced : 1 = false; bool added_to_osec : 1 = false; }; @@ -821,11 +822,9 @@ struct Context { text = OutputSection::get_instance(*this, "__TEXT", "__text"); data = OutputSection::get_instance(*this, "__DATA", "__data"); bss = OutputSection::get_instance(*this, "__DATA", "__bss"); - cstring = OutputSection::get_instance(*this, "__TEXT", "__cstring"); common = OutputSection::get_instance(*this, "__DATA", "__common"); bss->hdr.type = S_ZEROFILL; - cstring->hdr.type = S_CSTRING_LITERALS; common->hdr.type = S_ZEROFILL; } @@ -959,7 +958,6 @@ struct Context { OutputSection *text = nullptr; OutputSection *data = nullptr; OutputSection *bss = nullptr; - OutputSection *cstring = nullptr; OutputSection *common = nullptr; };