Read authored dialogue lists structurally and accept shared definition hashes

A type-53 dialogue list pairs every cue definition, a hash and its authored
duration, with a content tree sorted by hash. The tree is a sequence of
children, a selection of conditioned entries, or a line holding two takes of
the same text. Text extraction scanned the tree blob for container and string
pairs and refused the whole list when two trees shared a hash, which left six
missions, Homecoming among them, with no dialogue text.

The list is now walked through its node classes, and each cue takes the one
tree whose computed duration equals the authored one. A shared hash is logged,
and a cue that cannot be matched keeps no lines while the rest of the list is
still extracted. All 260 missions with dialogue cues now carry text.

Sunrise/Sunrise.vcxproj

@@ -819,6 +819,7 @@
     <ClCompile Include="src\client\content\activity\activity_sdk_native_pack_pipeline.cpp" />
     <ClCompile Include="src\client\content\activity\activity_sdk_dialogue_group_index.cpp" />
     <ClCompile Include="src\client\content\activity\activity_sdk_actor_ability_inventory.cpp" />
+    <ClCompile Include="src\client\content\activity\activity_sdk_dialogue_list.cpp" />
     <ClCompile Include="src\client\content\activity\activity_sdk_pack_composer.cpp" />
     <ClCompile Include="src\client\content\activity\activity_sdk_pack_composer_rows.cpp" />
     <ClCompile Include="src\client\content\activity\activity_sdk_pack_composer_validate.cpp" />
@@ -2048,6 +2049,7 @@
     <ClInclude Include="src\client\content\activity\activity_sdk_native_pack_pipeline.h" />
     <ClInclude Include="src\client\content\activity\activity_sdk_dialogue_group_index.h" />
     <ClInclude Include="src\client\content\activity\activity_sdk_actor_ability_inventory.h" />
+    <ClInclude Include="src\client\content\activity\activity_sdk_dialogue_list.h" />
     <ClInclude Include="src\client\content\activity\activity_sdk_pack_composer.h" />
     <ClInclude Include="src\client\content\activity\activity_sdk_pack_composer_internal.h" />
     <ClInclude Include="src\client\content\activity\activity_sdk_policy_input_adapter.h" />

Sunrise/src/client/content/activity/activity_sdk_dialogue_list.cpp

@@ -0,0 +1,291 @@
+#include "activity_sdk_dialogue_list.h"
+
+#include <algorithm>
+#include <cmath>
+#include <cstring>
+#include <limits>
+
+namespace sunrise::client::content::activity::sdk_generation::dialogue_list {
+namespace {
+
+// List fields: the cue definitions in cue order, then the content trees sorted by hash.
+constexpr std::size_t kDefinitionField = 0x08;
+constexpr std::size_t kDefinitionStride = 8;
+constexpr std::uint32_t kDefinitionArrayClass = 0x80808D18U;
+constexpr std::size_t kTreeField = 0x18;
+constexpr std::size_t kTreeStride = 16;
+constexpr std::uint32_t kTreeArrayClass = 0x80808D19U;
+// Every tree object repeats the definition hash it belongs to, after a per-object id.
+constexpr std::size_t kObjectHashOffset = 4;
+// A sequence plays its children in order through an array of relative object pointers.
+constexpr std::uint32_t kSequenceClass = 0x80808D1DU;
+constexpr std::size_t kSequenceChildField = 0x18;
+constexpr std::uint32_t kPointerArrayClass = 0x80808D20U;
+constexpr std::size_t kPointerStride = 8;
+// A selection plays one entry; each entry holds its conditions and a pointer to its child.
+constexpr std::uint32_t kSelectionClass = 0x80808D1AU;
+constexpr std::size_t kSelectionEntryField = 0x20;
+constexpr std::uint32_t kSelectionEntryClass = 0x80808D1FU;
+constexpr std::size_t kSelectionEntryStride = 0x38;
+constexpr std::size_t kSelectionChildOffset = 0x30;
+// A line carries two takes of the same text.
+constexpr std::uint32_t kLineClass = 0x80808D23U;
+constexpr std::size_t kFirstTakeOffset = 0x18;
+constexpr std::size_t kSecondTakeOffset = 0x28;
+constexpr std::size_t kLineByteCount = 0x38;
+// Ceilings that stop a corrupt or cyclic tree from driving an unbounded walk.
+constexpr std::size_t kMaximumDepth = 16;
+constexpr std::size_t kMaximumObjects = 1024;
+constexpr std::size_t kMaximumArrayRows = 4096;
+// Authored totals are float sums of the take lengths, so they match within float rounding only.
+constexpr float kSecondsTolerance = 0.001F;
+
+template <typename Value>
+[[nodiscard]] bool
+read_value(std::span<const std::byte> bytes, std::size_t offset, Value& output) noexcept {
+    output = {};
+    if (offset > bytes.size() || sizeof output > bytes.size() - offset) {
+        return false;
+    }
+    std::memcpy(&output, bytes.data() + offset, sizeof output);
+    return true;
+}
+
+/** Follows one signed self-relative pointer stored at `member`. @return False outside the blob. */
+[[nodiscard]] bool
+follow(std::span<const std::byte> bytes, std::size_t member, std::size_t& target) noexcept {
+    std::int64_t relative = 0;
+    if (!read_value(bytes, member, relative)) {
+        return false;
+    }
+    if (relative >= 0) {
+        const auto distance = static_cast<std::uint64_t>(relative);
+        if (distance > bytes.size() - member) {
+            return false;
+        }
+        target = member + static_cast<std::size_t>(distance);
+        return true;
+    }
+    const auto distance = static_cast<std::uint64_t>(-(relative + 1)) + 1U;
+    if (distance > member) {
+        return false;
+    }
+    target = member - static_cast<std::size_t>(distance);
+    return true;
+}
+
+/** Reads one array field's data offset and count, checking its element class and stride. */
+[[nodiscard]] bool read_array(std::span<const std::byte> bytes,
+                              std::size_t field,
+                              std::size_t stride,
+                              std::uint32_t elementClass,
+                              std::size_t& data,
+                              std::size_t& count) noexcept {
+    data = 0;
+    count = 0;
+    std::uint64_t rawCount = 0;
+    std::size_t header = 0;
+    std::uint64_t repeated = 0;
+    std::uint32_t classId = 0;
+    std::uint32_t padding = 0;
+    if (!read_value(bytes, field, rawCount) || rawCount > kMaximumArrayRows) {
+        return false;
+    }
+    if (rawCount == 0) {
+        return true;
+    }
+    if (!follow(bytes, field + 8U, header) || !read_value(bytes, header, repeated)
+        || repeated != rawCount || !read_value(bytes, header + 8U, classId)
+        || classId != elementClass || !read_value(bytes, header + 12U, padding) || padding != 0) {
+        return false;
+    }
+    data = header + 16U;
+    count = static_cast<std::size_t>(rawCount);
+    return data <= bytes.size() && count * stride <= bytes.size() - data;
+}
+
+/** Reads one take stored at `offset`. */
+[[nodiscard]] bool
+read_take(std::span<const std::byte> bytes, std::size_t offset, Take& output) noexcept {
+    return read_value(bytes, offset, output.audioTag)
+           && read_value(bytes, offset + 4U, output.containerTag)
+           && read_value(bytes, offset + 8U, output.stringHash)
+           && read_value(bytes, offset + 12U, output.seconds) && std::isfinite(output.seconds)
+           && output.seconds >= 0.0F;
+}
+
+/** Walk state shared by one tree: its hash, its object budget, and the lines it collects. */
+struct Walk final {
+    std::uint32_t definitionHash{};
+    std::size_t objects{};
+    std::vector<Line> lines{};
+};
+
+/**
+ * Collects the lines below one tree object and computes how long it plays.
+ * @return False when the object or any descendant is malformed.
+ */
+[[nodiscard]] bool walk_object(std::span<const std::byte> bytes,
+                               std::size_t object,
+                               std::size_t depth,
+                               Walk& walk,
+                               float& seconds) {
+    seconds = 0.0F;
+    std::uint32_t classId = 0;
+    std::uint32_t hash = 0;
+    if (depth > kMaximumDepth || ++walk.objects > kMaximumObjects || object < sizeof classId
+        || !read_value(bytes, object - sizeof classId, classId)
+        || !read_value(bytes, object + kObjectHashOffset, hash) || hash != walk.definitionHash) {
+        return false;
+    }
+    if (classId == kLineClass) {
+        Line line{};
+        if (object + kLineByteCount > bytes.size()
+            || !read_take(bytes, object + kFirstTakeOffset, line.first)
+            || !read_take(bytes, object + kSecondTakeOffset, line.second)) {
+            return false;
+        }
+        seconds = (std::max)(line.first.seconds, line.second.seconds);
+        walk.lines.push_back(line);
+        return true;
+    }
+    std::size_t data = 0;
+    std::size_t count = 0;
+    if (classId == kSequenceClass) {
+        if (!read_array(bytes,
+                        object + kSequenceChildField,
+                        kPointerStride,
+                        kPointerArrayClass,
+                        data,
+                        count)) {
+            return false;
+        }
+        for (std::size_t index = 0; index < count; ++index) {
+            std::size_t child = 0;
+            float childSeconds = 0.0F;
+            if (!follow(bytes, data + index * kPointerStride, child)
+                || !walk_object(bytes, child, depth + 1U, walk, childSeconds)) {
+                return false;
+            }
+            seconds += childSeconds;
+        }
+        return true;
+    }
+    if (classId == kSelectionClass) {
+        if (!read_array(bytes,
+                        object + kSelectionEntryField,
+                        kSelectionEntryStride,
+                        kSelectionEntryClass,
+                        data,
+                        count)) {
+            return false;
+        }
+        for (std::size_t index = 0; index < count; ++index) {
+            std::size_t child = 0;
+            float childSeconds = 0.0F;
+            if (!follow(bytes, data + index * kSelectionEntryStride + kSelectionChildOffset, child)
+                || !walk_object(bytes, child, depth + 1U, walk, childSeconds)) {
+                return false;
+            }
+            seconds = (std::max)(seconds, childSeconds);
+        }
+        return true;
+    }
+    return false;
+}
+
+/** One tree root and its hash, in blob order. */
+struct Tree final {
+    std::uint32_t definitionHash{};
+    std::size_t root{};
+};
+
+} // namespace
+
+bool read(std::span<const std::byte> list, Snapshot& output) {
+    output = {};
+    std::size_t definitions = 0;
+    std::size_t definitionCount = 0;
+    std::size_t treeRows = 0;
+    std::size_t treeCount = 0;
+    if (!read_array(list,
+                    kDefinitionField,
+                    kDefinitionStride,
+                    kDefinitionArrayClass,
+                    definitions,
+                    definitionCount)
+        || !read_array(list, kTreeField, kTreeStride, kTreeArrayClass, treeRows, treeCount)) {
+        return false;
+    }
+    std::vector<Tree> trees(treeCount);
+    for (std::size_t index = 0; index < treeCount; ++index) {
+        const std::size_t row = treeRows + index * kTreeStride;
+        if (!read_value(list, row, trees[index].definitionHash)
+            || !follow(list, row + 8U, trees[index].root)) {
+            return false;
+        }
+    }
+    std::stable_sort(trees.begin(), trees.end(), [](const Tree& left, const Tree& right) {
+        return left.definitionHash < right.definitionHash;
+    });
+    for (auto first = trees.begin(); first != trees.end();) {
+        const auto last = std::find_if(first, trees.end(), [&](const Tree& tree) {
+            return tree.definitionHash != first->definitionHash;
+        });
+        if (last - first > 1) {
+            output.sharedHashes.push_back(
+                {first->definitionHash, static_cast<std::uint32_t>(last - first)});
+        }
+        first = last;
+    }
+
+    output.cues.resize(definitionCount);
+    for (std::size_t index = 0; index < definitionCount; ++index) {
+        Cue& cue = output.cues[index];
+        const std::size_t row = definitions + index * kDefinitionStride;
+        if (!read_value(list, row, cue.definitionHash) || !read_value(list, row + 4U, cue.seconds)
+            || !std::isfinite(cue.seconds) || cue.seconds < 0.0F) {
+            return false;
+        }
+        const auto range = std::equal_range(trees.begin(),
+                                            trees.end(),
+                                            Tree{cue.definitionHash, 0},
+                                            [](const Tree& left, const Tree& right) {
+                                                return left.definitionHash < right.definitionHash;
+                                            });
+        std::size_t matches = 0;
+        bool malformed = false;
+        std::vector<Line> matched{};
+        for (auto tree = range.first; tree != range.second; ++tree) {
+            Walk walk{};
+            walk.definitionHash = cue.definitionHash;
+            float seconds = 0.0F;
+            if (!walk_object(list, tree->root, 0, walk, seconds)) {
+                malformed = true;
+                continue;
+            }
+            if (std::fabs(seconds - cue.seconds) <= kSecondsTolerance) {
+                ++matches;
+                matched = std::move(walk.lines);
+            }
+        }
+        const std::size_t candidates = static_cast<std::size_t>(range.second - range.first);
+        if (candidates == 0) {
+            cue.status = CueStatus::missing;
+        } else if (matches == 1) {
+            cue.status = CueStatus::resolved;
+        } else {
+            cue.status = candidates > 1 && !malformed ? CueStatus::ambiguous : CueStatus::malformed;
+        }
+        if (matches != 1 || matched.size() > (std::numeric_limits<std::uint32_t>::max)()
+            || output.lines.size() > (std::numeric_limits<std::uint32_t>::max)() - matched.size()) {
+            continue;
+        }
+        cue.firstLine = static_cast<std::uint32_t>(output.lines.size());
+        cue.lineCount = static_cast<std::uint32_t>(matched.size());
+        output.lines.insert(output.lines.end(), matched.begin(), matched.end());
+    }
+    return true;
+}
+
+} // namespace sunrise::client::content::activity::sdk_generation::dialogue_list

Sunrise/src/client/content/activity/activity_sdk_dialogue_list.h

@@ -0,0 +1,65 @@
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <span>
+#include <vector>
+
+namespace sunrise::client::content::activity::sdk_generation::dialogue_list {
+
+/** One recorded take of a line: its audio, its localized text, and its length. */
+struct Take final {
+    std::uint32_t audioTag{};
+    std::uint32_t containerTag{};
+    std::uint32_t stringHash{};
+    float seconds{};
+};
+
+/** One line. Both takes carry the same text; the line lasts as long as the longer take. */
+struct Line final {
+    Take first{};
+    Take second{};
+};
+
+/** How one cue definition was matched to the content tree the list authors for it. */
+enum class CueStatus : std::uint8_t {
+    /** Exactly one tree carrying the definition hash matches the authored duration. */
+    resolved,
+    /** Several trees share the definition hash and none or several match the authored duration. */
+    ambiguous,
+    /** No tree carries the definition hash. */
+    missing,
+    /** The matched tree is malformed or its duration differs from the authored one. */
+    malformed,
+};
+
+/** One cue in list order. Lines are present only for resolved cues. */
+struct Cue final {
+    std::uint32_t definitionHash{};
+    /** Authored total: a sequence adds its children, a selection lasts its longest branch. */
+    float seconds{};
+    std::uint32_t firstLine{};
+    std::uint32_t lineCount{};
+    CueStatus status{CueStatus::missing};
+};
+
+/** One definition hash carried by more than one content tree. */
+struct SharedHash final {
+    std::uint32_t definitionHash{};
+    std::uint32_t treeCount{};
+};
+
+/** Every cue of one dialogue list, its lines in cue order, and the hashes several trees share. */
+struct Snapshot final {
+    std::vector<Cue> cues{};
+    std::vector<Line> lines{};
+    std::vector<SharedHash> sharedHashes{};
+};
+
+/**
+ * Reads one dialogue list blob.
+ * @return False when the definition or tree arrays are malformed; a bad tree only marks its cues.
+ */
+[[nodiscard]] bool read(std::span<const std::byte> list, Snapshot& output);
+
+} // namespace sunrise::client::content::activity::sdk_generation::dialogue_list

Sunrise/src/client/content/activity/activity_sdk_native_pack_text.cpp

@@ -16,6 +16,7 @@
 #include "../../../core/logging/log.h"
 #include "../../../middleware/content/packages/tables/activity_display_name_reader.h"
 #include "activity_sdk_dialogue_group_index.h"
+#include "activity_sdk_dialogue_list.h"
 #include "activity_sdk_native_pack_internal.h"
 
 namespace sunrise::client::content::activity::sdk_generation::native_pack_pipeline {
@@ -150,6 +151,34 @@ struct AuthoredDirectiveCandidate final {
     std::uint32_t elementCount{};
 };
 
+/** Logs one dialogue list fact that changes which cues carry lines, and its cue context. */
+void log_dialogue_list(const squads::DescriptorFact& descriptor,
+                       std::uint32_t listTag,
+                       const char* result,
+                       std::uint32_t definitionHash,
+                       std::uint32_t count,
+                       core::log::Level level) noexcept {
+    std::array<char, 192> line{};
+    const int written =
+        std::snprintf(line.data(),
+                      line.size(),
+                      "ev=activity_sdk_dialogue_list result=%s config=0x%08X offset=0x%X "
+                      "slot_row=%u list=0x%08X definition=0x%08X count=%u",
+                      result,
+                      static_cast<unsigned>(descriptor.configTag),
+                      static_cast<unsigned>(descriptor.descriptorOffset),
+                      static_cast<unsigned>(descriptor.slotIndex),
+                      static_cast<unsigned>(listTag),
+                      static_cast<unsigned>(definitionHash),
+                      static_cast<unsigned>(count));
+    if (written > 0) {
+        core::log::write(
+            core::log::Channel::client,
+            level,
+            {line.data(), (std::min)(static_cast<std::size_t>(written), line.size() - 1U)});
+    }
+}
+
 } // namespace
 
 /**
@@ -294,51 +323,53 @@ bool attach_authored_text(const topology_inventory::Snapshot& topology,
                 if (resource->classId != format::kDialogueAuthoredListClass) {
                     continue;
                 }
-                std::size_t groups = 0;
-                std::size_t groupCount = 0;
-                if (!read_array(
-                        bytes, 0x18U, 16U, format::kDialogueGroupArrayClass, groups, groupCount)) {
+                dialogue_list::Snapshot list{};
+                if (!dialogue_list::read(bytes, list)) {
+                    log_dialogue_list(
+                        descriptor, resourceTag, "malformed", 0, 0, core::log::Level::warn);
                     continue;
                 }
-                std::size_t definitions = 0;
-                std::size_t definitionCount = 0;
-                if (!read_array(bytes,
-                                8U,
-                                8U,
-                                format::kDialogueDefinitionArrayClass,
-                                definitions,
-                                definitionCount)) {
-                    continue;
-                }
-                std::vector<dialogue_groups::Span> groupIndex{};
-                if (!dialogue_groups::build(bytes, groups, groupCount, groupIndex)) {
-                    continue;
+                for (const dialogue_list::SharedHash& shared : list.sharedHashes) {
+                    log_dialogue_list(descriptor,
+                                      resourceTag,
+                                      "shared_hash",
+                                      shared.definitionHash,
+                                      shared.treeCount,
+                                      core::log::Level::info);
                 }
-                for (std::size_t cue = 0; cue < definitionCount; ++cue) {
-                    std::uint32_t definitionHash = 0;
-                    if (!read_value(bytes, definitions + cue * 8U, definitionHash)
-                        || definitionHash == 0 || definitionHash == 0x811C9DC5U) {
+                for (std::size_t cue = 0; cue < list.cues.size(); ++cue) {
+                    const dialogue_list::Cue& row = list.cues[cue];
+                    if (row.status == dialogue_list::CueStatus::ambiguous
+                        || row.status == dialogue_list::CueStatus::missing
+                        || row.status == dialogue_list::CueStatus::malformed) {
+                        log_dialogue_list(
+                            descriptor,
+                            resourceTag,
+                            row.status == dialogue_list::CueStatus::ambiguous ? "cue_ambiguous"
+                            : row.status == dialogue_list::CueStatus::missing ? "cue_missing"
+                                                                              : "cue_malformed",
+                            row.definitionHash,
+                            static_cast<std::uint32_t>(cue),
+                            core::log::Level::warn);
                         continue;
                     }
-                    dialogue_groups::Span group{};
-                    if (!dialogue_groups::find(groupIndex, definitionHash, group)) {
-                        continue;
-                    }
-                    for (std::size_t offset = group.begin; offset + 8U <= group.end; offset += 4U) {
-                        std::uint32_t containerTag = 0;
-                        std::uint32_t stringHash = 0;
-                        const CachedTag* container = nullptr;
-                        if (!read_value(bytes, offset, containerTag)
-                            || !read_value(bytes, offset + 4U, stringHash)
-                            || containerTag < 0x80A12000U || containerTag >= 0xC0000000U
-                            || !package(containerTag, container) || container == nullptr
-                            || container->classId != display::kStringContainerClass) {
-                            continue;
+                    // Both takes of a line normally reference the same text; each distinct
+                    // reference is still a candidate, and the Lua publisher folds equal texts.
+                    for (std::uint32_t offset = 0; offset < row.lineCount; ++offset) {
+                        const dialogue_list::Line& line = list.lines[row.firstLine + offset];
+                        for (const dialogue_list::Take* take : {&line.first, &line.second}) {
+                            const CachedTag* container = nullptr;
+                            if (take->containerTag == 0
+                                || take->containerTag == format::kAbsentIndex
+                                || !package(take->containerTag, container) || container == nullptr
+                                || container->classId != display::kStringContainerClass) {
+                                continue;
+                            }
+                            dialogueCandidates.push_back({{take->containerTag, take->stringHash},
+                                                          descriptor.slotIndex,
+                                                          static_cast<std::uint32_t>(cue),
+                                                          row.definitionHash});
                         }
-                        dialogueCandidates.push_back({{containerTag, stringHash},
-                                                      descriptor.slotIndex,
-                                                      static_cast<std::uint32_t>(cue),
-                                                      definitionHash});
                     }
                 }
             } else {