feat: capture and encode real per-character appearance

Both character encoders (family-4 and the family-3/family-0 character
record) wrote a fixed 36-byte header for every character, unconditionally
-- authored client-side ids, not extracted data, identical for every
character regardless of race or gender. Real per-character appearance
(face, hair, skin, etc.) was never captured: the creation request's
appearance header was parsed only far enough to read race/gender/class,
then the remaining 234 bits were explicitly skipped.

Adds the missing half: parses the full appearance header from the
creation request (8/16/32-bit fields, three different biases, mapped
into the record's own little-endian byte layout), stores it per
character, and has both encoders use it when present, falling back to
the fixed header only for a character that predates this (existing
saves keep working via a schema migration).

Persistence: appearance_header/appearance_header_valid added to the
characters table (schema v2 -> v3, migrated in place for existing
databases). The generic Statement bind/column helpers gained BLOB
support for fixed-size byte arrays, since nothing previously stored in
this database needed anything but integers, floats, or text.

Sunrise/resources/database/investment_defaults.sql

@@ -1,5 +1,5 @@
 INSERT INTO account VALUES (1, -7013740688821976832, 1);
-INSERT INTO characters VALUES (0, -7013740688821976831, 0, 0, 1, 50, 1, 1.0, 308080871, 1, 65535, -1, 57);
+INSERT INTO characters VALUES (0, -7013740688821976831, 0, 0, 1, 50, 1, 1.0, 308080871, 1, 65535, -1, 57, X'000000000000000000000000000000000000000000000000000000000000000000000000', 0);
 INSERT INTO items VALUES (0, 0, 0, 4611686018427387905, 3843477312, 106, 1, 0, 0, 1, 12, 4, 7, 10, 11, 2, 0);
 INSERT INTO sockets VALUES (4611686018427387905, 0, 2874284214);
 INSERT INTO sockets VALUES (4611686018427387905, 1, 3433396027);

Sunrise/resources/database/investment_schema.sql

@@ -1,5 +1,5 @@
 PRAGMA application_id = 1397902921;
-PRAGMA user_version = 2;
+PRAGMA user_version = 3;
 
 CREATE TABLE account (
     id INTEGER PRIMARY KEY CHECK (id = 1),
@@ -20,7 +20,9 @@ CREATE TABLE characters (
     content_bypass INTEGER NOT NULL CHECK (content_bypass IN (0, 1)),
     equipped_title INTEGER NOT NULL CHECK (equipped_title BETWEEN 0 AND 65535),
     acquired_subclass_mask INTEGER NOT NULL,
-    next_inventory_serial INTEGER NOT NULL CHECK (next_inventory_serial BETWEEN 0 AND 4294967295)
+    next_inventory_serial INTEGER NOT NULL CHECK (next_inventory_serial BETWEEN 0 AND 4294967295),
+    appearance_header BLOB NOT NULL CHECK (length(appearance_header) = 36),
+    appearance_header_valid INTEGER NOT NULL CHECK (appearance_header_valid IN (0, 1))
 ) STRICT;
 
 CREATE TABLE items (

Sunrise/src/middleware/datagen/character_record/character_record_encoder.cpp

@@ -42,9 +42,9 @@ constexpr std::size_t kCardFlagOffset = 8;
         static_cast<std::int8_t>(character.gender),
         static_cast<std::int8_t>(character.characterClass),
     };
-    std::memcpy(identity.headerBlock.data(),
-                layout::kHeaderBlockBytes.data(),
-                layout::kHeaderBlockBytes.size());
+    const auto& headerSource =
+        character.appearanceHeaderValid ? character.appearanceHeader : layout::kHeaderBlockBytes;
+    std::memcpy(identity.headerBlock.data(), headerSource.data(), headerSource.size());
 
     output = {};
     appearance::apply_sentinels(output);

Sunrise/src/middleware/datagen/family4/character/character_encoder.cpp

@@ -270,9 +270,10 @@ bool encode(const state::CharacterState& state,
     object.identity.race = static_cast<std::int8_t>(state.race);
     object.identity.gender = static_cast<std::int8_t>(state.gender);
     object.identity.characterClass = static_cast<std::int8_t>(state.characterClass);
-    std::memcpy(object.customisationHeader.data(),
-                character_record::layout::kHeaderBlockBytes.data(),
-                character_record::layout::kHeaderBlockBytes.size());
+    const auto& headerSource = state.appearanceHeaderValid
+                                   ? state.appearanceHeader
+                                   : character_record::layout::kHeaderBlockBytes;
+    std::memcpy(object.customisationHeader.data(), headerSource.data(), headerSource.size());
     object.lastOrbitedDestination = state.lastOrbitedDestination;
     // No previous activity is tracked, and a zero here would name the orbit activity.
     object.previousActivityIndex = kAbsentActivityIndex;

Sunrise/src/middleware/web_service/messages/opcode501_request_codec.cpp

@@ -5,17 +5,29 @@
 namespace sunrise::middleware::web_service::messages::opcode501 {
 namespace {
 
-/** The identity triple and appearance header end here; the 64-byte name follows. Only the
- * triple is decoded; the rest is skipped as one span. */
+/** The identity triple and appearance header end here; the 64-byte name follows. */
 constexpr std::size_t kIdentityAndHeaderBits = 261;
-/** Each triple entry is a 1-bit presence flag plus an 8-bit +0x80-biased value. */
-constexpr std::uint8_t kTripleEntryBits = 9;
+
 constexpr std::uint64_t kByteBias = 0x80U;
-/** Bits left to skip after the triple: the appearance header and the request's trailing field. */
-constexpr std::size_t kSkipBits = kIdentityAndHeaderBits - 3U * kTripleEntryBits;
+constexpr std::uint64_t kWordBias = 0x8000U;
+
+/** Writes the record's little-endian byte order, which is not the wire's bit order. */
+void put8(state::AppearanceHeader& header, std::size_t offset, std::uint64_t value) noexcept {
+    header[offset] = static_cast<std::uint8_t>(value & 0xFFU);
+}
+
+void put16(state::AppearanceHeader& header, std::size_t offset, std::uint64_t value) noexcept {
+    header[offset] = static_cast<std::uint8_t>(value & 0xFFU);
+    header[offset + 1U] = static_cast<std::uint8_t>((value >> 8U) & 0xFFU);
+}
+
+void put32(state::AppearanceHeader& header, std::size_t offset, std::uint64_t value) noexcept {
+    put16(header, offset, value & 0xFFFFU);
+    put16(header, offset + 2U, (value >> 16U) & 0xFFFFU);
+}
 
 /**
- * Reads one 1-bit presence flag followed by an 8-bit +0x80-biased value.
+ * Reads one 1-bit presence flag followed by an 8-bit biased value.
  * @return True when the field is present and readable. An absent field fails closed: nothing on
  *         the wire carries the value that was skipped, so there is nothing to fall back to.
  */
@@ -32,6 +44,53 @@ constexpr std::size_t kSkipBits = kIdentityAndHeaderBits - 3U * kTripleEntryBits
     return true;
 }
 
+/** Reads an 8-bit +0x80-biased header field straight into the record block. */
+[[nodiscard]] bool
+copy_byte(encoding::bits::Reader& reader, state::AppearanceHeader& header, std::size_t offset) noexcept {
+    std::uint64_t raw = 0;
+    if (!reader.read(8, raw)) {
+        return false;
+    }
+    put8(header, offset, raw - kByteBias);
+    return true;
+}
+
+/** Reads a 16-bit +0x8000-biased header field straight into the record block. */
+[[nodiscard]] bool
+copy_word(encoding::bits::Reader& reader, state::AppearanceHeader& header, std::size_t offset) noexcept {
+    std::uint64_t raw = 0;
+    if (!reader.read(16, raw)) {
+        return false;
+    }
+    put16(header, offset, raw - kWordBias);
+    return true;
+}
+
+/**
+ * Byte offsets of every appearance-header field inside the 36-byte record block. The gaps at 5,
+ * 26 and 33 are the struct's own alignment padding and stay zero.
+ */
+constexpr std::size_t kOffsetF0 = 0;
+constexpr std::size_t kOffsetF2 = 2;
+constexpr std::size_t kOffsetF3 = 3;
+constexpr std::size_t kOffsetF4 = 4;
+constexpr std::size_t kOffsetF6 = 6;
+constexpr std::size_t kOffsetF8 = 8;
+constexpr std::size_t kOffsetF10 = 10;
+constexpr std::size_t kOffsetF12 = 12;
+constexpr std::size_t kOffsetF18 = 18;
+constexpr std::size_t kOffsetF24 = 24;
+constexpr std::size_t kOffsetF28 = 28;
+constexpr std::size_t kOffsetF32 = 32;
+
+/** Both triples of 16-bit customization indices hold 3 entries. */
+constexpr std::size_t kIndexTripleCount = 3;
+/** Each entry in those triples is 2 bytes wide in the record block. */
+constexpr std::size_t kIndexTripleStride = 2;
+
+/** The record requires 1 in the trailing field; the request always carries 0 there. */
+constexpr std::uint64_t kRecordTrailingField = 1;
+
 /** Highest valid race, gender and class wire values, matching the state enums. */
 constexpr std::uint8_t kMaxRace = 2;
 constexpr std::uint8_t kMaxGender = 1;
@@ -53,7 +112,40 @@ bool parse_request(const Message& message, Request& request) noexcept {
         || request.characterClass > kMaxClass) {
         return false;
     }
-    return reader.skip(kSkipBits);
+
+    auto& header = request.appearanceHeader;
+    if (!copy_word(reader, header, kOffsetF0) || !copy_byte(reader, header, kOffsetF2)
+        || !copy_byte(reader, header, kOffsetF3) || !copy_byte(reader, header, kOffsetF4)
+        || !copy_word(reader, header, kOffsetF6) || !copy_word(reader, header, kOffsetF8)
+        || !copy_word(reader, header, kOffsetF10)) {
+        return false;
+    }
+    for (std::size_t index = 0; index < kIndexTripleCount; ++index) {
+        if (!copy_word(reader, header, kOffsetF12 + index * kIndexTripleStride)) {
+            return false;
+        }
+    }
+    for (std::size_t index = 0; index < kIndexTripleCount; ++index) {
+        if (!copy_word(reader, header, kOffsetF18 + index * kIndexTripleStride)) {
+            return false;
+        }
+    }
+    if (!copy_word(reader, header, kOffsetF24)) {
+        return false;
+    }
+    std::uint64_t emptyHashSentinel = 0;
+    if (!reader.read(32, emptyHashSentinel)) {
+        return false;
+    }
+    put32(header, kOffsetF28, emptyHashSentinel);
+
+    // Read but not copied: the request's own value is 0 and the record rejects it.
+    std::uint64_t requestTrailingField = 0;
+    if (!reader.read(2, requestTrailingField)) {
+        return false;
+    }
+    put8(header, kOffsetF32, kRecordTrailingField);
+    return true;
 }
 
 } // namespace sunrise::middleware::web_service::messages::opcode501

Sunrise/src/middleware/web_service/messages/opcode501_request_codec.h

@@ -2,31 +2,44 @@
 
 #include <cstdint>
 
+#include "../../../state/account/account_state.h"
 #include "../web_service_envelope.h"
 #include "opcode501_codec.h"
 
 namespace sunrise::middleware::web_service::messages::opcode501 {
 
-/** The create-character request's identity triple: race, gender and class. */
+/** The create-character request: the identity triple plus the authored appearance header. */
 struct Request {
     std::uint8_t characterClass{};
     std::uint8_t gender{};
     std::uint8_t race{};
+    /**
+     * The appearance header in record byte order, ready to memcpy into either character
+     * encoder's block. Its values index per-race customization tables, so it is only
+     * meaningful beside the `race` it arrived with.
+     */
+    state::AppearanceHeader appearanceHeader{};
 };
 
 /**
- * Parses the create-character request's identity triple.
+ * Parses the create-character request body.
  *
- * The payload is a big-endian bitstream, not a byte-aligned struct. Wire order is depth-first
- * over `{ int8 triple[3]; CharHeader header; char name[64]; }`, where `triple` is race, gender
- * and class in that order; each `triple` entry is a 1-bit presence flag followed by 8 value bits
- * biased by +0x80. This build has no per-character authored appearance to capture, so the header
- * and name that follow are consumed and discarded rather than decoded field by field.
+ * The payload is a big-endian bitstream, not a byte-aligned struct: every field is written at
+ * whatever bit offset the previous one ended on. Wire order is depth-first over
+ * `{ int8 triple[3]; CharHeader header; char name[64]; }`, where `triple` is race, gender and
+ * class in that order. Each `triple` entry is a 1-bit presence flag followed by 8 value bits
+ * biased by +0x80; the header's fields are 8, 16 or 32 bits biased by +0x80, +0x8000 or not at
+ * all; and the 64-byte name begins at bit 261. This parser consumes the first 261 bits and
+ * ignores the name, which every observed capture leaves as the placeholder "Player".
+ *
+ * The header is emitted in the record's own little-endian struct layout rather than as parsed
+ * fields, since both character encoders want it as an opaque block. Its last field is the one
+ * value that is not copied through: the request carries 0 where the record requires 1.
  *
  * @param message Parsed Web Service envelope.
- * @param request Receives the parsed identity triple.
- * @return True when the payload holds the full identity-and-header span, every presence flag is
- *         set, and race, gender and class are all in range.
+ * @param request Receives the parsed creation fields.
+ * @return True when the payload holds all 261 bits, every presence flag is set, and race,
+ *         gender and class are all in range.
  */
 [[nodiscard]] bool parse_request(const Message& message, Request& request) noexcept;
 

Sunrise/src/server/bap/encrypted/body/bap_service_body.cpp

@@ -278,8 +278,11 @@ bool process(const ServiceRoute& route,
             middleware::web_service::messages::opcode501::Request creation{};
             std::uint64_t characterSoid = 0;
             if (!middleware::web_service::messages::opcode501::parse_request(message, creation)
-                || !state::create_character(
-                    creation.characterClass, creation.gender, creation.race, characterSoid)) {
+                || !state::create_character(creation.characterClass,
+                                            creation.gender,
+                                            creation.race,
+                                            creation.appearanceHeader,
+                                            characterSoid)) {
                 core::log::write(core::log::Channel::server,
                                  core::log::Level::warn,
                                  "ev=ws501 stage=create result=fail");

Sunrise/src/state/account/account_state.h

@@ -11,6 +11,10 @@ namespace sunrise::state {
 
 /** One account can own at most the 3 playable character slots. */
 inline constexpr std::size_t kCharacterCapacity = 3;
+/** The character record's own appearance-header block size, record byte order. */
+inline constexpr std::size_t kAppearanceHeaderSize = 36;
+/** Opaque authored appearance header, ready to memcpy into either character encoder's block. */
+using AppearanceHeader = std::array<std::uint8_t, kAppearanceHeaderSize>;
 /** A server-authored dismantle policy: a few rows per rarity and gear class. */
 inline constexpr std::size_t kDismantleRewardPolicyCapacity = 32;
 /** Native sentinel used when a character has no title equipped. */
@@ -165,6 +169,11 @@ struct CharacterState {
     account::inventory::CharacterStacks stacks;
     /** Next row generation; equip transactions consume two values for the two moved items. */
     std::uint32_t nextInventorySerial{};
+    /** Authored appearance header captured at creation. Meaningless without its own `race`. */
+    AppearanceHeader appearanceHeader{};
+    /** False for a character predating appearance capture; both encoders fall back to the
+     * fixed default header when this is unset. */
+    bool appearanceHeaderValid{};
 };
 
 /** Call-local account snapshot shared by backend object families. */

Sunrise/src/state/investment/investment_account_store.cpp

@@ -22,7 +22,9 @@ bool read_characters(AccountState& output) noexcept {
                           character.contentBypass,
                           character.equippedTitleRecordIndex,
                           character.acquiredSubclassAbilityMask,
-                          character.nextInventorySerial)
+                          character.nextInventorySerial,
+                          character.appearanceHeader,
+                          character.appearanceHeaderValid)
             || slot != output.characterCount || slot >= output.characters.size()) {
             return false;
         }
@@ -60,7 +62,7 @@ bool read_rewards(AccountState& output) noexcept {
 
 /** Characters are saved before their items to satisfy foreign keys. */
 bool write_characters(const AccountState& value) noexcept {
-    Statement rows("INSERT INTO characters VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)");
+    Statement rows("INSERT INTO characters VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)");
     for (std::size_t slot = 0; slot < value.characterCount; ++slot) {
         const auto& character = value.characters[slot];
         if (!rows.write(slot,
@@ -75,7 +77,9 @@ bool write_characters(const AccountState& value) noexcept {
                         character.contentBypass,
                         character.equippedTitleRecordIndex,
                         character.acquiredSubclassAbilityMask,
-                        character.nextInventorySerial)) {
+                        character.nextInventorySerial,
+                        character.appearanceHeader,
+                        character.appearanceHeaderValid)) {
             return false;
         }
     }

Sunrise/src/state/investment/investment_database.cpp

@@ -119,13 +119,15 @@ bool open(std::string_view path,
                 && execute(preferenceSchema.c_str()) && execute(preferenceDefaults.c_str())
                 && transaction.commit();
     } else if (ready) {
-        // Version 2 adds account preferences and per-item seen state.
-        constexpr int kSchemaVersion = 2;
+        // Version 3 adds the per-character appearance header. Version 2 added account
+        // preferences and per-item seen state.
+        constexpr int kSchemaVersion = 3;
         constexpr int kApplicationId = 1397902921;
         int application = 0;
         Statement query("PRAGMA application_id");
-        ready = (version == 1 || version == kSchemaVersion) && query.step() == SQLITE_ROW
-                && query.column(0, application) && application == kApplicationId;
+        ready = (version == 1 || version == 2 || version == kSchemaVersion)
+                && query.step() == SQLITE_ROW && query.column(0, application)
+                && application == kApplicationId;
     }
     if (ready && version == 1) {
         Transaction transaction;
@@ -137,6 +139,25 @@ bool open(std::string_view path,
                 "IN(0,1));")
             && execute(preferenceSchema.c_str()) && execute(preferenceDefaults.c_str())
             && execute("PRAGMA user_version=2") && transaction.commit();
+        if (ready) {
+            version = 2;
+        }
+    }
+    if (ready && version == 2) {
+        Transaction transaction;
+        // A character predating this column has no captured header; the zero blob is never
+        // read, since appearance_header_valid stays 0 and both encoders fall back to the fixed
+        // default header for it.
+        ready = transaction.ready()
+                // SQLite's ADD COLUMN refuses a non-constant default (zeroblob(36) included),
+                // so the fill value has to be a literal blob, not a function call.
+                && execute(
+                    "ALTER TABLE characters ADD COLUMN appearance_header BLOB NOT NULL "
+                    "DEFAULT (X'000000000000000000000000000000000000000000000000000000000000"
+                    "000000000000') CHECK (length(appearance_header) = 36);"
+                    "ALTER TABLE characters ADD COLUMN appearance_header_valid INTEGER "
+                    "NOT NULL DEFAULT 0 CHECK (appearance_header_valid IN (0, 1));")
+                && execute("PRAGMA user_version=3") && transaction.commit();
     }
     if (!ready) {
         shutdown();

Sunrise/src/state/investment/store_internal.h

@@ -1,7 +1,9 @@
 #pragma once
 
+#include <array>
 #include <bit>
 #include <cmath>
+#include <cstring>
 #include <limits>
 #include <mutex>
 #include <sqlite3.h>
@@ -12,6 +14,11 @@
 
 namespace sunrise::state::investment::store {
 
+/** True for a fixed-size byte array, the only shape this store binds as a BLOB. */
+template <typename T> inline constexpr bool kIsByteArray = false;
+template <std::size_t N>
+inline constexpr bool kIsByteArray<std::array<std::uint8_t, N>> = true;
+
 extern sqlite3* g_database;
 extern std::recursive_mutex g_mutex;
 extern std::uint64_t g_failureSerial;
@@ -61,7 +68,14 @@ public:
         if (statement_ == nullptr) {
             return false;
         }
-        if constexpr (std::is_enum_v<T>) {
+        if constexpr (kIsByteArray<T>) {
+            if (sqlite3_column_type(statement_, index) != SQLITE_BLOB
+                || static_cast<std::size_t>(sqlite3_column_bytes(statement_, index))
+                       != value.size()) {
+                return false;
+            }
+            std::memcpy(value.data(), sqlite3_column_blob(statement_, index), value.size());
+        } else if constexpr (std::is_enum_v<T>) {
             std::underlying_type_t<T> raw{};
             if (!column(index, raw)) {
                 return false;
@@ -113,8 +127,12 @@ public:
 
 private:
     /** SQL integers preserve all 64 identity bits; text bindings copy their input. */
-    template <typename T> [[nodiscard]] bool bind(int index, T value) noexcept {
-        if constexpr (std::is_convertible_v<T, std::string_view>) {
+    template <typename T> [[nodiscard]] bool bind(int index, const T& value) noexcept {
+        if constexpr (kIsByteArray<T>) {
+            return sqlite3_bind_blob64(
+                       statement_, index, value.data(), value.size(), copy_text())
+                   == SQLITE_OK;
+        } else if constexpr (std::is_convertible_v<T, std::string_view>) {
             const std::string_view textValue(value);
             return sqlite3_bind_text64(statement_,
                                        index,

Sunrise/src/state/runtime/runtime.h

@@ -478,6 +478,8 @@ void publish_sign_in_time(std::uint64_t seconds) noexcept;
  * @param characterClass Character class (0=titan, 1=hunter, 2=warlock).
  * @param gender Character gender (0=male, 1=female).
  * @param race Character race (0=human, 1=awoken, 2=exo).
+ * @param appearanceHeader Authored appearance header captured from the creation request, in
+ *                        record byte order. Only meaningful beside the `race` it arrived with.
  * @param characterSoid Receives the SOID of the newly created character.
  * @return True when the character was created, the account still validates, and the write
  *         committed.
@@ -485,6 +487,7 @@ void publish_sign_in_time(std::uint64_t seconds) noexcept;
 [[nodiscard]] bool create_character(std::uint8_t characterClass,
                                     std::uint8_t gender,
                                     std::uint8_t race,
+                                    const AppearanceHeader& appearanceHeader,
                                     std::uint64_t& characterSoid) noexcept;
 
 /**

Sunrise/src/state/runtime/state_account_roster_runtime.cpp

@@ -145,6 +145,7 @@ constexpr std::array<std::array<StarterItem, 7>, 3> kPerClassStarterItems{{
 bool create_character(std::uint8_t characterClass,
                       std::uint8_t gender,
                       std::uint8_t race,
+                      const AppearanceHeader& appearanceHeader,
                       std::uint64_t& characterSoid) noexcept {
     characterSoid = 0;
     if (characterClass > static_cast<std::uint8_t>(CharacterClass::warlock)
@@ -210,6 +211,10 @@ bool create_character(std::uint8_t characterClass,
     character.race = static_cast<CharacterRace>(race);
     character.gender = static_cast<CharacterGender>(gender);
     character.characterClass = static_cast<CharacterClass>(characterClass);
+    // Captured from the creation request, so the character keeps the face the player built
+    // instead of the single fixed header every character used to share.
+    character.appearanceHeader = appearanceHeader;
+    character.appearanceHeaderValid = true;
 
     // Select the new character and deselect all others.
     for (std::size_t i = 0; i < candidate.characterCount; ++i) {