diff --git a/README.md b/README.md index 21ec089e..6438168a 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Documentation for the library is located [here](imessage-database/README.md). ### Supported Features -This crate supports every iMessage feature as of macOS Tahoe 26.5 (25F71) and iOS 26.5.1 (23F81): +This crate supports every iMessage feature as of macOS Tahoe 26.5.1 (25F80) and iOS 26.5.1 (23F81): - iMessage, RCS, SMS, and MMS - Multi-part messages diff --git a/docs/features.md b/docs/features.md index 9317f894..ff738f08 100644 --- a/docs/features.md +++ b/docs/features.md @@ -124,7 +124,17 @@ This tool targets the current latest public release for Messages.app. It may wor - `clone, basic, full`: saved as an `svg` file - Digital Touch - Parses the protobuf payload to extract [Digital Touch](https://support.apple.com/guide/ipod-touch/send-a-digital-touch-effect-iph3fadba219/ios) message data - - Displayed as text that describes the type of message sent in HTML and TXT exports + - Supports all Digital Touch effects: + - Taps + - Sketches + - Kisses + - Heartbeats, including heartbreaks + - Fireballs + - Photos and videos + - HTML exports render a static frame depicting the captured data as an embedded `svg` on a black `4:5` canvas + - A photo is shown as the canvas backdrop, with any sketch drawn over it + - A video is shown as an embedded video player + - TXT exports describe each effect on a single line, including stroke and point counts or beats per minute - Duplicated group chats - Handles (participants) and chats (threads) can become duplicated - On startup: diff --git a/imessage-database/src/error/digital_touch.rs b/imessage-database/src/error/digital_touch.rs new file mode 100644 index 00000000..7e13359c --- /dev/null +++ b/imessage-database/src/error/digital_touch.rs @@ -0,0 +1,56 @@ +/*! + Errors that can happen when parsing `digital touch` data. +*/ + +use std::fmt::{Display, Formatter, Result}; + +/// Errors that can happen when parsing [`digital touch`](crate::message_types::digital_touch) data. +#[derive(Debug)] +pub enum DigitalTouchError { + /// Wraps an error returned by the protobuf parser. + ProtobufError(protobuf::Error), + /// The `TouchKind` discriminant was not a value we know how to parse. + UnknownDigitalTouchKind(i32), + /// Two parallel arrays that are expected to describe the same events had + /// different lengths (name, length, other name, other length). + ArraysDoNotMatch(&'static str, usize, &'static str, usize), + /// A length-prefixed stroke ran past the end of its buffer (needed, available). + InvalidStrokesLength(usize, usize), + /// Wraps an error returned while reading an embedded `NSKeyedArchiver` archive. + ArchiveError(plist::Error), +} + +impl std::error::Error for DigitalTouchError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match self { + DigitalTouchError::ProtobufError(why) => Some(why), + DigitalTouchError::ArchiveError(why) => Some(why), + _ => None, + } + } +} + +impl Display for DigitalTouchError { + fn fmt(&self, fmt: &mut Formatter<'_>) -> Result { + match self { + DigitalTouchError::ProtobufError(why) => { + write!(fmt, "failed to parse digital touch protobuf: {why}") + } + DigitalTouchError::UnknownDigitalTouchKind(kind) => { + write!(fmt, "unknown digital touch kind: {kind}") + } + DigitalTouchError::ArraysDoNotMatch(n1, v1, n2, v2) => { + write!(fmt, "mismatched array lengths: {n1} ({v1}) != {n2} ({v2})") + } + DigitalTouchError::InvalidStrokesLength(needed, available) => { + write!( + fmt, + "stroke needs {needed} bytes but only {available} remain" + ) + } + DigitalTouchError::ArchiveError(why) => { + write!(fmt, "failed to read digital touch media archive: {why}") + } + } + } +} diff --git a/imessage-database/src/error/mod.rs b/imessage-database/src/error/mod.rs index e8bebb99..f00f8d45 100644 --- a/imessage-database/src/error/mod.rs +++ b/imessage-database/src/error/mod.rs @@ -3,6 +3,7 @@ */ pub mod attachment; +pub mod digital_touch; pub mod handwriting; pub mod message; pub mod plist; diff --git a/imessage-database/src/error/plist.rs b/imessage-database/src/error/plist.rs index 2d377da8..f6456eb2 100644 --- a/imessage-database/src/error/plist.rs +++ b/imessage-database/src/error/plist.rs @@ -4,6 +4,7 @@ use crabstep::error::TypedStreamError; +use crate::error::digital_touch::DigitalTouchError; use crate::error::handwriting::HandwritingError; use crate::error::streamtyped::StreamTypedError; use std::fmt::{Display, Formatter, Result}; @@ -33,8 +34,8 @@ pub enum PlistParseError { TypedStreamError(TypedStreamError), /// Error from handwriting data parsing HandwritingError(HandwritingError), - /// Error parsing Digital Touch message - DigitalTouchError, + /// Error from Digital Touch data parsing + DigitalTouchError(DigitalTouchError), /// Error parsing a poll message PollError, /// Exceeded the maximum UID-reference resolution depth (likely a reference cycle) @@ -68,9 +69,7 @@ impl Display for PlistParseError { } PlistParseError::StreamTypedError(why) => write!(fmt, "{why}"), PlistParseError::HandwritingError(why) => write!(fmt, "{why}"), - PlistParseError::DigitalTouchError => { - write!(fmt, "Unable to parse Digital Touch Message!") - } + PlistParseError::DigitalTouchError(why) => write!(fmt, "{why}"), PlistParseError::TypedStreamError(typed_stream_error) => { write!(fmt, "TypedStream error: {typed_stream_error}") } @@ -89,6 +88,7 @@ impl std::error::Error for PlistParseError { PlistParseError::StreamTypedError(e) => Some(e), PlistParseError::TypedStreamError(e) => Some(e), PlistParseError::HandwritingError(e) => Some(e), + PlistParseError::DigitalTouchError(e) => Some(e), _ => None, } } diff --git a/imessage-database/src/message_types/digital_touch/digital_touch.proto b/imessage-database/src/message_types/digital_touch/digital_touch.proto index 6691ab3e..7528fb9a 100644 --- a/imessage-database/src/message_types/digital_touch/digital_touch.proto +++ b/imessage-database/src/message_types/digital_touch/digital_touch.proto @@ -1,6 +1,13 @@ syntax = 'proto3'; package digital_touch; +// Outer envelope shared by every Digital Touch effect. +// +// The raw `payload_data` blob decodes into this message. `TouchPayload` holds a +// second, effect-specific protobuf selected by `TouchKind`. The sender also +// stores a creation timestamp, a global color, and two trailing fields that are +// always zero in observed data; those are left out here and parsed as unknown +// fields because the per-effect payloads carry the meaningful state. message BaseMessage { TouchKind TouchKind = 1; bytes TouchPayload = 3; @@ -14,39 +21,57 @@ enum TouchKind { Heartbeat = 3; // Also broken heart Sketch = 4; // 5? - // 6? + Media = 6; // Still image with optional overlays, or video Kiss = 7; Fireball = 8; } +// Coordinates throughout are stored as little-endian `uint16` pairs (x, y) where +// the full `0..=65535` range maps to `0.0..=1.0` of the canvas (a 4:5 portrait +// on device), origin bottom-left (y grows upward). Colors are four bytes in RGBA +// order. Delays are little-endian `uint16` values in milliseconds. + message TapMessage { - bytes Delays = 2; - bytes Location = 3; - bytes Color = 4; + bytes Delays = 2; // [uint16] one delay (ms) per tap + bytes Location = 3; // [(uint16 x, uint16 y)] one point per tap + bytes Color = 4; // [RGBA] one color per tap } message SketchMessage { - int64 StrokesCount = 1; + int64 StrokesCount = 1; // number of strokes encoded in `Strokes` + // Concatenated strokes. Each stroke is: + // uint16 ?? (per-stroke header, observed as 0) + // uint16 Count (number of points in the stroke) + // [Count](uint16 x, uint16 y) bytes Strokes = 2; - bytes Colors = 3; + bytes Colors = 3; // [RGBA] one color per stroke } message KissMessage { - bytes Delays = 1; - bytes Points = 2; - bytes Rotations = 3; + bytes Delays = 1; // [uint16] one delay (ms) per kiss + bytes Points = 2; // [(uint16 x, uint16 y)] one point per kiss + bytes Rotations = 3; // [uint16] one rotation per kiss, in milliradians } message HeartbeatMessage { float BPM = 1; - uint64 Duration = 2; - float HeartBrokenAt = 6; + uint64 Duration = 2; // seconds + float HeartBrokenAt = 6; // seconds into the animation, 0 when not broken } message FireballMessage { - float Duration = 1; - float StartX = 2; + float Duration = 1; // seconds + float StartX = 2; // start offset, centered (roughly -1.0..=1.0) float StartY = 3; - bytes Delays = 4; - bytes Points = 5; -} \ No newline at end of file + bytes Delays = 4; // [uint16] one delay (ms) per point + bytes Points = 5; // [(uint16 x, uint16 y)] the dragged path +} + +message MediaMessage { + // For image media, an `NSKeyedArchiver` archive holding an `NSMutableArray` + // of overlay effects. Each element is `NSData` containing a complete nested + // `BaseMessage`; supported overlays are any non-media `TouchKind`. The array + // is empty when nothing was drawn on top. + bytes Archive = 2; + uint64 MediaType = 4; // 1 = video, 2 = image +} diff --git a/imessage-database/src/message_types/digital_touch/digital_touch_proto.rs b/imessage-database/src/message_types/digital_touch/digital_touch_proto.rs index d17f08bd..5eba8ee0 100644 --- a/imessage-database/src/message_types/digital_touch/digital_touch_proto.rs +++ b/imessage-database/src/message_types/digital_touch/digital_touch_proto.rs @@ -1008,6 +1008,146 @@ impl ::protobuf::reflect::ProtobufValue for FireballMessage { type RuntimeType = ::protobuf::reflect::rt::RuntimeTypeMessage; } +// @@protoc_insertion_point(message:digital_touch.MediaMessage) +#[derive(PartialEq,Clone,Default,Debug)] +pub struct MediaMessage { + // message fields + // @@protoc_insertion_point(field:digital_touch.MediaMessage.Archive) + pub Archive: ::std::vec::Vec, + // @@protoc_insertion_point(field:digital_touch.MediaMessage.MediaType) + pub MediaType: u64, + // special fields + // @@protoc_insertion_point(special_field:digital_touch.MediaMessage.special_fields) + pub special_fields: ::protobuf::SpecialFields, +} + +impl<'a> ::std::default::Default for &'a MediaMessage { + fn default() -> &'a MediaMessage { + ::default_instance() + } +} + +impl MediaMessage { + pub fn new() -> MediaMessage { + ::std::default::Default::default() + } + + fn generated_message_descriptor_data() -> ::protobuf::reflect::GeneratedMessageDescriptorData { + let mut fields = ::std::vec::Vec::with_capacity(2); + let mut oneofs = ::std::vec::Vec::with_capacity(0); + fields.push(::protobuf::reflect::rt::v2::make_simpler_field_accessor::<_, _>( + "Archive", + |m: &MediaMessage| { &m.Archive }, + |m: &mut MediaMessage| { &mut m.Archive }, + )); + fields.push(::protobuf::reflect::rt::v2::make_simpler_field_accessor::<_, _>( + "MediaType", + |m: &MediaMessage| { &m.MediaType }, + |m: &mut MediaMessage| { &mut m.MediaType }, + )); + ::protobuf::reflect::GeneratedMessageDescriptorData::new_2::( + "MediaMessage", + fields, + oneofs, + ) + } +} + +impl ::protobuf::Message for MediaMessage { + const NAME: &'static str = "MediaMessage"; + + fn is_initialized(&self) -> bool { + true + } + + fn merge_from(&mut self, is: &mut ::protobuf::CodedInputStream<'_>) -> ::protobuf::Result<()> { + while let Some(tag) = is.read_raw_tag_or_eof()? { + match tag { + 18 => { + self.Archive = is.read_bytes()?; + }, + 32 => { + self.MediaType = is.read_uint64()?; + }, + tag => { + ::protobuf::rt::read_unknown_or_skip_group(tag, is, self.special_fields.mut_unknown_fields())?; + }, + }; + } + ::std::result::Result::Ok(()) + } + + // Compute sizes of nested messages + #[allow(unused_variables)] + fn compute_size(&self) -> u64 { + let mut my_size = 0; + if !self.Archive.is_empty() { + my_size += ::protobuf::rt::bytes_size(2, &self.Archive); + } + if self.MediaType != 0 { + my_size += ::protobuf::rt::uint64_size(4, self.MediaType); + } + my_size += ::protobuf::rt::unknown_fields_size(self.special_fields.unknown_fields()); + self.special_fields.cached_size().set(my_size as u32); + my_size + } + + fn write_to_with_cached_sizes(&self, os: &mut ::protobuf::CodedOutputStream<'_>) -> ::protobuf::Result<()> { + if !self.Archive.is_empty() { + os.write_bytes(2, &self.Archive)?; + } + if self.MediaType != 0 { + os.write_uint64(4, self.MediaType)?; + } + os.write_unknown_fields(self.special_fields.unknown_fields())?; + ::std::result::Result::Ok(()) + } + + fn special_fields(&self) -> &::protobuf::SpecialFields { + &self.special_fields + } + + fn mut_special_fields(&mut self) -> &mut ::protobuf::SpecialFields { + &mut self.special_fields + } + + fn new() -> MediaMessage { + MediaMessage::new() + } + + fn clear(&mut self) { + self.Archive.clear(); + self.MediaType = 0; + self.special_fields.clear(); + } + + fn default_instance() -> &'static MediaMessage { + static instance: MediaMessage = MediaMessage { + Archive: ::std::vec::Vec::new(), + MediaType: 0, + special_fields: ::protobuf::SpecialFields::new(), + }; + &instance + } +} + +impl ::protobuf::MessageFull for MediaMessage { + fn descriptor() -> ::protobuf::reflect::MessageDescriptor { + static descriptor: ::protobuf::rt::Lazy<::protobuf::reflect::MessageDescriptor> = ::protobuf::rt::Lazy::new(); + descriptor.get(|| file_descriptor().message_by_package_relative_name("MediaMessage").unwrap()).clone() + } +} + +impl ::std::fmt::Display for MediaMessage { + fn fmt(&self, f: &mut ::std::fmt::Formatter<'_>) -> ::std::fmt::Result { + ::protobuf::text_format::fmt(self, f) + } +} + +impl ::protobuf::reflect::ProtobufValue for MediaMessage { + type RuntimeType = ::protobuf::reflect::rt::RuntimeTypeMessage; +} + #[derive(Clone,Copy,PartialEq,Eq,Debug,Hash)] // @@protoc_insertion_point(enum:digital_touch.TouchKind) pub enum TouchKind { @@ -1019,6 +1159,8 @@ pub enum TouchKind { Heartbeat = 3, // @@protoc_insertion_point(enum_value:digital_touch.TouchKind.Sketch) Sketch = 4, + // @@protoc_insertion_point(enum_value:digital_touch.TouchKind.Media) + Media = 6, // @@protoc_insertion_point(enum_value:digital_touch.TouchKind.Kiss) Kiss = 7, // @@protoc_insertion_point(enum_value:digital_touch.TouchKind.Fireball) @@ -1038,6 +1180,7 @@ impl ::protobuf::Enum for TouchKind { 1 => ::std::option::Option::Some(TouchKind::Tap), 3 => ::std::option::Option::Some(TouchKind::Heartbeat), 4 => ::std::option::Option::Some(TouchKind::Sketch), + 6 => ::std::option::Option::Some(TouchKind::Media), 7 => ::std::option::Option::Some(TouchKind::Kiss), 8 => ::std::option::Option::Some(TouchKind::Fireball), _ => ::std::option::Option::None @@ -1050,6 +1193,7 @@ impl ::protobuf::Enum for TouchKind { "Tap" => ::std::option::Option::Some(TouchKind::Tap), "Heartbeat" => ::std::option::Option::Some(TouchKind::Heartbeat), "Sketch" => ::std::option::Option::Some(TouchKind::Sketch), + "Media" => ::std::option::Option::Some(TouchKind::Media), "Kiss" => ::std::option::Option::Some(TouchKind::Kiss), "Fireball" => ::std::option::Option::Some(TouchKind::Fireball), _ => ::std::option::Option::None @@ -1061,6 +1205,7 @@ impl ::protobuf::Enum for TouchKind { TouchKind::Tap, TouchKind::Heartbeat, TouchKind::Sketch, + TouchKind::Media, TouchKind::Kiss, TouchKind::Fireball, ]; @@ -1078,8 +1223,9 @@ impl ::protobuf::EnumFull for TouchKind { TouchKind::Tap => 1, TouchKind::Heartbeat => 2, TouchKind::Sketch => 3, - TouchKind::Kiss => 4, - TouchKind::Fireball => 5, + TouchKind::Media => 4, + TouchKind::Kiss => 5, + TouchKind::Fireball => 6, }; Self::enum_descriptor().value_by_index(index) } @@ -1116,9 +1262,12 @@ static file_descriptor_proto_data: &'static [u8] = b"\ \x20\x01(\x02R\x08Duration\x12\x16\n\x06StartX\x18\x02\x20\x01(\x02R\x06\ StartX\x12\x16\n\x06StartY\x18\x03\x20\x01(\x02R\x06StartY\x12\x16\n\x06\ Delays\x18\x04\x20\x01(\x0cR\x06Delays\x12\x16\n\x06Points\x18\x05\x20\ - \x01(\x0cR\x06Points*T\n\tTouchKind\x12\x0b\n\x07Unknown\x10\0\x12\x07\n\ - \x03Tap\x10\x01\x12\r\n\tHeartbeat\x10\x03\x12\n\n\x06Sketch\x10\x04\x12\ - \x08\n\x04Kiss\x10\x07\x12\x0c\n\x08Fireball\x10\x08b\x06proto3\ + \x01(\x0cR\x06Points\"F\n\x0cMediaMessage\x12\x18\n\x07Archive\x18\x02\ + \x20\x01(\x0cR\x07Archive\x12\x1c\n\tMediaType\x18\x04\x20\x01(\x04R\tMe\ + diaType*_\n\tTouchKind\x12\x0b\n\x07Unknown\x10\0\x12\x07\n\x03Tap\x10\ + \x01\x12\r\n\tHeartbeat\x10\x03\x12\n\n\x06Sketch\x10\x04\x12\t\n\x05Med\ + ia\x10\x06\x12\x08\n\x04Kiss\x10\x07\x12\x0c\n\x08Fireball\x10\x08b\x06p\ + roto3\ "; /// `FileDescriptorProto` object which was a source for this generated file @@ -1136,13 +1285,14 @@ pub fn file_descriptor() -> &'static ::protobuf::reflect::FileDescriptor { file_descriptor.get(|| { let generated_file_descriptor = generated_file_descriptor_lazy.get(|| { let mut deps = ::std::vec::Vec::with_capacity(0); - let mut messages = ::std::vec::Vec::with_capacity(6); + let mut messages = ::std::vec::Vec::with_capacity(7); messages.push(BaseMessage::generated_message_descriptor_data()); messages.push(TapMessage::generated_message_descriptor_data()); messages.push(SketchMessage::generated_message_descriptor_data()); messages.push(KissMessage::generated_message_descriptor_data()); messages.push(HeartbeatMessage::generated_message_descriptor_data()); messages.push(FireballMessage::generated_message_descriptor_data()); + messages.push(MediaMessage::generated_message_descriptor_data()); let mut enums = ::std::vec::Vec::with_capacity(1); enums.push(TouchKind::generated_enum_descriptor_data()); ::protobuf::reflect::GeneratedFileDescriptor::new_generated( diff --git a/imessage-database/src/message_types/digital_touch/fireball.rs b/imessage-database/src/message_types/digital_touch/fireball.rs new file mode 100644 index 00000000..86ec73c9 --- /dev/null +++ b/imessage-database/src/message_types/digital_touch/fireball.rs @@ -0,0 +1,89 @@ +/*! +[Fireball](super) Digital Touch effect: a ball of fire dragged along a path. +*/ + +use protobuf::Message; + +use crate::{ + error::digital_touch::DigitalTouchError, + message_types::digital_touch::{ + digital_touch_proto::{BaseMessage, FireballMessage}, + models::{DigitalTouchMessage, Point, decode_points, decode_u16s, pluralize}, + svg::Canvas, + }, +}; + +/// A fireball dragged across the canvas. +#[derive(Debug, Clone, PartialEq)] +pub struct DigitalTouchFireball { + /// Unique identifier for the message. + pub id: String, + /// Start offset along x, centered (roughly `-1.0..=1.0`). + pub start_x: f32, + /// Start offset along y, centered (roughly `-1.0..=1.0`). + pub start_y: f32, + /// Total duration of the animation, in seconds. + pub duration: f32, + /// The dragged path; each point's `extra` is its delay in milliseconds. + pub points: Vec>, +} + +impl DigitalTouchFireball { + /// Parse the [`FireballMessage`] carried by `base` into a [`DigitalTouchMessage`]. + pub(super) fn from_payload( + base: &BaseMessage, + ) -> Result { + let msg = FireballMessage::parse_from_bytes(&base.TouchPayload) + .map_err(DigitalTouchError::ProtobufError)?; + + let delays = decode_u16s(&msg.Delays); + + Ok(DigitalTouchMessage::Fireball(DigitalTouchFireball { + id: base.ID.clone(), + start_x: msg.StartX, + start_y: msg.StartY, + duration: msg.Duration, + points: decode_points(&msg.Points, delays)?, + })) + } + + /// One-line summary, e.g. `"Digital Touch Fireball (3 points, 2.08s)"`. + pub(super) fn summary(&self) -> String { + format!( + "Digital Touch Fireball ({}, {:.2}s)", + pluralize(self.points.len(), "point"), + self.duration, + ) + } + + /// Draw the dragged trail and a glowing ball at its end. + pub(super) fn append_svg(&self, canvas: &mut Canvas) { + canvas.push_def( + r#""#, + ); + + let Some(last) = self.points.last() else { + return; + }; + + if self.points.len() > 1 { + let trail = self + .points + .iter() + .map(|p| format!("{},{}", canvas.fit_x(p.x), canvas.fit_y(p.y))) + .collect::>() + .join(" "); + let stroke_width = canvas.width() / 40; + canvas.push(&format!( + r#""# + )); + } + + let cx = canvas.fit_x(last.x); + let cy = canvas.fit_y(last.y); + let r = canvas.width() / 10; + canvas.push(&format!( + r#""# + )); + } +} diff --git a/imessage-database/src/message_types/digital_touch/heartbeat.rs b/imessage-database/src/message_types/digital_touch/heartbeat.rs new file mode 100644 index 00000000..63bb1a9d --- /dev/null +++ b/imessage-database/src/message_types/digital_touch/heartbeat.rs @@ -0,0 +1,114 @@ +/*! +[Heartbeat](super) Digital Touch effect: a pulse at a given rate, which may +break partway through (a "heartbreak"). +*/ + +use protobuf::Message; + +use crate::{ + error::digital_touch::DigitalTouchError, + message_types::digital_touch::{ + digital_touch_proto::{BaseMessage, HeartbeatMessage}, + models::DigitalTouchMessage, + svg::Canvas, + }, +}; + +/// A whole heart, centered on the origin and pointing down, spanning roughly +/// `-40..=40` in each axis. +const HEART: &str = "M 0,-22 C -10,-40 -40,-40 -40,-12 C -40,10 -15,22 0,38 C 15,22 40,10 40,-12 C 40,-40 10,-40 0,-22 Z"; +/// The left half of [`HEART`], split down the middle. +const HEART_LEFT: &str = "M 0,-22 C -10,-40 -40,-40 -40,-12 C -40,10 -15,22 0,38 Z"; +/// The right half of [`HEART`], split down the middle. +const HEART_RIGHT: &str = "M 0,-22 C 10,-40 40,-40 40,-12 C 40,10 15,22 0,38 Z"; + +/// A heartbeat effect. +#[derive(Debug, Clone, PartialEq)] +pub struct DigitalTouchHeartbeat { + /// Unique identifier for the message. + pub id: String, + /// Heart rate, in beats per minute. + pub bpm: f32, + /// Total duration of the animation, in seconds. + pub duration: u64, + /// When the heart breaks, in seconds from the start; `None` if it never does. + pub broken_at: Option, +} + +impl DigitalTouchHeartbeat { + /// Parse the [`HeartbeatMessage`] carried by `base` into a [`DigitalTouchMessage`]. + pub(super) fn from_payload( + base: &BaseMessage, + ) -> Result { + let msg = HeartbeatMessage::parse_from_bytes(&base.TouchPayload) + .map_err(DigitalTouchError::ProtobufError)?; + + Ok(DigitalTouchMessage::Heartbeat(DigitalTouchHeartbeat { + id: base.ID.clone(), + bpm: msg.BPM, + duration: msg.Duration, + broken_at: (msg.HeartBrokenAt > 0.0).then_some(msg.HeartBrokenAt), + })) + } + + /// One-line summary, e.g. `"Digital Touch Heartbeat (84 BPM, 2s)"` or + /// `"Digital Touch Heartbreak (84 BPM, broke at 1.71s)"`. + pub(super) fn summary(&self) -> String { + match self.broken_at { + Some(at) => format!( + "Digital Touch Heartbreak ({:.0} BPM, broke at {at:.2}s)", + self.bpm, + ), + None => format!( + "Digital Touch Heartbeat ({:.0} BPM, {}s)", + self.bpm, self.duration, + ), + } + } + + /// Draw a heart (whole, or split when broken) with a rate label. + pub(super) fn append_svg(&self, canvas: &mut Canvas) { + let width = canvas.width(); + let height = canvas.height(); + let cx = width / 2; + let cy = height * 2 / 5; + let scale = width as f64 / 200.0; + let font = width / 16; + let label_y = height * 4 / 5; + + match self.broken_at { + Some(at) => { + canvas.push(&format!( + r#""# + )); + canvas.push(&format!( + r#""# + )); + canvas.push(&label( + cx, + label_y, + font, + &format!("{:.0} BPM · broke at {at:.2}s", self.bpm), + )); + } + None => { + canvas.push(&format!( + r#""# + )); + canvas.push(&label( + cx, + label_y, + font, + &format!("{:.0} BPM · {}s", self.bpm, self.duration), + )); + } + } + } +} + +/// Build a centered white text label. +fn label(cx: usize, y: usize, font: usize, text: &str) -> String { + format!( + r#"{text}"# + ) +} diff --git a/imessage-database/src/message_types/digital_touch/kiss.rs b/imessage-database/src/message_types/digital_touch/kiss.rs new file mode 100644 index 00000000..9d568311 --- /dev/null +++ b/imessage-database/src/message_types/digital_touch/kiss.rs @@ -0,0 +1,107 @@ +/*! +[Kiss](super) Digital Touch effect: one or more placed, rotated kisses. +*/ + +use std::f64::consts::PI; + +use protobuf::Message; + +use crate::{ + error::digital_touch::DigitalTouchError, + message_types::digital_touch::{ + digital_touch_proto::{BaseMessage, KissMessage}, + models::{DigitalTouchMessage, Point, decode_points, decode_u16s}, + svg::Canvas, + }, +}; + +/// Silhouette of a pair of lips (a kiss mark), centered on the origin and +/// spanning roughly `-50..=50` horizontally. The top edge traces the two peaks +/// and central notch of a cupid's bow; the bottom edge is the fuller lower lip. +const LIPS_PATH: &str = "M -50,2 \ + C -40,-16 -22,-18 -10,-10 C -4,-14 -2,-14 0,-7 \ + C 2,-14 4,-14 10,-10 C 22,-18 40,-16 50,2 \ + C 44,11 30,17 18,22 C 8,27 -8,27 -18,22 C -30,17 -44,11 -50,2 Z"; + +/// Per-kiss data: its delay and its rotation. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct KissData { + /// Delay before this kiss, in milliseconds. + pub delay_ms: u16, + /// Rotation of the kiss, in milliradians (thousandths of a radian). + pub rotation_milliradians: u16, +} + +impl KissData { + /// Rotation in degrees, suitable for an SVG `rotate(…)` transform. Negated + /// because screen-space y grows downward. + #[must_use] + pub fn degrees(&self) -> f64 { + -(f64::from(self.rotation_milliradians) / 1000.0 * 180.0 / PI) + } +} + +/// A series of kisses, each placed and rotated. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DigitalTouchKiss { + /// Unique identifier for the message. + pub id: String, + /// The kisses, in order. + pub kisses: Vec>, +} + +impl DigitalTouchKiss { + /// Parse the [`KissMessage`] carried by `base` into a [`DigitalTouchMessage`]. + pub(super) fn from_payload( + base: &BaseMessage, + ) -> Result { + let msg = KissMessage::parse_from_bytes(&base.TouchPayload) + .map_err(DigitalTouchError::ProtobufError)?; + + let delays = decode_u16s(&msg.Delays); + let rotations = decode_u16s(&msg.Rotations); + if delays.len() != rotations.len() { + return Err(DigitalTouchError::ArraysDoNotMatch( + "delays", + delays.len(), + "rotations", + rotations.len(), + )); + } + + let extras = delays + .into_iter() + .zip(rotations) + .map(|(delay_ms, rotation_milliradians)| KissData { + delay_ms, + rotation_milliradians, + }) + .collect(); + + Ok(DigitalTouchMessage::Kiss(DigitalTouchKiss { + id: base.ID.clone(), + kisses: decode_points(&msg.Points, extras)?, + })) + } + + /// One-line summary, e.g. `"Digital Touch Kiss (1 kiss)"`. + pub(super) fn summary(&self) -> String { + let count = self.kisses.len(); + let noun = if count == 1 { "kiss" } else { "kisses" }; + format!("Digital Touch Kiss ({count} {noun})") + } + + /// Draw each kiss as a rotated pair of lips. + pub(super) fn append_svg(&self, canvas: &mut Canvas) { + // Scale the ~100-unit-wide lips to a recognizable size on the canvas. + let scale = canvas.width() as f64 / 293.0; + for kiss in &self.kisses { + let x = canvas.fit_x(kiss.x); + let y = canvas.fit_y(kiss.y); + let deg = kiss.extra.degrees(); + canvas.push(&format!( + r#""# + )); + } + } +} diff --git a/imessage-database/src/message_types/digital_touch/media.rs b/imessage-database/src/message_types/digital_touch/media.rs new file mode 100644 index 00000000..3295745a --- /dev/null +++ b/imessage-database/src/message_types/digital_touch/media.rs @@ -0,0 +1,189 @@ +/*! +[Media](super) Digital Touch effect: a photo or video. + +A still photo can be drawn on. The [`Image`](MediaKind::Image) overlay holds the +effects layered on top. A video cannot: the Digital Touch UI disables drawing +while capturing or sending video, so a video cannot carry an overlay. + +The effect carries a media-type discriminator and an `NSKeyedArchiver` archive. +For a photo, the archive holds an `NSMutableArray` of overlay effects, each an +`NSData` blob containing a complete, nested Digital Touch message parsed through +the same dispatcher as a top-level message; a nested media blob is skipped so a +crafted archive cannot drive unbounded recursion. The photo or video itself is +delivered as a normal message attachment, not embedded here. +*/ + +use std::io::Cursor; + +use plist::Value; +use protobuf::Message; + +use crate::{ + error::digital_touch::DigitalTouchError, + message_types::digital_touch::{ + digital_touch_proto::{BaseMessage, MediaMessage, TouchKind}, + models::{DigitalTouchMessage, pluralize}, + svg::Canvas, + }, +}; + +/// The media a [`DigitalTouchMedia`] effect carries. +/// +/// Only a still image can be drawn on: the Digital Touch UI disables the drawing +/// tools while capturing or sending video, so a video never carries an overlay. +#[derive(Debug, Clone, PartialEq)] +pub enum MediaKind { + /// A still image, with the effects drawn on top + /// of it. + Image { + /// Effects layered over the image; empty when nothing was drawn. + overlay: Vec, + }, + /// A video + Video, + /// An unrecognized media-type discriminator. + Other(u64), +} + +impl MediaKind { + /// Human-readable label. + fn label(&self) -> String { + match self { + MediaKind::Image { .. } => "Image".to_string(), + MediaKind::Video => "Video".to_string(), + MediaKind::Other(media_type) => format!("Media (type {media_type})"), + } + } +} + +/// A photo or video Digital Touch effect. +#[derive(Debug, Clone, PartialEq)] +pub struct DigitalTouchMedia { + /// Unique identifier for the message. + pub id: String, + /// The media carried, and (for an image) the effects drawn on top of it. + pub kind: MediaKind, +} + +impl DigitalTouchMedia { + /// Parse the [`MediaMessage`] carried by `base` into a [`DigitalTouchMessage`]. + pub(super) fn from_payload( + base: &BaseMessage, + ) -> Result { + let msg = MediaMessage::parse_from_bytes(&base.TouchPayload) + .map_err(DigitalTouchError::ProtobufError)?; + + // Only a still image can be drawn on, so only an image decodes an overlay + // from the archive; a video (or unknown kind) is shown on its own. + let kind = match msg.MediaType { + 1 => MediaKind::Video, + 2 => MediaKind::Image { + overlay: decode_overlays(&msg.Archive)?, + }, + other => MediaKind::Other(other), + }; + + Ok(DigitalTouchMessage::Media(DigitalTouchMedia { + id: base.ID.clone(), + kind, + })) + } + + /// One-line summary, e.g. `"Digital Touch Image"` or + /// `"Digital Touch Image with drawing (5 strokes)"`. + pub(super) fn summary(&self) -> String { + let MediaKind::Image { overlay } = &self.kind else { + return format!("Digital Touch {}", self.kind.label()); + }; + + let mut summary = "Digital Touch Image".to_string(); + if !overlay.is_empty() { + // Sketch overlays expose a precise stroke count; any other overlay + // type falls back to a generic label. + let strokes: usize = overlay + .iter() + .map(|effect| match effect { + DigitalTouchMessage::Sketch(sketch) => sketch.strokes.len(), + _ => 0, + }) + .sum(); + if strokes > 0 { + summary.push_str(&format!(" with drawing ({})", pluralize(strokes, "stroke"))); + } else { + summary.push_str(" with overlay"); + } + } + summary + } + + /// Draw the image overlay effects (if any). When the backing photo is supplied + /// as the canvas background it shows through beneath them. + pub(super) fn append_svg(&self, canvas: &mut Canvas) { + if let MediaKind::Image { overlay } = &self.kind { + for effect in overlay { + effect.append_svg(canvas); + } + } + + if !canvas.has_background() { + let width = canvas.width(); + let font = width / 16; + let y = canvas.height() * 9 / 10; + canvas.push(&format!( + r#"{}"#, + width / 2, + self.kind.label(), + )); + } + } +} + +/// Decode the overlay effects from the effect's `NSKeyedArchiver` archive. +/// +/// The archive is an `NSMutableArray` of `NSData` blobs, each a complete, nested +/// Digital Touch message. Any non-media [`TouchKind`] may appear. A blob that +/// fails to parse is skipped rather than failing the whole message, since the +/// media kind is still meaningful without its overlay. A nested +/// [`Media`](DigitalTouchMessage::Media) blob is skipped so a crafted archive +/// cannot drive unbounded recursion. +fn decode_overlays(archive: &[u8]) -> Result, DigitalTouchError> { + if archive.is_empty() { + return Ok(Vec::new()); + } + + let value = + Value::from_reader(Cursor::new(archive)).map_err(DigitalTouchError::ArchiveError)?; + + let Some(objects) = value + .as_dictionary() + .and_then(|dict| dict.get("$objects")) + .and_then(Value::as_array) + else { + return Ok(Vec::new()); + }; + + let mut overlays = Vec::new(); + for object in objects { + let Some(data) = object + .as_dictionary() + .and_then(|dict| dict.get("NS.data")) + .and_then(Value::as_data) + else { + continue; + }; + + let Ok(base) = BaseMessage::parse_from_bytes(data) else { + continue; + }; + // Refuse a nested media blob before dispatching, so a crafted archive + // can't nest media-in-media and drive unbounded recursion. + if base.TouchKind.enum_value_or_default() == TouchKind::Media { + continue; + } + if let Ok(message) = DigitalTouchMessage::from_base(&base) { + overlays.push(message); + } + } + + Ok(overlays) +} diff --git a/imessage-database/src/message_types/digital_touch/mod.rs b/imessage-database/src/message_types/digital_touch/mod.rs index 68a314e6..f69bcb6d 100644 --- a/imessage-database/src/message_types/digital_touch/mod.rs +++ b/imessage-database/src/message_types/digital_touch/mod.rs @@ -1,10 +1,21 @@ /*! -[Digital Touch](https://support.apple.com/guide/ipod-touch/send-a-digital-touch-effect-iph3fadba219/ios) messages are animated sketches, taps, fireballs, kisses, and heartbeats. +[Digital Touch](https://support.apple.com/guide/ipod-touch/send-a-digital-touch-effect-iph3fadba219/ios) messages are animated sketches, taps, fireballs, kisses, and heartbeats, as well as still photos with optional drawing overlays and videos. + +Parse a `payload_data` blob with [`DigitalTouchMessage::from_payload`], then +render it with [`render_svg`](DigitalTouchMessage::render_svg) or +[`render_text`](DigitalTouchMessage::render_text). */ -pub use crate::message_types::digital_touch::{ - digital_touch_proto::TouchKind as DigitalTouch, models::from_payload, +pub use crate::message_types::digital_touch::models::{ + Color, DigitalTouchMessage, ImageBackdrop, Point, }; pub(crate) mod digital_touch_proto; +pub mod fireball; +pub mod heartbeat; +pub mod kiss; +pub mod media; pub mod models; +pub mod sketch; +mod svg; +pub mod tap; diff --git a/imessage-database/src/message_types/digital_touch/models.rs b/imessage-database/src/message_types/digital_touch/models.rs index 789f2ddc..e4e2c7af 100644 --- a/imessage-database/src/message_types/digital_touch/models.rs +++ b/imessage-database/src/message_types/digital_touch/models.rs @@ -1,111 +1,519 @@ /*! Parser for [Digital Touch](https://support.apple.com/guide/ipod-touch/send-a-digital-touch-effect-iph3fadba219/ios) iMessages. -This message type is not documented by Apple, but represents messages displayed as `com.apple.DigitalTouchBalloonProvider`. + +This message type is not documented by Apple, but represents messages displayed +as `com.apple.DigitalTouchBalloonProvider`. Each message is a `BaseMessage` +envelope wrapping an effect-specific protobuf selected by its `TouchKind`. + +The effects share a handful of binary encodings, captured by the helpers here: + +- Coordinates are little-endian `u16` pairs `(x, y)` where `0..=u16::MAX` spans + the canvas in each axis, origin bottom-left, y growing upward ([`decode_points`]). +- Timing delays are little-endian `u16` milliseconds ([`decode_u16s`]). +- Colors are four bytes in RGBA order ([`Color`]). */ -use crate::message_types::digital_touch::digital_touch_proto::{ - BaseMessage, TouchKind as DigitalTouch, -}; +use std::borrow::Cow; use protobuf::Message; -/// Converts a raw byte payload from the database into a [`DigitalTouch`]. +use crate::{ + error::digital_touch::DigitalTouchError, + message_types::digital_touch::{ + digital_touch_proto::{BaseMessage, TouchKind}, + fireball::DigitalTouchFireball, + heartbeat::DigitalTouchHeartbeat, + kiss::DigitalTouchKiss, + media::DigitalTouchMedia, + sketch::DigitalTouchSketch, + svg::Canvas, + tap::DigitalTouchTap, + }, +}; + +/// A parsed [Digital Touch](https://support.apple.com/guide/ipod-touch/send-a-digital-touch-effect-iph3fadba219/ios) message. +/// +/// Construct one with [`DigitalTouchMessage::from_payload`], then render it with +/// [`render_svg`](DigitalTouchMessage::render_svg) or +/// [`render_text`](DigitalTouchMessage::render_text). +#[derive(Debug, Clone, PartialEq)] +pub enum DigitalTouchMessage { + /// One or more taps, each a colored burst at a point. + Tap(DigitalTouchTap), + /// A freehand drawing made of colored strokes. + Sketch(DigitalTouchSketch), + /// One or more kisses, each placed and rotated. + Kiss(DigitalTouchKiss), + /// A heartbeat, optionally breaking partway through. + Heartbeat(DigitalTouchHeartbeat), + /// A fireball dragged along a path. + Fireball(DigitalTouchFireball), + /// A photo or video; a photo may have effects drawn on top of it. + Media(DigitalTouchMedia), +} + +/// A still-image backdrop for [`render_svg`](DigitalTouchMessage::render_svg), +/// drawn behind the effect in place of the default black canvas. +/// +/// Only a still image is ever a backdrop. The Digital Touch UI disables drawing +/// over video, so a video [`Media`](DigitalTouchMessage::Media) message has no +/// overlay to composite. +/// +/// This is a render-time input, not parsed state: the caller resolves the image +/// via [`Attachment::from_message`](crate::tables::attachment::Attachment::from_message) +/// and supplies it here. The value becomes the SVG `` `href` after XML +/// attribute escaping. The renderer treats it as an opaque reference and never +/// reads the file. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ImageBackdrop<'a>(pub Cow<'a, str>); + +impl ImageBackdrop<'_> { + /// Render the `` markup that displays this backdrop, sized to cover a + /// `width`×`height` canvas (`slice`/`cover` crops the overflow). + pub(super) fn render(&self, width: usize, height: usize) -> String { + format!( + r#""#, + escape_attr(&self.0) + ) + } +} + +/// Escape the characters significant inside a double-quoted XML attribute value, +/// for references such as a backdrop `href`. +fn escape_attr(text: &str) -> String { + text.replace('&', "&") + .replace('<', "<") + .replace('>', ">") + .replace('"', """) +} + +impl DigitalTouchMessage { + /// Convert a raw `payload_data` byte blob into a [`DigitalTouchMessage`]. + pub fn from_payload(payload: &[u8]) -> Result { + let msg = + BaseMessage::parse_from_bytes(payload).map_err(DigitalTouchError::ProtobufError)?; + Self::from_base(&msg) + } + + /// Dispatch an already-parsed [`BaseMessage`] envelope to the parser for its + /// [`TouchKind`]. Used both for top-level messages and for the nested effects + /// a [`Media`](DigitalTouchMessage::Media) message draws over its photo. + pub(super) fn from_base(msg: &BaseMessage) -> Result { + match msg.TouchKind.enum_value_or_default() { + TouchKind::Tap => DigitalTouchTap::from_payload(msg), + TouchKind::Sketch => DigitalTouchSketch::from_payload(msg), + TouchKind::Kiss => DigitalTouchKiss::from_payload(msg), + TouchKind::Heartbeat => DigitalTouchHeartbeat::from_payload(msg), + TouchKind::Fireball => DigitalTouchFireball::from_payload(msg), + TouchKind::Media => DigitalTouchMedia::from_payload(msg), + TouchKind::Unknown => Err(DigitalTouchError::UnknownDigitalTouchKind( + msg.TouchKind.value(), + )), + } + } + + /// Render a static SVG depiction of the effect. + /// + /// `backdrop` optionally references a still image to draw behind the effect in + /// place of the default black canvas. Pass `None` for a plain black background. + #[must_use] + pub fn render_svg(&self, backdrop: Option>) -> String { + let mut canvas = Canvas::new(self.summary(), backdrop); + self.append_svg(&mut canvas); + canvas.finish() + } + + /// Append this effect's markup to `canvas`. Factored out of + /// [`render_svg`](Self::render_svg) so a [`Media`](Self::Media) message can + /// draw its overlay effects through the same dispatch. + pub(super) fn append_svg(&self, canvas: &mut Canvas) { + match self { + DigitalTouchMessage::Tap(t) => t.append_svg(canvas), + DigitalTouchMessage::Sketch(s) => s.append_svg(canvas), + DigitalTouchMessage::Kiss(k) => k.append_svg(canvas), + DigitalTouchMessage::Heartbeat(h) => h.append_svg(canvas), + DigitalTouchMessage::Fireball(f) => f.append_svg(canvas), + DigitalTouchMessage::Media(m) => m.append_svg(canvas), + } + } + + /// Render a one-line, human-readable summary of the effect. + #[must_use] + pub fn render_text(&self) -> String { + self.summary() + } + + /// Concise label used for both the text rendering and the SVG ``. + fn summary(&self) -> String { + match self { + DigitalTouchMessage::Tap(t) => t.summary(), + DigitalTouchMessage::Sketch(s) => s.summary(), + DigitalTouchMessage::Kiss(k) => k.summary(), + DigitalTouchMessage::Heartbeat(h) => h.summary(), + DigitalTouchMessage::Fireball(f) => f.summary(), + DigitalTouchMessage::Media(m) => m.summary(), + } + } +} + +/// A point captured by an effect, paired with effect-specific `extra` data +/// (a color, a delay, a rotation, …). Coordinates are normalized `u16` values. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Point<T> { + /// X coordinate, `0..=u16::MAX` spanning the canvas left-to-right. + pub x: u16, + /// Y coordinate, `0..=u16::MAX` spanning the canvas bottom-to-top + /// (Digital Touch uses a bottom-left origin). + pub y: u16, + /// Effect-specific data associated with this point. + pub extra: T, +} + +/// An RGBA color, one byte per channel. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Color { + /// Red channel. + pub r: u8, + /// Green channel. + pub g: u8, + /// Blue channel. + pub b: u8, + /// Alpha channel. + pub a: u8, +} + +impl Color { + /// Opaque white, used as a fallback when an effect carries no color. + pub const WHITE: Color = Color { + r: 255, + g: 255, + b: 255, + a: 255, + }; + + /// Decode a packed buffer of consecutive RGBA colors. + #[must_use] + pub fn decode_all(buf: &[u8]) -> Vec<Color> { + buf.chunks_exact(4) + .map(|c| Color { + r: c[0], + g: c[1], + b: c[2], + a: c[3], + }) + .collect() + } + + /// Render as an SVG/CSS `rgba(…)` color string. + #[must_use] + pub fn css(&self) -> String { + // SVG/CSS `rgba()` expects alpha in `0..=1`, not the raw `0..=255` byte. + format!( + "rgba({}, {}, {}, {})", + self.r, + self.g, + self.b, + f32::from(self.a) / 255.0 + ) + } +} + +/// Decode a packed buffer of little-endian `u16` values (delays, rotations). #[must_use] -pub fn from_payload(payload: &[u8]) -> Option<DigitalTouch> { - let msg = BaseMessage::parse_from_bytes(payload).ok()?; +pub fn decode_u16s(buf: &[u8]) -> Vec<u16> { + buf.chunks_exact(2) + .map(|c| u16::from_le_bytes([c[0], c[1]])) + .collect() +} + +/// Decode `(x, y)` points from `raw` and zip each with the parallel `extras`. +/// +/// Returns [`DigitalTouchError::ArraysDoNotMatch`] if the number of points does +/// not match the number of `extras`, which would mean the effect's parallel +/// arrays disagree on how many events it contains. +pub fn decode_points<T>(raw: &[u8], extras: Vec<T>) -> Result<Vec<Point<T>>, DigitalTouchError> { + let coords: Vec<(u16, u16)> = raw + .chunks_exact(4) + .map(|c| { + ( + u16::from_le_bytes([c[0], c[1]]), + u16::from_le_bytes([c[2], c[3]]), + ) + }) + .collect(); + + if coords.len() != extras.len() { + return Err(DigitalTouchError::ArraysDoNotMatch( + "points", + coords.len(), + "values", + extras.len(), + )); + } + + Ok(coords + .into_iter() + .zip(extras) + .map(|((x, y), extra)| Point { x, y, extra }) + .collect()) +} + +/// Decode `count` concatenated strokes from a single buffer. +/// +/// Each stroke is laid out as a two-byte per-stroke header (observed as zero), a +/// little-endian `u16` point count, then that many `(x, y)` points. `count` is +/// the number of strokes to read, e.g. a [`SketchMessage`](super::sketch)'s +/// `StrokesCount`. +pub fn decode_strokes(raw: &[u8], count: usize) -> Result<Vec<Vec<(u16, u16)>>, DigitalTouchError> { + // Guard against pathological counts that would cause us to allocate more memory than the input buffer size + let mut strokes = Vec::with_capacity(count.min(raw.len() / 4)); + let mut idx = 0; + + for _ in 0..count { + // Every stroke begins with a four-byte header: two reserved bytes + // followed by the point count. + if idx + 4 > raw.len() { + return Err(DigitalTouchError::InvalidStrokesLength(idx + 4, raw.len())); + } - Some(msg.TouchKind.enum_value_or_default()) + let points = usize::from(u16::from_le_bytes([raw[idx + 2], raw[idx + 3]])); + idx += 4; + + let end = idx + points * 4; + if end > raw.len() { + return Err(DigitalTouchError::InvalidStrokesLength(end, raw.len())); + } + + strokes.push( + raw[idx..end] + .chunks_exact(4) + .map(|c| { + ( + u16::from_le_bytes([c[0], c[1]]), + u16::from_le_bytes([c[2], c[3]]), + ) + }) + .collect(), + ); + idx = end; + } + + Ok(strokes) +} + +/// Pluralize a noun for a count: `pluralize(1, "tap")` → `"1 tap"`. +pub(super) fn pluralize(count: usize, noun: &str) -> String { + if count == 1 { + format!("{count} {noun}") + } else { + format!("{count} {noun}s") + } } #[cfg(test)] mod tests { - use crate::message_types::digital_touch::{DigitalTouch, from_payload}; + use super::{Color, DigitalTouchMessage, ImageBackdrop}; + use crate::message_types::digital_touch::media::MediaKind; use std::env::current_dir; - use std::fs::File; - use std::io::Read; + use std::fs::read; - #[test] - fn can_parse_tap() { - let protobuf_path = current_dir() + /// Magenta, the color the tap and sketch fixtures were drawn with. + const MAGENTA: Color = Color { + r: 255, + g: 0, + b: 252, + a: 255, + }; + + fn load(name: &str) -> Vec<u8> { + let path = current_dir() .unwrap() - .as_path() - .join("test_data/digital_touch_message/tap.bin"); - let mut proto_data = File::open(protobuf_path).unwrap(); - let mut data = vec![]; - proto_data.read_to_end(&mut data).unwrap(); + .join("test_data/digital_touch_message") + .join(name); + read(path).unwrap() + } - let actual = from_payload(&data); - assert_eq!(actual, Some(DigitalTouch::Tap)); + fn parse(name: &str) -> DigitalTouchMessage { + DigitalTouchMessage::from_payload(&load(name)).unwrap() } #[test] - fn can_parse_heartbeat() { - let protobuf_path = current_dir() - .unwrap() - .as_path() - .join("test_data/digital_touch_message/heartbeat.bin"); - let mut proto_data = File::open(protobuf_path).unwrap(); - let mut data = vec![]; - proto_data.read_to_end(&mut data).unwrap(); + fn parses_tap() { + let DigitalTouchMessage::Tap(tap) = parse("tap.bin") else { + panic!("expected a tap"); + }; + assert_eq!(tap.id, "E3F4E72A-A863-43C3-8277-E17680251B06"); + assert_eq!(tap.taps.len(), 1); + assert_eq!((tap.taps[0].x, tap.taps[0].y), (30809, 37418)); + assert_eq!(tap.taps[0].extra.delay_ms, 0); + assert_eq!(tap.taps[0].extra.color, MAGENTA); + } - let actual = from_payload(&data); - assert_eq!(actual, Some(DigitalTouch::Heartbeat)); + #[test] + fn parses_sketch() { + let DigitalTouchMessage::Sketch(sketch) = parse("sketch.bin") else { + panic!("expected a sketch"); + }; + assert_eq!(sketch.id, "F7D92232-92B3-4C5A-8DC7-2704BE93890E"); + assert_eq!(sketch.strokes.len(), 1); + assert_eq!(sketch.strokes[0].len(), 81); + assert_eq!( + (sketch.strokes[0][0].x, sketch.strokes[0][0].y), + (14168, 43154) + ); + assert_eq!(sketch.strokes[0][0].extra, MAGENTA); } #[test] - fn can_parse_heartbreak() { - let protobuf_path = current_dir() - .unwrap() - .as_path() - .join("test_data/digital_touch_message/heartbreak.bin"); - let mut proto_data = File::open(protobuf_path).unwrap(); - let mut data = vec![]; - proto_data.read_to_end(&mut data).unwrap(); + fn parses_kiss() { + let DigitalTouchMessage::Kiss(kiss) = parse("kiss.bin") else { + panic!("expected a kiss"); + }; + assert_eq!(kiss.id, "24AA9029-0725-4318-B449-10C2D255AB9E"); + assert_eq!(kiss.kisses.len(), 1); + assert_eq!((kiss.kisses[0].x, kiss.kisses[0].y), (33913, 34117)); + assert_eq!(kiss.kisses[0].extra.delay_ms, 0); + assert_eq!(kiss.kisses[0].extra.rotation_milliradians, 294); + } - let actual = from_payload(&data); - assert_eq!(actual, Some(DigitalTouch::Heartbeat)); + #[test] + fn parses_heartbeat() { + let DigitalTouchMessage::Heartbeat(heartbeat) = parse("heartbeat.bin") else { + panic!("expected a heartbeat"); + }; + assert_eq!(heartbeat.id, "12864C14-0F81-4362-953C-82D1008E46EC"); + assert_eq!(heartbeat.bpm, 84.0); + assert_eq!(heartbeat.duration, 2); + assert_eq!(heartbeat.broken_at, None); } #[test] - fn can_parse_sketch() { - let protobuf_path = current_dir() - .unwrap() - .as_path() - .join("test_data/digital_touch_message/sketch.bin"); - let mut proto_data = File::open(protobuf_path).unwrap(); - let mut data = vec![]; - proto_data.read_to_end(&mut data).unwrap(); + fn parses_heartbreak() { + let DigitalTouchMessage::Heartbeat(heartbeat) = parse("heartbreak.bin") else { + panic!("expected a heartbeat"); + }; + assert_eq!(heartbeat.id, "6BAB7D7D-2E3C-4995-9887-01AD37C3B0E2"); + assert_eq!(heartbeat.bpm, 84.0); + assert_eq!(heartbeat.duration, 2); + let broken_at = heartbeat.broken_at.expect("heartbreak should be broken"); + assert!((broken_at - 1.714_62).abs() < 0.001, "got {broken_at}"); + } - let actual = from_payload(&data); - assert_eq!(actual, Some(DigitalTouch::Sketch)); + #[test] + fn parses_fireball() { + let DigitalTouchMessage::Fireball(fireball) = parse("fireball.bin") else { + panic!("expected a fireball"); + }; + assert_eq!(fireball.id, "0AC74C8E-BEF0-4AB5-97C9-C35DADE5EC65"); + assert_eq!(fireball.points.len(), 3); + assert!( + (fireball.duration - 2.079_86).abs() < 0.001, + "got {}", + fireball.duration + ); + assert_eq!((fireball.points[0].x, fireball.points[0].y), (32252, 31366)); + // Delays are decoded as each point's `extra`. + let delays: Vec<u16> = fireball.points.iter().map(|p| p.extra).collect(); + assert_eq!(delays, vec![859, 0, 83]); } #[test] - fn can_parse_kiss() { - let protobuf_path = current_dir() - .unwrap() - .as_path() - .join("test_data/digital_touch_message/kiss.bin"); - let mut proto_data = File::open(protobuf_path).unwrap(); - let mut data = vec![]; - proto_data.read_to_end(&mut data).unwrap(); + fn parses_image() { + let DigitalTouchMessage::Media(media) = parse("image.bin") else { + panic!("expected media"); + }; + let MediaKind::Image { overlay } = &media.kind else { + panic!("expected an image"); + }; + assert!(overlay.is_empty()); + } - let actual = from_payload(&data); - assert_eq!(actual, Some(DigitalTouch::Kiss)); + #[test] + fn parses_video() { + let DigitalTouchMessage::Media(media) = parse("video.bin") else { + panic!("expected media"); + }; + // A video carries no overlay by construction + assert!(matches!(media.kind, MediaKind::Video)); } #[test] - fn can_parse_fireball() { - let protobuf_path = current_dir() - .unwrap() - .as_path() - .join("test_data/digital_touch_message/fireball.bin"); - let mut proto_data = File::open(protobuf_path).unwrap(); - let mut data = vec![]; - proto_data.read_to_end(&mut data).unwrap(); - - let actual = from_payload(&data); - assert_eq!(actual, Some(DigitalTouch::Fireball)); + fn parses_image_with_drawing() { + let DigitalTouchMessage::Media(media) = parse("image_with_drawing.bin") else { + panic!("expected media"); + }; + let MediaKind::Image { overlay } = &media.kind else { + panic!("expected an image"); + }; + // The overlay is a single nested sketch message with five strokes. + assert_eq!(overlay.len(), 1); + let DigitalTouchMessage::Sketch(sketch) = &overlay[0] else { + panic!("expected a sketch overlay"); + }; + assert_eq!(sketch.strokes.len(), 5); + let points: usize = sketch.strokes.iter().map(Vec::len).sum(); + assert_eq!(points, 97); + } + + #[test] + fn renders_svg_and_text() { + let sketch = parse("sketch.bin"); + let svg = sketch.render_svg(None); + assert!(svg.starts_with("<svg")); + assert!(svg.contains("<polyline")); + assert!(svg.contains(&MAGENTA.css())); + assert_eq!( + sketch.render_text(), + "Digital Touch Sketch (1 stroke, 81 points)" + ); + + assert_eq!( + parse("heartbreak.bin").render_text(), + "Digital Touch Heartbreak (84 BPM, broke at 1.71s)" + ); + assert_eq!( + parse("image_with_drawing.bin").render_text(), + "Digital Touch Image with drawing (5 strokes)" + ); + assert_eq!(parse("video.bin").render_text(), "Digital Touch Video"); + } + + #[test] + fn renders_single_point_stroke_as_dot() { + let message = parse("hi.bin"); + let DigitalTouchMessage::Sketch(sketch) = &message else { + panic!("expected a sketch"); + }; + // "hi": four multi-point strokes plus the single-point dot on the "i". + let lengths: Vec<usize> = sketch.strokes.iter().map(Vec::len).collect(); + assert_eq!(lengths, vec![38, 10, 1, 17, 38]); + + // SVG won't stroke a one-vertex polyline, so that 1-point stroke renders as + // a <circle> while the other four stay <polyline>s. + let svg = message.render_svg(None); + assert_eq!(svg.matches("<circle").count(), 1); + assert_eq!(svg.matches("<polyline").count(), 4); + } + + #[test] + fn image_backdrop_references_media_and_drops_label() { + let media = parse("image.bin"); + + // An image backdrop is referenced via <image>, and the placeholder kind + // label is omitted because the media itself is shown. + let with_image = media.render_svg(Some(ImageBackdrop("attachments/0/bg.png".into()))); + assert!(with_image.contains(r#"<image href="attachments/0/bg.png""#)); + assert!(!with_image.contains(">Image</text>")); + + // Without a backdrop, nothing is referenced and the kind is labeled. + let without_backdrop = media.render_svg(None); + assert!(!without_backdrop.contains("<image")); + assert!(without_backdrop.contains(">Image</text>")); + } + + #[test] + fn decode_strokes_rejects_oversized_count_without_allocating() { + assert!(super::decode_strokes(&[], 1_000_000_000).is_err()); } } diff --git a/imessage-database/src/message_types/digital_touch/sketch.rs b/imessage-database/src/message_types/digital_touch/sketch.rs new file mode 100644 index 00000000..fd082584 --- /dev/null +++ b/imessage-database/src/message_types/digital_touch/sketch.rs @@ -0,0 +1,97 @@ +/*! +[Sketch](super) Digital Touch effect: a freehand drawing of colored strokes. +*/ + +use protobuf::Message; + +use crate::{ + error::digital_touch::DigitalTouchError, + message_types::digital_touch::{ + digital_touch_proto::{BaseMessage, SketchMessage}, + models::{Color, DigitalTouchMessage, Point, decode_strokes, pluralize}, + svg::Canvas, + }, +}; + +/// Width of a sketched stroke, in canvas user units. +const STROKE_WIDTH: usize = 8; + +/// A freehand sketch made of one or more colored strokes. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DigitalTouchSketch { + /// Unique identifier for the message. + pub id: String, + /// Strokes, each a polyline of points sharing a single color. + pub strokes: Vec<Vec<Point<Color>>>, +} + +impl DigitalTouchSketch { + /// Parse the [`SketchMessage`] carried by `base` into a [`DigitalTouchMessage`]. + pub(super) fn from_payload( + base: &BaseMessage, + ) -> Result<DigitalTouchMessage, DigitalTouchError> { + let msg = SketchMessage::parse_from_bytes(&base.TouchPayload) + .map_err(DigitalTouchError::ProtobufError)?; + + let colors = Color::decode_all(&msg.Colors); + let count = usize::try_from(msg.StrokesCount).unwrap_or(0); + let strokes = decode_strokes(&msg.Strokes, count)? + .into_iter() + .enumerate() + .map(|(stroke, points)| { + let color = colors.get(stroke).copied().unwrap_or(Color::WHITE); + points + .into_iter() + .map(|(x, y)| Point { x, y, extra: color }) + .collect() + }) + .collect(); + + Ok(DigitalTouchMessage::Sketch(DigitalTouchSketch { + id: base.ID.clone(), + strokes, + })) + } + + /// One-line summary, e.g. `"Digital Touch Sketch (1 stroke, 81 points)"`. + pub(super) fn summary(&self) -> String { + let points: usize = self.strokes.iter().map(Vec::len).sum(); + format!( + "Digital Touch Sketch ({}, {})", + pluralize(self.strokes.len(), "stroke"), + pluralize(points, "point"), + ) + } + + /// Draw each stroke as a colored polyline, or a dot for a single-point stroke. + pub(super) fn append_svg(&self, canvas: &mut Canvas) { + for stroke in &self.strokes { + let Some(first) = stroke.first() else { + continue; + }; + let color = first.extra.css(); + + // A single-point stroke has no segment to stroke, and SVG does not + // render a one-vertex polyline. Draw it as a filled dot the size of + // the round stroke cap instead. + if stroke.len() == 1 { + canvas.push(&format!( + r#"<circle cx="{}" cy="{}" r="{}" fill="{color}" />"#, + canvas.fit_x(first.x), + canvas.fit_y(first.y), + STROKE_WIDTH / 2, + )); + continue; + } + + let points = stroke + .iter() + .map(|p| format!("{},{}", canvas.fit_x(p.x), canvas.fit_y(p.y))) + .collect::<Vec<_>>() + .join(" "); + canvas.push(&format!( + r#"<polyline points="{points}" fill="none" stroke="{color}" stroke-width="{STROKE_WIDTH}" stroke-linecap="round" stroke-linejoin="round" />"# + )); + } + } +} diff --git a/imessage-database/src/message_types/digital_touch/svg.rs b/imessage-database/src/message_types/digital_touch/svg.rs new file mode 100644 index 00000000..d8536f48 --- /dev/null +++ b/imessage-database/src/message_types/digital_touch/svg.rs @@ -0,0 +1,116 @@ +/*! +Minimal SVG builder shared by the Digital Touch effect renderers. + +Digital Touch effects are animated on-device. We render a single, static frame +that depicts the captured data (the drawn strokes, the tap and kiss locations, +the fireball's path, a heart with its rate) onto a black canvas, matching the +black backdrop the effects are composed on in Messages. The canvas is `4:5`, +the (device-dependent) aspect ratio of the Apple Watch screen the effects are +authored and displayed on. +*/ + +use std::fmt::Write; + +use crate::message_types::digital_touch::models::ImageBackdrop; + +/// Canvas width in user units. The emitted `<svg>` scales to its container, so +/// this only fixes the internal coordinate space. +pub(super) const WIDTH: usize = 768; +/// Canvas height in user units, giving a `4:5` portrait aspect ratio. +pub(super) const HEIGHT: usize = 960; + +/// Accumulates SVG markup for one effect, then wraps it in an `<svg>` root. +pub(super) struct Canvas { + title: String, + defs: String, + body: String, + /// Pre-rendered `<image>` markup drawn behind the effect, in place of the + /// plain black canvas. + backdrop: Option<String>, +} + +impl Canvas { + /// Create a canvas with the given accessible `<title>` and optional still-image backdrop. + pub(super) fn new(title: impl Into<String>, background: Option<ImageBackdrop<'_>>) -> Self { + Self { + title: title.into(), + defs: String::new(), + body: String::new(), + backdrop: background.map(|background| background.render(WIDTH, HEIGHT)), + } + } + + /// Whether a still-image backdrop was supplied. + pub(super) fn has_background(&self) -> bool { + self.backdrop.is_some() + } + + /// Canvas width, for renderers that place elements absolutely. + pub(super) fn width(&self) -> usize { + WIDTH + } + + /// Canvas height, for renderers that place elements absolutely. + pub(super) fn height(&self) -> usize { + HEIGHT + } + + /// Map a normalized x coordinate (`0..=u16::MAX`, left to right) onto canvas + /// user units. + pub(super) fn fit_x(&self, value: u16) -> usize { + usize::from(value) * WIDTH / usize::from(u16::MAX) + } + + /// Map a normalized y coordinate onto canvas user units. Digital Touch uses a + /// bottom-left origin (y grows upward), so this inverts onto SVG's top-left, + /// y-down space. + pub(super) fn fit_y(&self, value: u16) -> usize { + HEIGHT - usize::from(value) * HEIGHT / usize::from(u16::MAX) + } + + /// Append an element to the canvas body. + pub(super) fn push(&mut self, markup: &str) { + self.body.push_str(markup); + self.body.push('\n'); + } + + /// Append an entry to the `<defs>` block (e.g. a gradient). + pub(super) fn push_def(&mut self, markup: &str) { + self.defs.push_str(markup); + self.defs.push('\n'); + } + + /// Render the accumulated markup into a complete `<svg>` document. + pub(super) fn finish(self) -> String { + let mut svg = String::with_capacity(self.body.len() + self.defs.len() + 256); + let _ = write!( + svg, + r#"<svg viewBox="0 0 {WIDTH} {HEIGHT}" preserveAspectRatio="xMidYMid meet" width="100%" height="100%" xmlns="http://www.w3.org/2000/svg">"# + ); + svg.push('\n'); + let _ = writeln!(svg, "<title>{}", escape(&self.title)); + if !self.defs.is_empty() { + svg.push_str("\n"); + svg.push_str(&self.defs); + svg.push_str("\n"); + } + let _ = writeln!( + svg, + r#""# + ); + if let Some(backdrop) = &self.backdrop { + svg.push_str(backdrop); + svg.push('\n'); + } + svg.push_str(&self.body); + svg.push_str("\n"); + svg + } +} + +/// Escape the characters that are significant inside SVG/XML text content. +fn escape(text: &str) -> String { + text.replace('&', "&") + .replace('<', "<") + .replace('>', ">") +} diff --git a/imessage-database/src/message_types/digital_touch/tap.rs b/imessage-database/src/message_types/digital_touch/tap.rs new file mode 100644 index 00000000..5d13f883 --- /dev/null +++ b/imessage-database/src/message_types/digital_touch/tap.rs @@ -0,0 +1,86 @@ +/*! +[Tap](super) Digital Touch effect: one or more colored bursts at points. +*/ + +use protobuf::Message; + +use crate::{ + error::digital_touch::DigitalTouchError, + message_types::digital_touch::{ + digital_touch_proto::{BaseMessage, TapMessage}, + models::{Color, DigitalTouchMessage, Point, decode_points, decode_u16s, pluralize}, + svg::Canvas, + }, +}; + +/// Per-tap data: the burst color and its delay from the previous tap. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct TapData { + /// Color of the burst. + pub color: Color, + /// Delay before this tap, in milliseconds. + pub delay_ms: u16, +} + +/// A series of taps, each a colored burst at a point. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DigitalTouchTap { + /// Unique identifier for the message. + pub id: String, + /// The taps, in order. + pub taps: Vec>, +} + +impl DigitalTouchTap { + /// Parse the [`TapMessage`] carried by `base` into a [`DigitalTouchMessage`]. + pub(super) fn from_payload( + base: &BaseMessage, + ) -> Result { + let msg = TapMessage::parse_from_bytes(&base.TouchPayload) + .map_err(DigitalTouchError::ProtobufError)?; + + let colors = Color::decode_all(&msg.Color); + let delays = decode_u16s(&msg.Delays); + if colors.len() != delays.len() { + return Err(DigitalTouchError::ArraysDoNotMatch( + "colors", + colors.len(), + "delays", + delays.len(), + )); + } + + let extras = colors + .into_iter() + .zip(delays) + .map(|(color, delay_ms)| TapData { color, delay_ms }) + .collect(); + + Ok(DigitalTouchMessage::Tap(DigitalTouchTap { + id: base.ID.clone(), + taps: decode_points(&msg.Location, extras)?, + })) + } + + /// One-line summary, e.g. `"Digital Touch Tap (1 tap)"`. + pub(super) fn summary(&self) -> String { + format!("Digital Touch Tap ({})", pluralize(self.taps.len(), "tap")) + } + + /// Draw each tap as a colored ring with a filled center. + pub(super) fn append_svg(&self, canvas: &mut Canvas) { + let ring = canvas.width() / 12; + let dot = canvas.width() / 40; + for tap in &self.taps { + let cx = canvas.fit_x(tap.x); + let cy = canvas.fit_y(tap.y); + let color = tap.extra.color.css(); + canvas.push(&format!( + r#""# + )); + canvas.push(&format!( + r#""# + )); + } + } +} diff --git a/imessage-database/src/message_types/text_effects/text_effect.rs b/imessage-database/src/message_types/text_effects/text_effect.rs index 3fbdd824..e937a7b4 100644 --- a/imessage-database/src/message_types/text_effects/text_effect.rs +++ b/imessage-database/src/message_types/text_effects/text_effect.rs @@ -1,4 +1,4 @@ -use super::{ +use crate::message_types::text_effects::{ animation::Animation, detected::{ address::DetectedAddress, currency::DetectedCurrency, flight::Flight, diff --git a/imessage-database/test_data/digital_touch_message/bg.png b/imessage-database/test_data/digital_touch_message/bg.png new file mode 100644 index 00000000..72b8680e Binary files /dev/null and b/imessage-database/test_data/digital_touch_message/bg.png differ diff --git a/imessage-database/test_data/digital_touch_message/hi.bin b/imessage-database/test_data/digital_touch_message/hi.bin new file mode 100644 index 00000000..6300b992 Binary files /dev/null and b/imessage-database/test_data/digital_touch_message/hi.bin differ diff --git a/imessage-database/test_data/digital_touch_message/image.bin b/imessage-database/test_data/digital_touch_message/image.bin new file mode 100644 index 00000000..d00cb7bd Binary files /dev/null and b/imessage-database/test_data/digital_touch_message/image.bin differ diff --git a/imessage-database/test_data/digital_touch_message/image_with_drawing.bin b/imessage-database/test_data/digital_touch_message/image_with_drawing.bin new file mode 100644 index 00000000..14fa9d5f Binary files /dev/null and b/imessage-database/test_data/digital_touch_message/image_with_drawing.bin differ diff --git a/imessage-database/test_data/digital_touch_message/video.bin b/imessage-database/test_data/digital_touch_message/video.bin new file mode 100644 index 00000000..e0d935bc Binary files /dev/null and b/imessage-database/test_data/digital_touch_message/video.bin differ diff --git a/imessage-exporter/src/exporters/formatter.rs b/imessage-exporter/src/exporters/formatter.rs index ef15aa68..15a2cf74 100644 --- a/imessage-exporter/src/exporters/formatter.rs +++ b/imessage-exporter/src/exporters/formatter.rs @@ -5,7 +5,7 @@ use imessage_database::{ app::AppMessage, app_store::AppStoreMessage, collaboration::CollaborationMessage, - digital_touch::DigitalTouch, + digital_touch::DigitalTouchMessage, edited::EditedMessage, handwriting::HandwrittenMessage, music::MusicMessage, @@ -151,7 +151,7 @@ pub(crate) trait BalloonFormatter { /// Format a handwritten note message. fn format_handwriting(&self, msg: &Message, balloon: &HandwrittenMessage) -> String; /// Format a digital touch message. - fn format_digital_touch(&self, msg: &Message, balloon: &DigitalTouch) -> String; + fn format_digital_touch(&self, msg: &Message, balloon: &DigitalTouchMessage) -> String; /// Format an Apple Pay message. fn format_apple_pay(&self, balloon: &AppMessage) -> String; /// Format a Fitness message. diff --git a/imessage-exporter/src/exporters/html/balloons.rs b/imessage-exporter/src/exporters/html/balloons.rs index e2390ddc..eb0b33f2 100644 --- a/imessage-exporter/src/exporters/html/balloons.rs +++ b/imessage-exporter/src/exporters/html/balloons.rs @@ -1,11 +1,19 @@ +use std::path::PathBuf; + use imessage_database::{ message_types::{ - app::AppMessage, app_store::AppStoreMessage, collaboration::CollaborationMessage, - digital_touch::DigitalTouch, handwriting::HandwrittenMessage, music::MusicMessage, - placemark::PlacemarkMessage, polls::Poll, url::URLMessage, + app::AppMessage, + app_store::AppStoreMessage, + collaboration::CollaborationMessage, + digital_touch::{DigitalTouchMessage, ImageBackdrop, media::MediaKind}, + handwriting::HandwrittenMessage, + music::MusicMessage, + placemark::PlacemarkMessage, + polls::Poll, + url::URLMessage, }, tables::{ - attachment::Attachment, + attachment::{Attachment, MediaType}, messages::{Message, models::AttachmentMeta}, }, }; @@ -16,13 +24,54 @@ use crate::exporters::{ HTML, safe::Html, view_model::{ - AppCardVM, AppStoreVM, ApplePayVM, CheckInVM, CollaborationVM, DigitalTouchVM, - FindMyVM, MusicVM, PlacemarkVM, PollOptionVM, PollVM, UrlVM, + AppCardVM, AppStoreVM, ApplePayVM, AttachmentVM, AttachmentVariant, CheckInVM, + CollaborationVM, FindMyVM, MusicVM, PlacemarkVM, PollOptionVM, PollVM, UrlVM, }, }, - shared::{balloon::resolve_check_in_footer, render::render_template}, + shared::{ + attachment::prepare_attachment, balloon::resolve_check_in_footer, driver::ExportState, + render::render_template, + }, }; +use crate::app::runtime::Config; + +/// Resolve and prepare the photo or video backing a Digital Touch media message. +/// +/// The media is a normal attachment on the message row: it is looked up with +/// [`Attachment::from_message`], run through the attachment manager, and +/// confirmed present on disk. Returns `None` for any condition that prevents a +/// usable backing file, including a missing attachment row, unresolved source +/// path, missing filename, copy/convert/decryption failure, or missing final +/// file. The caller intentionally falls back to the labeled black canvas. +fn digital_touch_attachment( + config: &Config, + state: &ExportState, + msg: &Message, +) -> Option { + let mut attachment = Attachment::from_message(config.data_source.db(), msg) + .ok()? + .into_iter() + .next()?; + + // Prepare this as a normal attachment. Depending on the attachment-manager + // mode this may copy, convert, reuse an existing export copy, or leave the + // original path in place. If preparation fails, the render falls back to the + // labeled black canvas. + prepare_attachment(config, state, &mut attachment, msg).ok()?; + + // Keep the attachment only when the referenced file is actually present. + let on_disk = match &attachment.copied_path { + Some(path) => path.clone(), + None => PathBuf::from(attachment.resolved_attachment_path( + &config.options.platform, + &config.options.db_path, + config.options.attachment_root.as_deref(), + )?), + }; + on_disk.exists().then_some(attachment) +} + // MARK: Balloons impl BalloonFormatter for HTML<'_> { fn format_url(&self, msg: &Message, balloon: &URLMessage) -> String { @@ -89,10 +138,14 @@ impl BalloonFormatter for HTML<'_> { balloon.render_svg() } - fn format_digital_touch(&self, _: &Message, balloon: &DigitalTouch) -> String { - render_template(&DigitalTouchVM { - debug: format!("{balloon:?}"), - }) + fn format_digital_touch(&self, msg: &Message, balloon: &DigitalTouchMessage) -> String { + // Wrap in `.digital_touch` (inside the standard `app` card) as the CSS + // hook that caps the bubble to the card's width; the opaque content then + // fills that bubble, covering the `app` card's white background. + format!( + r#"
{}
"#, + self.digital_touch_body(msg, balloon) + ) } fn format_apple_pay(&self, balloon: &AppMessage) -> String { @@ -181,6 +234,36 @@ impl BalloonFormatter for HTML<'_> { } impl HTML<'_> { + /// Render the inner markup for a Digital Touch message, dispatching on the + /// parsed [`MediaKind`]: a video plays as a standalone `