parse recently added long tweets

src/consts.nim

@@ -85,7 +85,7 @@ const
   "responsive_web_edit_tweet_api_enabled": false,
   "tweetypie_unmention_optimization_enabled": false,
   "vibe_api_enabled": false,
-  "longform_notetweets_consumption_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
   "responsive_web_text_conversations_enabled": false,
   "responsive_web_enhance_cards_enabled": false,
   "interactive_text_enabled": false

src/parser.nim

@@ -385,6 +385,10 @@ proc parseGraphTweet(js: JsonNode): Tweet =
   result = parseTweet(js{"legacy"}, jsCard)
   result.user = parseUser(js{"core", "user_results", "result", "legacy"})
 
+  var note_tweet = js{"note_tweet", "note_tweet_results", "result"}
+  if note_tweet.kind != JNull:
+    result.expandNoteTweetEntities(note_tweet)
+
   if result.quote.isSome:
     result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
 

src/parserutils.nim

@@ -289,3 +289,45 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
 
   tweet.text = orig.replacedWith(replacements, textSlice)
                    .strip(leading=false)
+
+proc expandNoteTweetEntities*(tweet: Tweet; noteTweet: JsonNode) =
+  let
+    text = noteTweet{"text"}.getStr
+    orig = text.toRunes
+    ent = ? noteTweet{"entity_set"}
+    hasCard = tweet.card.isSome
+
+  var replacements = newSeq[ReplaceSlice]()
+
+  with urls, ent{"urls"}:
+    for u in urls:
+      let urlStr = u["url"].getStr
+      if urlStr.len == 0 or urlStr notin text:
+        continue
+      replacements.extractUrls(u, orig.len, hideTwitter = false)
+      if hasCard and u{"url"}.getStr == get(tweet.card).url:
+        get(tweet.card).url = u{"expanded_url"}.getStr
+
+  if "hashtags" in ent:
+    for hashtag in ent["hashtags"]:
+      replacements.extractHashtags(hashtag)
+
+  if "symbols" in ent:
+    for symbol in ent["symbols"]:
+      replacements.extractHashtags(symbol)
+
+  if "user_mentions" in ent:
+    for mention in ent["user_mentions"]:
+      let
+        name = mention{"screen_name"}.getStr
+        slice = mention.extractSlice
+        idx = tweet.reply.find(name)
+
+      replacements.add ReplaceSlice(kind: rkMention, slice: slice,
+        url: "/" & name, display: mention["name"].getStr)
+
+  replacements.deduplicate
+  replacements.sort(cmp)
+
+  tweet.text = orig.replacedWith(replacements, 0..orig.len)
+                   .strip(leading=false)