Update API endpoints

src/api.nim

@@ -25,27 +25,27 @@ proc mediaUrl(id, cursor: string; count=20): ApiReq =
   )
 
 proc userTweetsUrl(id: string; cursor: string): ApiReq =
-  result = ApiReq(
-    # cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
-  )
-  # might change this in the future pending testing
-  result.cookie = result.oauth
+  return apiReq(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
+  # result = ApiReq(
+  #   cookie: apiUrl(graphUserTweets, userTweetsVars % [id, cursor], userTweetsFieldToggles),
+  #   oauth: apiUrl(graphUserTweetsV2, restIdVars % [id, cursor, "20"])
+  # )
 
 proc userTweetsAndRepliesUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = userTweetsAndRepliesVars % [id, cursor]
-  result = ApiReq(
-    cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
-    oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
-  )
+  return apiReq(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
+  #let cookieVars = userTweetsAndRepliesVars % [id, cursor]
+  # result = ApiReq(
+  #   cookie: apiUrl(graphUserTweetsAndReplies, cookieVars, userTweetsFieldToggles),
+  #   oauth: apiUrl(graphUserTweetsAndRepliesV2, restIdVars % [id, cursor, "20"])
+  # )
 
 proc tweetDetailUrl(id: string; cursor: string): ApiReq =
-  let cookieVars = tweetDetailVars % [id, cursor]
-  result = ApiReq(
-    # cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
-    cookie: apiUrl(graphTweet, tweetVars % [id, cursor]),
-    oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
-  )
+  return apiReq(graphTweet, tweetVars % [id, cursor])
+  # let cookieVars = tweetDetailVars % [id, cursor]
+  # result = ApiReq(
+  #   cookie: apiUrl(graphTweetDetail, cookieVars, tweetDetailFieldToggles),
+  #   oauth: apiUrl(graphTweet, tweetVars % [id, cursor])
+  # )
 
 proc userUrl(username: string): ApiReq =
   let cookieVars = """{"screen_name":"$1","withGrokTranslatedBio":false}""" % username
@@ -184,13 +184,13 @@ proc getGraphTweetSearch*(query: Query; after=""): Future[Timeline] {.async.} =
   var
     variables = %*{
       "rawQuery": q,
-      "query_source": "typedQuery",
       "count": 20,
+      "querySource": "typed_query",
       "product": "Latest",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
     }
+
   if after.len > 0 and maxId.len == 0:
     variables["cursor"] = % after
   let
@@ -212,12 +212,11 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   var
     variables = %*{
       "rawQuery": query.text,
-      "query_source": "typedQuery",
       "count": 20,
+      "querySource": "typed_query",
       "product": "People",
-      "withDownvotePerspective": false,
-      "withReactionsMetadata": false,
-      "withReactionsPerspective": false
+      "withGrokTranslatedBio":true,
+      "withQuickPromoteEligibilityTweetFields":false
     }
   if after.len > 0:
     variables["cursor"] = % after

src/apiutils.nim

@@ -84,12 +84,13 @@ proc genHeaders*(session: Session, url: Uri): Future[HttpHeaders] {.async.} =
     result["x-twitter-auth-type"] = "OAuth2Session"
     result["x-csrf-token"] = session.ct0
     result["cookie"] = getCookieHeader(session.authToken, session.ct0)
+    result["referer"] = "https://x.com/"
     result["sec-ch-ua"] = """"Google Chrome";v="142", "Chromium";v="142", "Not A(Brand";v="24""""
     result["sec-ch-ua-mobile"] = "?0"
     result["sec-ch-ua-platform"] = "Windows"
     result["sec-fetch-dest"] = "empty"
     result["sec-fetch-mode"] = "cors"
-    result["sec-fetch-site"] = "same-site"
+    result["sec-fetch-site"] = "same-origin"
     if disableTid or "/1.1/" in url.path:
       result["authorization"] = bearerToken2
     else:
@@ -114,7 +115,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
 
   try:
     var resp: AsyncResponse
-    pool.use(await genHeaders(session, url)):
+    let headers = await genHeaders(session, url)
+
+    pool.use(headers):
       template getContent =
         # TODO: this is a temporary simple implementation
         if apiProxy.len > 0 and "/1.1/" notin url.path:

src/consts.nim

@@ -7,109 +7,70 @@ const
   bearerToken* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
   bearerToken2* = "Bearer AAAAAAAAAAAAAAAAAAAAAFXzAwAAAAAAMHCxpeSDG1gLNLghVe8d74hl6k4%3DRUMF4xAQLsbeBhTSRrCiQpJtxoGWeyHrDb5te2jpGskWDFW82F"
 
-  graphUser* = "-oaLodhGbbnzJBACb1kk2Q/UserByScreenName"
-  graphUserV2* = "WEoGnYB0EG1yGwamDCF6zg/UserResultByScreenNameQuery"
-  graphUserById* = "VN33vKXrPT7p35DgNR27aw/UserResultByIdQuery"
-  graphUserTweetsV2* = "6QdSuZ5feXxOadEdXa4XZg/UserWithProfileTweetsQueryV2"
-  graphUserTweetsAndRepliesV2* = "BDX77Xzqypdt11-mDfgdpQ/UserWithProfileTweetsAndRepliesQueryV2"
-  graphUserTweets* = "oRJs8SLCRNRbQzuZG93_oA/UserTweets"
-  graphUserTweetsAndReplies* = "kkaJ0Mf34PZVarrxzLihjg/UserTweetsAndReplies"
-  graphUserMedia* = "36oKqyQ7E_9CmtONGjJRsA/UserMedia"
-  graphUserMediaV2* = "bp0e_WdXqgNBIwlLukzyYA/MediaTimelineV2"
-  graphTweet* = "b4pV7sWOe97RncwHcGESUA/ConversationTimeline"
-  graphTweetDetail* = "YVyS4SfwYW7Uw5qwy0mQCA/TweetDetail"
-  graphTweetResult* = "nzme9KiYhfIOrrLrPP_XeQ/TweetResultByIdQuery"
-  graphTweetEditHistory* = "upS9teTSG45aljmP9oTuXA/TweetEditHistory"
-  graphSearchTimeline* = "bshMIjqDk8LTXTq4w91WKw/SearchTimeline"
-  graphListById* = "cIUpT1UjuGgl_oWiY7Snhg/ListByRestId"
-  graphListBySlug* = "K6wihoTiTrzNzSF8y1aeKQ/ListBySlug"
-  graphListMembers* = "fuVHh5-gFn8zDBBxb8wOMA/ListMembers"
-  graphListTweets* = "VQf8_XQynI3WzH6xopOMMQ/ListTimeline"
-  graphAboutAccount* = "zs_jFPFT78rBpXv9Z3U2YQ/AboutAccountQuery"
+  graphUser* = "IGgvgiOx4QZndDHuD3x9TQ/UserByScreenName"
+  graphUserV2* = "-ZzAG_Bckx16LMbEvHC3lg/UserResultByScreenNameQuery"
+  graphUserById* = "-DAaa9jPxPswYeI2fZ9rug/UserResultByIdQuery"
+  graphUserTweetsV2* = "PHTSTXqZYuHIeK4B1HQprQ/UserWithProfileTweetsQueryV2"
+  graphUserTweetsAndRepliesV2* = "AcYHjc_YAx-9_rKWdMsKvA/UserWithProfileTweetsAndRepliesQueryV2"
+  graphUserTweets* = "PNd0vlufvrcIwrAnBYKE9g/UserTweets"
+  graphUserTweetsAndReplies* = "EqtpEwt0CoQXmDfq5DKH0A/UserTweetsAndReplies"
+  graphUserMedia* = "g_rGPF0fLON-M9cyVjXuzA/UserMedia"
+  graphUserMediaV2* = "WK111rbR0vM0ZX4lyZCYjw/MediaTimelineV2"
+  graphTweet* = "OZMbEnEa96AN8Pq6HyTWdw/ConversationTimeline"
+  graphTweetDetail* = "6uCvnic3m5reVuehkvHa3w/TweetDetail"
+  graphTweetResult* = "xYOrBQoTlfKJJPsX76MZEw/TweetResultByIdQuery"
+  graphTweetEditHistory* = "MGElmrYILE8wUfI8GorUYA/TweetEditHistory"
+  graphSearchTimeline* = "-TFXKoMnMTKdEXcCn-eahw/SearchTimeline"
 
-  graphBroadcast* = "0nMmbMh-_JwwRRFNXkyH3Q/BroadcastQuery"
+  graphListById* = "t9AbdyHaJVfjL9jsODwgpQ/ListByRestId"
+  graphListBySlug* = "LDQpQ89B5ipR8izCKrWU0g/ListBySlug"
+  graphListMembers* = "EM7YRaM3gCnzDESmchA7RA/ListMembers"
+  graphListTweets* = "0QJtcuMzVywHGAWD6Dtjlw/ListTimeline"
+  graphAboutAccount* = "zUnx-DLN9dkwOkNhTLySjg/AboutAccountQuery"
+
+  graphBroadcast* = "FJLCzpXCLPM1jUZqmM7oEA/BroadcastQuery"
   restLiveStream* = "1.1/live_video_stream/status/"
 
   gqlFeatures* = """{
-  "android_ad_formats_media_component_render_overlay_enabled": false,
-  "android_graphql_skip_api_media_color_palette": false,
-  "android_professional_link_spotlight_display_enabled": false,
-  "articles_api_enabled": false,
-  "articles_preview_enabled": true,
-  "blue_business_profile_image_shape_enabled": false,
-  "c9s_tweet_anatomy_moderator_badge_enabled": true,
-  "commerce_android_shop_module_enabled": false,
-  "communities_web_enable_tweet_community_results_fetch": true,
-  "creator_subscriptions_quote_tweet_preview_enabled": false,
-  "creator_subscriptions_subscription_count_enabled": false,
-  "creator_subscriptions_tweet_preview_api_enabled": true,
-  "freedom_of_speech_not_reach_fetch_enabled": true,
-  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
-  "grok_android_analyze_trend_fetch_enabled": false,
-  "grok_translations_community_note_auto_translation_is_enabled": false,
-  "grok_translations_community_note_translation_is_enabled": false,
-  "grok_translations_post_auto_translation_is_enabled": false,
-  "grok_translations_timeline_user_bio_auto_translation_is_enabled": false,
-  "hidden_profile_likes_enabled": false,
-  "highlights_tweets_tab_ui_enabled": false,
-  "immersive_video_status_linkable_timestamps": false,
-  "interactive_text_enabled": false,
-  "longform_notetweets_consumption_enabled": true,
-  "longform_notetweets_inline_media_enabled": true,
-  "longform_notetweets_richtext_consumption_enabled": true,
-  "longform_notetweets_rich_text_read_enabled": true,
-  "mobile_app_spotlight_module_enabled": false,
-  "payments_enabled": false,
-  "post_ctas_fetch_enabled": true,
-  "premium_content_api_read_enabled": false,
+  "rweb_video_screen_enabled": false,
+  "rweb_cashtags_enabled": true,
   "profile_label_improvements_pcf_label_in_post_enabled": true,
-  "profile_label_improvements_pcf_label_in_profile_enabled": false,
-  "responsive_web_edit_tweet_api_enabled": true,
-  "responsive_web_enhance_cards_enabled": false,
-  "responsive_web_graphql_exclude_directive_enabled": true,
-  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_profile_redirect_enabled": false,
+  "rweb_tipjar_consumption_enabled": false,
+  "verified_phone_label_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
   "responsive_web_graphql_timeline_navigation_enabled": true,
-  "responsive_web_grok_analysis_button_from_backend": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "premium_content_api_read_enabled": false,
+  "communities_web_enable_tweet_community_results_fetch": true,
+  "c9s_tweet_anatomy_moderator_badge_enabled": true,
   "responsive_web_grok_analyze_button_fetch_trends_enabled": false,
   "responsive_web_grok_analyze_post_followups_enabled": true,
-  "responsive_web_grok_annotations_enabled": true,
-  "responsive_web_grok_community_note_auto_translation_is_enabled": false,
-  "responsive_web_grok_image_annotation_enabled": true,
-  "responsive_web_grok_imagine_annotation_enabled": true,
-  "responsive_web_grok_share_attachment_enabled": true,
-  "responsive_web_grok_show_grok_translated_post": false,
+  "rweb_cashtags_composer_attachment_enabled": true,
   "responsive_web_jetfuel_frame": true,
-  "responsive_web_media_download_video_enabled": false,
-  "responsive_web_profile_redirect_enabled": false,
-  "responsive_web_text_conversations_enabled": false,
-  "responsive_web_twitter_article_notes_tab_enabled": false,
+  "responsive_web_grok_share_attachment_enabled": true,
+  "responsive_web_grok_annotations_enabled": true,
+  "articles_preview_enabled": true,
+  "responsive_web_edit_tweet_api_enabled": true,
+  "rweb_conversational_replies_downvote_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": true,
+  "view_counts_everywhere_api_enabled": true,
+  "longform_notetweets_consumption_enabled": true,
   "responsive_web_twitter_article_tweet_consumption_enabled": true,
-  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
-  "rweb_lists_timeline_redesign_enabled": true,
-  "rweb_tipjar_consumption_enabled": true,
-  "rweb_video_screen_enabled": false,
-  "rweb_video_timestamps_enabled": false,
-  "spaces_2022_h2_clipping": true,
-  "spaces_2022_h2_spaces_communities": true,
+  "content_disclosure_indicator_enabled": true,
+  "content_disclosure_ai_generated_indicator_enabled": true,
+  "responsive_web_grok_show_grok_translated_post": true,
+  "responsive_web_grok_analysis_button_from_backend": true,
+  "post_ctas_fetch_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": true,
   "standardized_nudges_misinfo": true,
-  "subscriptions_feature_can_gift_premium": false,
-  "subscriptions_verification_info_enabled": true,
-  "subscriptions_verification_info_is_identity_verified_enabled": false,
-  "subscriptions_verification_info_reason_enabled": true,
-  "subscriptions_verification_info_verified_since_enabled": true,
-  "super_follow_badge_privacy_enabled": false,
-  "super_follow_exclusive_tweet_notifications_enabled": false,
-  "super_follow_tweet_api_enabled": false,
-  "super_follow_user_api_enabled": false,
-  "tweet_awards_web_tipping_enabled": false,
   "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": true,
-  "tweetypie_unmention_optimization_enabled": false,
-  "unified_cards_ad_metadata_container_dynamic_card_content_query_enabled": false,
-  "unified_cards_destination_url_params_enabled": false,
-  "verified_phone_label_enabled": false,
-  "vibe_api_enabled": false,
-  "view_counts_everywhere_api_enabled": true,
-  "hidden_profile_subscriptions_enabled": false
+  "longform_notetweets_rich_text_read_enabled": true,
+  "longform_notetweets_inline_media_enabled": false,
+  "responsive_web_grok_image_annotation_enabled": true,
+  "responsive_web_grok_imagine_annotation_enabled": true,
+  "responsive_web_grok_community_note_auto_translation_is_enabled": true,
+  "responsive_web_enhance_cards_enabled": false
 }""".replace(" ", "").replace("\n", "")
 
   tweetVars* = """{

src/parser.nim

@@ -4,7 +4,7 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
-proc parseGraphTweet(js: JsonNode): Tweet
+proc parseGraphTweet*(js: JsonNode): Tweet
 
 proc parseVerifiedType(s: string; current: VerifiedType): VerifiedType =
   try: parseEnum[VerifiedType](s)
@@ -46,10 +46,10 @@ proc parseUser(js: JsonNode; id=""): User =
 proc parseGraphUser(js: JsonNode): User =
   var user = js{"user_result", "result"}
   if user.isNull:
-    user = ? js{"user_results", "result"}
+    user = js{"user_results", "result"}
 
   if user.isNull:
-    if js{"core"}.notNull and js{"legacy"}.notNull:
+    if js{"core"}.notNull:
       user = js
     else:
       return
@@ -61,6 +61,7 @@ proc parseGraphUser(js: JsonNode): User =
 
   # fallback to support UserMedia/recent GraphQL updates
   if result.username.len == 0:
+    result.id = user{"rest_id"}.getStr
     result.username = user{"core", "screen_name"}.getStr
     result.fullname = user{"core", "name"}.getStr
     result.userPic = user{"avatar", "image_url"}.getImageStr.replace("_normal", "")
@@ -261,6 +262,20 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
             durationMs: mediaInfo{"duration_millis"}.getInt,
             variants: parseVideoVariants(mediaInfo{"variants"})
           ))
+
+          # Parse source user for video attribution
+          with sourceUser, mediaEntity{"source_user_results", "result"}:
+            if result.attribution.isNone:
+              let
+                expanded = mediaEntity{"expanded_url"}.getStr
+                pathStart = expanded.find('/', expanded.find("://") + 3)
+              if pathStart >= 0:
+                result.attributionLink = expanded[pathStart .. ^1].replace("/video/1", "")
+              result.attribution = some(User(
+                id: sourceUser{"rest_id"}.getStr,
+                fullname: sourceUser{"core", "name"}.getStr,
+                userPic: sourceUser{"avatar", "image_url"}.getImageStr.replace("_normal", "")
+              ))
         of "ApiGif":
           parsedMedia.addMedia(Gif(
             url: mediaInfo{"variants"}[0]{"url"}.getImageStr,
@@ -428,13 +443,13 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
   # graphql
   with rt, js{"retweeted_status_result", "result"}:
     # needed due to weird edgecase where the actual tweet data isn't included
-    if "legacy" in rt:
+    if "legacy" in rt or "rest_id" in rt:
       result.retweet = some parseGraphTweet(rt)
       return
 
   with reposts, js{"repostedStatusResults"}:
     with rt, reposts{"result"}:
-      if "legacy" in rt:
+      if "legacy" in rt or "rest_id" in rt:
         result.retweet = some parseGraphTweet(rt)
         return
 
@@ -449,7 +464,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.poll = some parsePoll(jsCard)
     elif name == "amplify":
       result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-    else:
+    elif name.len > 0 and jsCard{"binding_values"}.notNull:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
   result.expandTweetEntities(js)
@@ -469,7 +484,7 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parseGraphTweet(js: JsonNode): Tweet =
+proc parseGraphTweet*(js: JsonNode): Tweet =
   if js.kind == JNull:
     return Tweet()
 
@@ -537,10 +552,21 @@ proc parseGraphTweet(js: JsonNode): Tweet =
         result.poll = some parsePoll(jsCard)
       elif name == "amplify":
         result.media.addMedia(parsePromoVideo(jsCard{"binding_values"}))
-      else:
+      elif name.len > 0 and jsCard{"binding_values"}.notNull:
         result.card = some parseCard(jsCard, js{"url_entities"})
 
     result.expandTweetEntitiesV2(js)
+
+    # Strip video source URL from text (for videos from other tweets)
+    with mediaEntities, js{"media_entities"}:
+      for m in mediaEntities:
+        if "source_status_id_str" in m:
+          let mediaUrl = m{"url"}.getStr
+          if mediaUrl.len > 0:
+            let idx = result.text.rfind(mediaUrl)
+            if idx >= 0:
+              result.text = result.text[0 ..< idx].strip()
+            break
   else:
     result = parseTweet(js{"legacy"}, jsCard, replyId)
     result.id = js{"rest_id"}.getId
@@ -559,6 +585,12 @@ proc parseGraphTweet(js: JsonNode): Tweet =
 
   parseMediaEntities(js, result)
 
+  # Handle retweets - check both legacy and top-level paths
+  with reposts, js{"legacy", "repostedStatusResults"}:
+    with rt, reposts{"result"}:
+      if "legacy" in rt or "rest_id" in rt:
+        result.retweet = some parseGraphTweet(rt)
+
   with quoted, js{"quoted_status_result", "result"}:
     result.quote = some(parseGraphTweet(quoted))