Replace old v1 photo rail API with gql

src/api.nim

@@ -136,13 +136,13 @@ proc getGraphUserSearch*(query: Query; after=""): Future[Result[User]] {.async.}
   result = parseGraphSearch[User](await fetch(url, Api.search), after)
   result.query = query
 
-proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
+proc getPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+  if id.len == 0: return
   let
-    ps = genParams({"screen_name": name, "trim_user": "true"},
-                    count="18", ext=false)
-    url = photoRail ? ps
-  result = parsePhotoRail(await fetch(url, Api.photoRail))
+    variables = userTweetsVariables % [id, ""]
+    params = {"variables": variables, "features": gqlFeatures}
+    url = graphUserMedia ? params
+  result = parseGraphPhotoRail(await fetch(url, Api.userMedia))
 
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)

src/apiutils.nim

@@ -10,25 +10,6 @@ const
 
 var pool: HttpPool
 
-proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
-                count="20"; ext=true): seq[(string, string)] =
-  result = timelineParams
-  for p in pars:
-    result &= p
-  if ext:
-    result &= ("include_ext_alt_text", "1")
-    result &= ("include_ext_media_stats", "1")
-    result &= ("include_ext_media_availability", "1")
-  if count.len > 0:
-    result &= ("count", count)
-  if cursor.len > 0:
-    # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to turn them back into a plus
-    if " " in cursor:
-      result &= ("cursor", cursor.replace(" ", "+"))
-    else:
-      result &= ("cursor", cursor)
-
 proc getOauthHeader(url, oauthToken, oauthTokenSecret: string): string =
   let
     encodedUrl = url.replace(",", "%2C").replace("+", "%20")

src/auth.nim

@@ -10,7 +10,6 @@ const
   apiMaxReqs: Table[Api, int] = {
     Api.search: 50,
     Api.tweetDetail: 150,
-    Api.photoRail: 180,
     Api.userTweets: 500,
     Api.userTweetsAndReplies: 500,
     Api.userMedia: 500,

src/consts.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils, strutils
+import uri, strutils
 
 const
   consumerKey* = "3nVuSoBZnx6U4vzUxf5w"
@@ -8,8 +8,6 @@ const
   api = parseUri("https://api.twitter.com")
   activate* = $(api / "1.1/guest/activate.json")
 
-  photoRail* = api / "1.1/statuses/media_timeline.json"
-
   graphql = api / "graphql"
   graphUser* = graphql / "u7wQyGi6oExe8_TRWGMq4Q/UserResultByScreenNameQuery"
   graphUserById* = graphql / "oPppcargziU1uDQHAUmH-A/UserResultByIdQuery"
@@ -24,22 +22,6 @@ const
   graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
   graphListTweets* = graphql / "BbGLL1ZfMibdFNWlk7a0Pw/ListTimeline"
 
-  timelineParams* = {
-    "include_can_media_tag": "1",
-    "include_cards": "1",
-    "include_entities": "1",
-    "include_profile_interstitial_type": "0",
-    "include_quote_count": "0",
-    "include_reply_count": "0",
-    "include_user_entities": "0",
-    "include_ext_reply_count": "0",
-    "include_ext_media_color": "0",
-    "cards_platform": "Web-13",
-    "tweet_mode": "extended",
-    "send_error_codes": "1",
-    "simple_quoted_tweet": "1"
-  }.toSeq
-
   gqlFeatures* = """{
   "android_graphql_skip_api_media_color_palette": false,
   "blue_business_profile_image_shape_enabled": false,

src/parser.nim

@@ -289,23 +289,6 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
       result.text.removeSuffix(" Learn more.")
       result.available = false
 
-proc parsePhotoRail*(js: JsonNode): PhotoRail =
-  with error, js{"error"}:
-    if error.getStr == "Not authorized.":
-      return
-
-  for tweet in js:
-    let
-      t = parseTweet(tweet, js{"tweet_card"})
-      url = if t.photos.len > 0: t.photos[0]
-            elif t.video.isSome: get(t.video).thumb
-            elif t.gif.isSome: get(t.gif).thumb
-            elif t.card.isSome: get(t.card).image
-            else: ""
-
-    if url.len == 0: continue
-    result.add GalleryPhoto(url: url, tweetId: $t.id)
-
 proc parseGraphTweet(js: JsonNode; isLegacy=false): Tweet =
   if js.kind == JNull:
     return Tweet()
@@ -445,6 +428,34 @@ proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Profile =
             tweet.id = parseBiggestInt(entryId)
         result.pinned = some tweet
 
+proc parseGraphPhotoRail*(js: JsonNode): PhotoRail =
+  result = @[]
+
+  let instructions =
+    ? js{"data", "user_result", "result", "timeline_response", "timeline", "instructions"}
+
+  for i in instructions:
+    if i{"__typename"}.getStr == "TimelineAddEntries":
+      for e in i{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "content", "tweetResult", "result"}:
+            let t = parseGraphTweet(tweetResult, false)
+            if not t.available:
+              t.id = parseBiggestInt(entryId.getId())
+
+            let url = 
+              if t.photos.len > 0: t.photos[0]
+              elif t.video.isSome: get(t.video).thumb
+              elif t.gif.isSome: get(t.gif).thumb
+              elif t.card.isSome: get(t.card).image
+              else: ""
+
+            result.add GalleryPhoto(url: url, tweetId: $t.id)
+
+            if result.len == 16:
+              break
+
 proc parseGraphSearch*[T: User | Tweets](js: JsonNode; after=""): Result[T] =
   result = Result[T](beginning: after.len == 0)
 

src/redis_cache.nim

@@ -86,7 +86,7 @@ proc cache*(data: List) {.async.} =
   await setEx(data.listKey, listCacheTime, compress(toFlatty(data)))
 
 proc cache*(data: PhotoRail; name: string) {.async.} =
-  await setEx("pr:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
+  await setEx("pr2:" & toLower(name), baseCacheTime * 2, compress(toFlatty(data)))
 
 proc cache*(data: User) {.async.} =
   if data.username.len == 0: return
@@ -158,14 +158,14 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
 #     if not result.isNil:
 #       await cache(result)
 
-proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =
-  if name.len == 0: return
-  let rail = await get("pr:" & toLower(name))
+proc getCachedPhotoRail*(id: string): Future[PhotoRail] {.async.} =
+  if id.len == 0: return
+  let rail = await get("pr2:" & toLower(id))
   if rail != redisNil:
     rail.deserialize(PhotoRail)
   else:
-    result = await getPhotoRail(name)
-    await cache(result, name)
+    result = await getPhotoRail(id)
+    await cache(result, id)
 
 proc getCachedList*(username=""; slug=""; id=""): Future[List] {.async.} =
   let list = if id.len == 0: redisNil

src/routes/timeline.nim

@@ -47,7 +47,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
   let
     rail =
       skipIf(skipRail or query.kind == media, @[]):
-        getCachedPhotoRail(name)
+        getCachedPhotoRail(userId)
 
     user = getCachedUser(name)
 

src/types.nim

@@ -15,7 +15,6 @@ type
   Api* {.pure.} = enum
     tweetDetail
     tweetResult
-    photoRail
     search
     list
     listBySlug