Cache video token

src/api.nim

@@ -18,12 +18,19 @@ const
   videoUrl = "videos/tweet/config/$1.json"
   tokenUrl = "guest/activate.json"
 
-proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} =
+var
+  token = ""
+  tokenUpdated: Time
+  tokenLifetime = initDuration(minutes=10)
+
+template newClient() {.dirty.} =
   var client = newAsyncHttpClient()
   defer: client.close()
-
   client.headers = headers
 
+proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} =
+  newClient()
+
   var resp = ""
   try:
     resp = await client.getContent($url)
@@ -37,10 +44,7 @@ proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.
     return parseHtml(resp)
 
 proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
-  var client = newAsyncHttpClient()
-  defer: client.close()
-
-  client.headers = headers
+  newClient()
 
   var resp = ""
   try:
@@ -50,38 +54,12 @@ proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
 
   return parseJson(resp)
 
-proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
-  let
-    url = base / profileIntentUrl ? {"screen_name": username}
-    html = await fetchHtml(url, headers)
-
-  result = parseIntentProfile(html)
-
-proc getProfile*(username: string): Future[Profile] {.async.} =
-  let headers = newHttpHeaders({
-    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
-    "Referer": $(base / username),
-    "User-Agent": agent,
-    "X-Twitter-Active-User": "yes",
-    "X-Requested-With": "XMLHttpRequest",
-    "Accept-Language": "en-US,en;q=0.9"
-  })
-
-  let
-    params = {
-      "screen_name": username,
-      "wants_hovercard": "true",
-      "_": $(epochTime().int)
-    }
-    url = base / profilePopupUrl ? params
-    html = await fetchHtml(url, headers, jsonKey="html")
-
-  if not html.querySelector(".ProfileCard-sensitiveWarningContainer").isNil:
-    return await getProfileFallback(username, headers)
+proc getGuestToken(): Future[string] {.async.} =
+  if getTime() - tokenUpdated < tokenLifetime:
+    return token
 
-  result = parsePopupProfile(html)
+  tokenUpdated = getTime()
 
-proc getGuestToken(): Future[string] {.async.} =
   let headers = newHttpHeaders({
     "Accept": "application/json, text/javascript, */*; q=0.01",
     "Referer": $base,
@@ -89,16 +67,18 @@ proc getGuestToken(): Future[string] {.async.} =
     "Authorization": auth
   })
 
-  let client = newAsyncHttpClient()
-  client.headers = headers
+  newClient()
 
   let
     url = apibase / tokenUrl
     json = parseJson(await client.postContent($url))
 
   result = json["guest_token"].to(string)
+  token = result
 
 proc getVideo*(tweet: Tweet; token: string) {.async.} =
+  if not tweet.video.isSome: return
+
   let headers = newHttpHeaders({
     "Accept": "application/json, text/javascript, */*; q=0.01",
     "Referer": tweet.link,
@@ -111,19 +91,14 @@ proc getVideo*(tweet: Tweet; token: string) {.async.} =
     url = apiBase / (videoUrl % tweet.id)
     json = await fetchJson(url, headers)
 
-
   tweet.video = some(parseVideo(json))
 
-proc getVideos*(tweets: Tweets; token="") {.async.} =
-  if not tweets.anyIt(it.video.isSome): return
-
-  var
-    token = if token.len > 0: token else: await getGuestToken()
-    videoFuts: seq[Future[void]]
+proc getVideos*(tweets: Tweets) {.async.} =
+  var token = await getGuestToken()
+  var videoFuts: seq[Future[void]]
 
-  for tweet in tweets:
-    if tweet.video.isSome:
-      videoFuts.add getVideo(tweet, token)
+  for tweet in tweets.filterIt(it.video.isSome):
+    videoFuts.add getVideo(tweet, token)
 
   await all(videoFuts)
 
@@ -132,12 +107,43 @@ proc getConversationVideos*(convo: Conversation) {.async.} =
   var futs: seq[Future[void]]
 
   futs.add getVideo(convo.tweet, token)
-  futs.add getVideos(convo.before, token=token)
-  futs.add getVideos(convo.after, token=token)
-  futs.add convo.replies.mapIt(getVideos(it, token=token))
+  futs.add getVideos(convo.before)
+  futs.add getVideos(convo.after)
+  futs.add convo.replies.mapIt(getVideos(it))
 
   await all(futs)
 
+proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
+  let
+    url = base / profileIntentUrl ? {"screen_name": username}
+    html = await fetchHtml(url, headers)
+
+  result = parseIntentProfile(html)
+
+proc getProfile*(username: string): Future[Profile] {.async.} =
+  let headers = newHttpHeaders({
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9",
+    "Referer": $(base / username),
+    "User-Agent": agent,
+    "X-Twitter-Active-User": "yes",
+    "X-Requested-With": "XMLHttpRequest",
+    "Accept-Language": "en-US,en;q=0.9"
+  })
+
+  let
+    params = {
+      "screen_name": username,
+      "wants_hovercard": "true",
+      "_": $(epochTime().int)
+    }
+    url = base / profilePopupUrl ? params
+    html = await fetchHtml(url, headers, jsonKey="html")
+
+  if not html.querySelector(".ProfileCard-sensitiveWarningContainer").isNil:
+    return await getProfileFallback(username, headers)
+
+  result = parsePopupProfile(html)
+
 proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
   let headers = newHttpHeaders({
     "Accept": "application/json, text/javascript, */*; q=0.01",

src/parserutils.nim

@@ -112,5 +112,5 @@ proc getTweetMedia*(tweet: Tweet; node: XmlNode) =
 
   if "gif" in player.getAttr("class"):
     tweet.gif = some(getGif(player.querySelector(".PlayableMedia-player")))
-  else:
+  elif "video" in player.getAttr("class"):
     tweet.video = some(Video())