Better video/gif support

public/style.css

@@ -593,3 +593,8 @@ nav {
     top: calc(50% - 20px);
     font-size: 20px;
 }
+
+video {
+    height: 100%;
+    width: 100%;
+}

src/api.nim

@@ -5,12 +5,18 @@ import nimquery, regex
 import ./types, ./parser
 
 const
-  base = parseUri("https://twitter.com/")
   agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36"
+  auth = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+
+  base = parseUri("https://twitter.com/")
+  apiBase = parseUri("https://api.twitter.com/1.1/")
+
   timelineUrl = "i/profiles/show/$1/timeline/tweets?include_available_features=1&include_entities=1&include_new_items_bar=true"
   profilePopupUrl = "i/profiles/popup"
   profileIntentUrl = "intent/user"
   tweetUrl = "i/status/"
+  videoUrl = "videos/tweet/config/$1.json"
+  tokenUrl = "guest/activate.json"
 
 proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.async.} =
   var client = newAsyncHttpClient()
@@ -30,6 +36,20 @@ proc fetchHtml(url: Uri; headers: HttpHeaders; jsonKey = ""): Future[XmlNode] {.
   else:
     return parseHtml(resp)
 
+proc fetchJson(url: Uri; headers: HttpHeaders): Future[JsonNode] {.async.} =
+  var client = newAsyncHttpClient()
+  defer: client.close()
+
+  client.headers = headers
+
+  var resp = ""
+  try:
+    resp = await client.getContent($url)
+  except:
+    return nil
+
+  return parseJson(resp)
+
 proc getProfileFallback(username: string; headers: HttpHeaders): Future[Profile] {.async.} =
   let
     url = base / profileIntentUrl ? {"screen_name": username}
@@ -61,6 +81,63 @@ proc getProfile*(username: string): Future[Profile] {.async.} =
 
   result = parsePopupProfile(html)
 
+proc getGuestToken(): Future[string] {.async.} =
+  let headers = newHttpHeaders({
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Referer": $base,
+    "User-Agent": agent,
+    "Authorization": auth
+  })
+
+  let client = newAsyncHttpClient()
+  client.headers = headers
+
+  let
+    url = apibase / tokenUrl
+    json = parseJson(await client.postContent($url))
+
+  result = json["guest_token"].to(string)
+
+proc getVideo*(tweet: Tweet; token: string) {.async.} =
+  let headers = newHttpHeaders({
+    "Accept": "application/json, text/javascript, */*; q=0.01",
+    "Referer": tweet.link,
+    "User-Agent": agent,
+    "Authorization": auth,
+    "x-guest-token": token
+  })
+
+  let
+    url = apiBase / (videoUrl % tweet.id)
+    json = await fetchJson(url, headers)
+
+
+  tweet.video = some(parseVideo(json))
+
+proc getVideos*(tweets: Tweets; token="") {.async.} =
+  if not tweets.anyIt(it.video.isSome): return
+
+  var
+    token = if token.len > 0: token else: await getGuestToken()
+    videoFuts: seq[Future[void]]
+
+  for tweet in tweets:
+    if tweet.video.isSome:
+      videoFuts.add getVideo(tweet, token)
+
+  await all(videoFuts)
+
+proc getConversationVideos*(convo: Conversation) {.async.} =
+  var token = await getGuestToken()
+  var futs: seq[Future[void]]
+
+  futs.add getVideo(convo.tweet, token)
+  futs.add getVideos(convo.before, token=token)
+  futs.add getVideos(convo.after, token=token)
+  futs.add convo.replies.mapIt(getVideos(it, token=token))
+
+  await all(futs)
+
 proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
   let headers = newHttpHeaders({
     "Accept": "application/json, text/javascript, */*; q=0.01",
@@ -78,6 +155,7 @@ proc getTimeline*(username: string; after=""): Future[Tweets] {.async.} =
   let html = await fetchHtml(base / url, headers, jsonKey="items_html")
 
   result = parseTweets(html)
+  await getVideos(result)
 
 proc getTweet*(id: string): Future[Conversation] {.async.} =
   let headers = newHttpHeaders({
@@ -96,3 +174,4 @@ proc getTweet*(id: string): Future[Conversation] {.async.} =
     html = await fetchHtml(url, headers)
 
   result = parseConversation(html)
+  await getConversationVideos(result)

src/formatters.nim

@@ -65,12 +65,6 @@ proc getUserpic*(userpic: string; style=""): string =
 proc getUserpic*(profile: Profile; style=""): string =
   getUserPic(profile.userpic, style)
 
-proc getGifSrc*(tweet: Tweet): string =
-  fmt"https://video.twimg.com/tweet_video/{tweet.gif.get()}.mp4"
-
-proc getGifThumb*(tweet: Tweet): string =
-  fmt"https://pbs.twimg.com/tweet_video_thumb/{tweet.gif.get()}.jpg"
-
 proc formatName*(profile: Profile): string =
   result = xmltree.escape(profile.fullname)
   if profile.verified:

src/parser.nim

@@ -1,4 +1,4 @@
-import xmltree, sequtils, strtabs, strutils, strformat
+import xmltree, sequtils, strtabs, strutils, strformat, json
 import nimquery
 
 import ./types, ./parserutils
@@ -40,7 +40,6 @@ proc parseTweetProfile*(profile: XmlNode): Profile =
 
 proc parseQuote*(tweet: XmlNode): Tweet =
   let tweet = tweet.querySelector(".QuoteTweet-innerContainer")
-
   result = Tweet(
     id:   tweet.getAttr("data-item-id"),
     link: tweet.getAttr("href"),
@@ -77,8 +76,10 @@ proc parseTweets*(node: XmlNode): Tweets =
   node.querySelectorAll(".tweet").map(parseTweet)
 
 proc parseConversation*(node: XmlNode): Conversation =
-  result.tweet = parseTweet(node.querySelector(".permalink-tweet-container > .tweet"))
-  result.before = parseTweets(node.querySelector(".in-reply-to"))
+  result = Conversation(
+    tweet: parseTweet(node.querySelector(".permalink-tweet-container > .tweet")),
+    before: parseTweets(node.querySelector(".in-reply-to"))
+  )
 
   let replies = node.querySelector(".replies-to")
   if replies.isNil: return
@@ -89,3 +90,14 @@ proc parseConversation*(node: XmlNode): Conversation =
     let thread = parseTweets(reply)
     if not thread.anyIt(it in result.after):
       result.replies.add thread
+
+proc parseVideo*(node: JsonNode): Video =
+  let track = node{"track"}
+  result = Video(
+    thumb: node["posterImage"].to(string),
+    id: track["contentId"].to(string),
+    length: track["durationMs"].to(int),
+    views: track["viewCount"].to(string),
+    url: track["playbackUrl"].to(string),
+    available: track{"mediaAvailability"}["status"].to(string) == "available"
+  )

src/parserutils.nim

@@ -1,7 +1,7 @@
-import xmltree, strtabs, times
+import xmltree, strtabs, strformat, times
 import nimquery, regex
 
-import ./types, ./formatters
+import ./types, ./formatters, ./api
 
 const
   thumbRegex = re".+:url\('([^']+)'\)"
@@ -84,11 +84,10 @@ proc getIntentStats*(profile: var Profile; node: XmlNode) =
     of "followers": profile.followers = text
     of "following": profile.following = text
 
-proc getTweetStats*(tweet: var Tweet; node: XmlNode) =
+proc getTweetStats*(tweet: Tweet; node: XmlNode) =
   tweet.replies = "0"
   tweet.retweets = "0"
   tweet.likes = "0"
-
   for action in node.querySelectorAll(".ProfileTweet-actionCountForAria"):
     let text = action.innerText.split()
     case text[1]
@@ -96,16 +95,22 @@ proc getTweetStats*(tweet: var Tweet; node: XmlNode) =
     of "likes":    tweet.likes = text[0]
     of "retweets": tweet.retweets = text[0]
 
-proc getTweetMedia*(tweet: var Tweet; node: XmlNode) =
+proc getGif(player: XmlNode): Gif =
+  let
+    thumb = player.getAttr("style").replace(thumbRegex, "$1")
+    id = thumb.replace(gifRegex, "$1")
+    url = fmt"https://video.twimg.com/tweet_video/{id}.mp4"
+  Gif(url: url, thumb: thumb)
+
+proc getTweetMedia*(tweet: Tweet; node: XmlNode) =
   for photo in node.querySelectorAll(".AdaptiveMedia-photoContainer"):
     tweet.photos.add photo.attrs["data-image-url"]
 
-  let player = node.selectAttr(".PlayableMedia-player", "style")
-  if player.len == 0:
+  let player = node.querySelector(".PlayableMedia")
+  if player.isNil:
     return
 
-  let thumb = player.replace(thumbRegex, "$1")
-  if "tweet_video" in thumb:
-    tweet.gif = some(thumb.replace(gifRegex, "$1"))
+  if "gif" in player.getAttr("class"):
+    tweet.gif = some(getGif(player.querySelector(".PlayableMedia-player")))
   else:
-    tweet.videoThumb = some(thumb)
+    tweet.video = some(Video())

src/types.nim

@@ -31,7 +31,26 @@ db("cache.db", "", "", ""):
         .}: Time
 
 type
-  Tweet* = object
+  Video* = object
+    id*: string
+    url*: string
+    thumb*: string
+    length*: int
+    views*: string
+    available*: bool
+
+  Gif* = object
+    url*: string
+    thumb*: string
+
+  Quote* = ref object
+    id*: string
+    profile*: Profile
+    link*: string
+    text*: string
+    video*: Option[Video]
+
+  Tweet* = ref object
     id*: string
     profile*: Profile
     link*: string
@@ -42,16 +61,16 @@ type
     retweets*: string
     likes*: string
     pinned*: bool
-    photos*: seq[string]
+    quote*: Option[Quote]
     retweetBy*: Option[string]
-    gif*: Option[string]
-    video*: Option[string]
-    videoThumb*: Option[string]
     retweetId*: Option[string]
+    gif*: Option[Gif]
+    video*: Option[Video]
+    photos*: seq[string]
 
   Tweets* = seq[Tweet]
 
-  Conversation* = object
+  Conversation* = ref object
     tweet*: Tweet
     before*: Tweets
     after*: Tweets

src/views/tweet.nim

@@ -54,11 +54,11 @@
 </div>
 #end proc
 #
-#proc renderVideo(tweet: Tweet): string =
+#proc renderVideo(video: Video): string =
 <div class="attachments media-body">
   <div class="gallery-row" style="max-height: unset;">
     <div class="attachment image">
-    <video poster=${tweet.videoThumb.get()} style="width: 100%; height: 100%;" autoplay muted loop></video>
+    <video poster=${video.thumb.getSigUrl("pic")} autoplay muted loop></video>
     <div class="video-overlay">
       <p>Video playback not supported</p>
     </div>
@@ -67,14 +67,12 @@
 </div>
 #end proc
 #
-#proc renderGif(tweet: Tweet): string =
-#let thumbUrl = getGifThumb(tweet).getSigUrl("pic")
-#let videoUrl = getGifSrc(tweet).getSigUrl("video")
-<div class="attachments media-body">
+#proc renderGif(gif: Gif): string =
+<div class="attachments media-body" style="display: table-cell;">
   <div class="gallery-row" style="max-height: unset;">
     <div class="attachment image">
-      <video poster=${thumbUrl} style="width: 100%; height: 100%;" autoplay muted loop>
-        <source src=${videoUrl} type="video/mp4">
+      <video class="gif" poster=${gif.thumb.getSigUrl("pic")} autoplay muted loop>
+        <source src=${gif.url.getSigUrl("video")} type="video/mp4">
       </video>
     </div>
   </div>
@@ -103,10 +101,10 @@
     </div>
     #if tweet.photos.len > 0:
     ${renderMediaGroup(tweet)}
-    #elif tweet.videoThumb.isSome:
-    ${renderVideo(tweet)}
+    #elif tweet.video.isSome:
+    ${renderVideo(tweet.video.get())}
     #elif tweet.gif.isSome:
-    ${renderGif(tweet)}
+    ${renderGif(tweet.gif.get())}
     #end if
     ${renderStats(tweet)}
   </div>