Merge remote-tracking branch 'upstream/master'

src/api/list.nim

@@ -23,8 +23,8 @@ proc getListTimeline*(username, list, agent, after: string): Future[Timeline] {.
 
   let last = result.content[^1]
   result.minId =
-    if last.retweet.isNone: last.id
-    else: get(last.retweet).id
+    if last.retweet.isNone: $last.id
+    else: $(get(last.retweet).id)
 
 proc getListMembers*(username, list, agent: string): Future[Result[Profile]] {.async.} =
   let

src/api/media.nim

@@ -68,7 +68,7 @@ proc getVideoFetch(tweet: Tweet; agent, token: string) {.async.} =
   let
     headers = genHeaders({"authorization": auth, "x-guest-token": token},
                          agent, base / getLink(tweet), lang=false)
-    url = apiBase / (videoUrl % tweet.id)
+    url = apiBase / (videoUrl % $tweet.id)
     json = await fetchJson(url, headers)
 
   if json == nil:
@@ -106,7 +106,7 @@ proc getPoll*(tweet: Tweet; agent: string) {.async.} =
 
   let
     headers = genHeaders(agent, base / getLink(tweet), auth=true)
-    url = base / (pollUrl % tweet.id)
+    url = base / (pollUrl % $tweet.id)
     html = await fetchHtml(url, headers)
 
   if html == nil: return

src/formatters.nim

@@ -1,4 +1,5 @@
-import strutils, strformat, sequtils, htmlgen, xmltree, times, uri, tables
+import strutils, strformat, sequtils, times, uri, tables
+import xmltree, htmlparser, htmlgen
 import regex
 
 import types, utils, query
@@ -15,6 +16,10 @@ const hostname {.strdefine.} = "nitter.net"
 proc stripText*(text: string): string =
   text.replace(nbsp, " ").strip()
 
+proc stripHtml*(text: string): string =
+  let html = parseHtml(text)
+  html.innerText()
+
 proc shortLink*(text: string; length=28): string =
   result = text.replace(re"https?://(www.)?", "")
   if result.len > length:
@@ -27,7 +32,7 @@ proc replaceUrl*(url: string; prefs: Prefs; rss=false): string =
   if prefs.replaceTwitter.len > 0:
     result = result.replace(twRegex, prefs.replaceTwitter)
   if rss:
-    result = result.replace("href=\"/", "href=\"" & hostname & "/")
+    result = result.replace("href=\"/", "href=\"https://" & hostname & "/")
 
 proc proxifyVideo*(manifest: string; proxy: bool): string =
   proc cb(m: RegexMatch; s: string): string =
@@ -42,7 +47,7 @@ proc getUserpic*(userpic: string; style=""): string =
 proc getUserpic*(profile: Profile; style=""): string =
   getUserPic(profile.userpic, style)
 
-proc getVideoEmbed*(id: string): string =
+proc getVideoEmbed*(id: int): string =
   &"https://twitter.com/i/videos/{id}?embed_source=facebook"
 
 proc pageTitle*(profile: Profile): string =
@@ -67,7 +72,7 @@ proc getTweetTime*(tweet: Tweet): string =
   tweet.time.format("h:mm tt' · 'MMM d', 'YYYY")
 
 proc getLink*(tweet: Tweet | Quote): string =
-  if tweet.id.len == 0: return
+  if tweet.id == 0: return
   &"/{tweet.profile.username}/status/{tweet.id}"
 
 proc getTombstone*(text: string): string =

src/parser.nim

@@ -72,7 +72,7 @@ proc parseTweetProfile*(profile: XmlNode): Profile =
 
 proc parseQuote*(quote: XmlNode): Quote =
   result = Quote(
-    id:    quote.attr("data-item-id"),
+    id:    parseInt(quote.attr("data-item-id")),
     text:  getQuoteText(quote),
     reply: parseTweetReply(quote),
     hasThread: quote.select(".self-thread-context") != nil,
@@ -99,8 +99,8 @@ proc parseTweet*(node: XmlNode): Tweet =
     return Tweet()
 
   result = Tweet(
-    id:        tweet.attr("data-item-id"),
-    threadId:  tweet.attr("data-conversation-id"),
+    id:        parseInt(tweet.attr("data-item-id")),
+    threadId:  parseInt(tweet.attr("data-conversation-id")),
     text:      getTweetText(tweet),
     time:      getTimestamp(tweet),
     shortTime: getShortTime(tweet),
@@ -119,7 +119,7 @@ proc parseTweet*(node: XmlNode): Tweet =
   if by.len > 0:
     result.retweet = some Retweet(
       by: stripText(by),
-      id: tweet.attr("data-retweet-id")
+      id: parseInt(tweet.attr("data-retweet-id"))
     )
 
   let quote = tweet.select(".QuoteTweet-innerContainer")
@@ -191,7 +191,7 @@ proc parseTimeline*(node: XmlNode; after: string): Timeline =
     beginning: after.len == 0
   )
 
-proc parseVideo*(node: JsonNode; tweetId: string): Video =
+proc parseVideo*(node: JsonNode; tweetId: int): Video =
   let
     track = node{"track"}
     cType = track["contentType"].to(string)
@@ -216,7 +216,7 @@ proc parseVideo*(node: JsonNode; tweetId: string): Video =
   else:
     echo "Can't parse video of type ", cType
 
-  result.videoId = tweetId
+  result.videoId = $tweetId
   result.thumb = node["posterImage"].to(string)
 
 proc parsePoll*(node: XmlNode): Poll =

src/parserutils.nim

@@ -57,7 +57,8 @@ proc parseText*(text: XmlNode; skipLink=""): string =
       if "data-expanded-url" in el.attrs:
         let url = el.attr("data-expanded-url")
         if url == skipLink: continue
-        elif "u-hidden" in class: result.add "\n"
+        if "u-hidden" in class and result.len > 0:
+          result.add "\n"
         result.add a(shortLink(url), href=url)
       elif "ashtag" in class:
         let hash = el.innerText()
@@ -240,7 +241,7 @@ proc getTweetCard*(tweet: Tweet; node: XmlNode) =
   if cardDiv == nil: return
 
   var card = Card(
-    id: tweet.id,
+    id: $tweet.id,
     query: cardDiv.attr("data-src")
   )
 

src/routes/status.nim

@@ -18,7 +18,7 @@ proc createStatusRouter*(cfg: Config) =
       let prefs = cookiePrefs()
 
       let conversation = await getTweet(@"name", @"id", @"max_position", getAgent())
-      if conversation == nil or conversation.tweet.id.len == 0:
+      if conversation == nil or conversation.tweet.id == 0:
         var error = "Tweet not found"
         if conversation != nil and conversation.tweet.tombstone.len > 0:
           error = conversation.tweet.tombstone

src/types.nim

@@ -115,7 +115,7 @@ type
     video*: Option[Video]
 
   Quote* = object
-    id*: string
+    id*: int
     profile*: Profile
     text*: string
     reply*: seq[string]
@@ -128,7 +128,7 @@ type
 
   Retweet* = object
     by*: string
-    id*: string
+    id*: int
 
   TweetStats* = object
     replies*: string
@@ -136,8 +136,8 @@ type
     likes*: string
 
   Tweet* = ref object
-    id*: string
-    threadId*: string
+    id*: int
+    threadId*: int
     profile*: Profile
     text*: string
     time*: Time

src/views/general.nim

@@ -57,7 +57,7 @@ proc renderMain*(body: VNode; req: Request; title="Nitter"; titleText=""; desc="
       meta(name="viewport", content="width=device-width, initial-scale=1.0")
       meta(property="og:type", content=`type`)
       meta(property="og:title", content=titleText)
-      meta(property="og:description", content=desc)
+      meta(property="og:description", content=stripHtml(desc))
       meta(property="og:site_name", content="Nitter")
 
       for url in images:

src/views/timeline.nim

@@ -34,13 +34,13 @@ proc renderNoneFound(): VNode =
 
 proc renderThread(thread: seq[Tweet]; prefs: Prefs; path: string): VNode =
   buildHtml(tdiv(class="thread-line")):
-    for i, threadTweet in thread.sortedByIt(it.time):
+    for i, threadTweet in thread.sortedByIt(it.id):
       let show = i == thread.len and thread[0].id != threadTweet.threadId
       renderTweet(threadTweet, prefs, path, class="thread",
                   index=i, total=thread.high, showThread=show)
 
-proc threadFilter(it: Tweet; tweetThread: string): bool =
-  it.retweet.isNone and it.reply.len == 0 and it.threadId == tweetThread
+proc threadFilter(it: Tweet; thread: int): bool =
+  it.retweet.isNone and it.reply.len == 0 and it.threadId == thread
 
 proc renderUser(user: Profile; prefs: Prefs): VNode =
   buildHtml(tdiv(class="timeline-item")):
@@ -81,8 +81,8 @@ proc renderTimelineTweets*(results: Result[Tweet]; prefs: Prefs; path: string):
     if results.content.len == 0:
       renderNoneFound()
     else:
-      var threads: seq[string]
-      var retweets: seq[string]
+      var threads: seq[int]
+      var retweets: seq[int]
       for tweet in results.content:
         if tweet.threadId in threads or tweet.id in retweets: continue
         let thread = results.content.filterIt(threadFilter(it, tweet.threadId))