Merge branch 'master' into feature/mp4-streaming

.github/workflows/build-docker.yml

@@ -1,4 +1,4 @@
-name: CI/CD
+name: Docker
 
 on:
   push:
@@ -8,31 +8,55 @@ on:
       - master
 
 jobs:
-  build-docker:
-    runs-on: ubuntu-latest
+  tests:
+    uses: ./.github/workflows/run-tests.yml
+  build-docker-amd64:
+    needs: [tests]
+    runs-on: buildjet-2vcpu-ubuntu-2204
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-        with:
-          platforms: all
       - name: Set up Docker Buildx
         id: buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
         with:
           version: latest
       - name: Login to DockerHub
-        uses: docker/login-action@v1
+        uses: docker/login-action@v2
         with:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
-      - name: Build and push
-        uses: docker/build-push-action@v2
+      - name: Build and push AMD64 Docker image
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./Dockerfile
           platforms: linux/amd64
           push: true
           tags: zedeus/nitter:latest,zedeus/nitter:${{ github.sha }}
+  build-docker-arm64:
+    needs: [tests]
+    runs-on: buildjet-2vcpu-ubuntu-2204-arm
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up Docker Buildx
+        id: buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          version: latest
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push ARM64 Docker image
+        uses: docker/build-push-action@v3
+        with:
+          context: .
+          file: ./Dockerfile.arm64
+          platforms: linux/arm64
+          push: true
+          tags: zedeus/nitter:latest-arm64,zedeus/nitter:${{ github.sha }}-arm64

.github/workflows/run-tests.yml

@@ -0,0 +1,45 @@
+name: Tests
+
+on:
+  push:
+    paths-ignore:
+      - "*.md"
+    branches-ignore:
+      - master
+  workflow_call:
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Cache nimble
+        id: cache-nimble
+        uses: actions/cache@v3
+        with:
+          path: ~/.nimble
+          key: nimble-${{ hashFiles('*.nimble') }}
+          restore-keys: "nimble-"
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.10"
+          cache: "pip"
+      - uses: jiro4989/setup-nim-action@v1
+        with:
+          nim-version: "1.x"
+      - run: nimble build -d:release -Y
+      - run: pip install seleniumbase
+      - run: seleniumbase install chromedriver
+      - uses: supercharge/redis-github-action@1.5.0
+      - name: Prepare Nitter
+        run: |
+          sudo apt install libsass-dev -y
+          cp nitter.example.conf nitter.conf
+          nimble md
+          nimble scss
+      - name: Run tests
+        run: |
+          ./nitter &
+          pytest -n4 tests

Dockerfile

@@ -1,4 +1,4 @@
-FROM nimlang/nim:1.6.2-alpine-regular as nim
+FROM nimlang/nim:1.6.10-alpine-regular as nim
 LABEL maintainer="setenforce@protonmail.com"
 
 RUN apk --no-cache add libsass-dev pcre
@@ -20,4 +20,6 @@ COPY --from=nim /src/nitter/nitter ./
 COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
 COPY --from=nim /src/nitter/public ./public
 EXPOSE 8080
+RUN adduser -h /src/ -D -s /bin/sh nitter
+USER nitter
 CMD ./nitter

Dockerfile.arm64

@@ -0,0 +1,23 @@
+FROM alpine:3.17 as nim
+LABEL maintainer="setenforce@protonmail.com"
+
+RUN apk --no-cache add gcc git libc-dev libsass-dev "nim=1.6.8-r0" nimble pcre
+
+WORKDIR /src/nitter
+
+COPY nitter.nimble .
+RUN nimble install -y --depsOnly
+
+COPY . .
+RUN nimble build -d:danger -d:lto -d:strip \
+    && nimble scss \
+    && nimble md
+
+FROM alpine:3.17
+WORKDIR /src/
+RUN apk --no-cache add ca-certificates pcre openssl1.1-compat
+COPY --from=nim /src/nitter/nitter ./
+COPY --from=nim /src/nitter/nitter.example.conf ./nitter.conf
+COPY --from=nim /src/nitter/public ./public
+EXPOSE 8080
+CMD ./nitter

README.md

@@ -1,6 +1,7 @@
 # Nitter
 
-[![Test Matrix](https://github.com/zedeus/nitter/workflows/CI/CD/badge.svg)](https://github.com/zedeus/nitter/actions?query=workflow%3ACI/CD)
+[![Test Matrix](https://github.com/zedeus/nitter/workflows/Tests/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/run-tests.yml)
+[![Test Matrix](https://github.com/zedeus/nitter/workflows/Docker/badge.svg)](https://github.com/zedeus/nitter/actions/workflows/build-docker.yml)
 [![License](https://img.shields.io/github/license/zedeus/nitter?style=flat)](#license)
 
 A free and open source alternative Twitter front-end focused on privacy and
@@ -34,7 +35,7 @@ XMR: 42hKayRoEAw4D6G6t8mQHPJHQcXqofjFuVfavqKeNMNUZfeJLJAcNU19i1bGdDvcdN6romiSscW
 
 ## Resources
 
-The wiki contains 
+The wiki contains
 [a list of instances](https://github.com/zedeus/nitter/wiki/Instances) and
 [browser extensions](https://github.com/zedeus/nitter/wiki/Extensions)
 maintained by the community.
@@ -67,9 +68,10 @@ Twitter account.
 ## Installation
 
 ### Dependencies
-* libpcre
-* libsass
-* redis
+
+- libpcre
+- libsass
+- redis
 
 To compile Nitter you need a Nim installation, see
 [nim-lang.org](https://nim-lang.org/install.html) for details. It is possible to
@@ -108,25 +110,32 @@ performance reasons.
 
 ### Docker
 
-#### NOTE: For ARM64/ARM support, please use [unixfox's image](https://quay.io/repository/unixfox/nitter?tab=tags), more info [here](https://github.com/zedeus/nitter/issues/399#issuecomment-997263495)
+Page for the Docker image: https://hub.docker.com/r/zedeus/nitter
+
+#### NOTE: For ARM64 support, please use the separate ARM64 docker image: [`zedeus/nitter:latest-arm64`](https://hub.docker.com/r/zedeus/nitter/tags).
 
 To run Nitter with Docker, you'll need to install and run Redis separately
 before you can run the container. See below for how to also run Redis using
 Docker.
 
 To build and run Nitter in Docker:
+
 ```bash
 docker build -t nitter:latest .
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host nitter:latest
 ```
 
+Note: For ARM64, use this Dockerfile: [`Dockerfile.arm64`](https://github.com/zedeus/nitter/blob/master/Dockerfile.arm64).
+
 A prebuilt Docker image is provided as well:
+
 ```bash
 docker run -v $(pwd)/nitter.conf:/src/nitter.conf -d --network host zedeus/nitter:latest
 ```
 
 Using docker-compose to run both Nitter and Redis as different containers:
 Change `redisHost` from `localhost` to `nitter-redis` in `nitter.conf`, then run:
+
 ```bash
 docker-compose up -d
 ```

config.nims

@@ -1,5 +1,6 @@
 --define:ssl
 --define:useStdLib
+--threads:off
 
 # workaround httpbeast file upload bug
 --assertions:off

docker-compose.yml

@@ -8,7 +8,7 @@ services:
     ports:
       - "127.0.0.1:8080:8080" # Replace with "8080:8080" if you don't use a reverse proxy
     volumes:
-      - ./nitter.conf:/src/nitter.conf:ro
+      - ./nitter.conf:/src/nitter.conf:Z,ro
     depends_on:
       - nitter-redis
     restart: unless-stopped
@@ -17,6 +17,12 @@ services:
       interval: 30s
       timeout: 5s
       retries: 2
+    user: "998:998"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
 
   nitter-redis:
     image: redis:6-alpine
@@ -30,6 +36,12 @@ services:
       interval: 30s
       timeout: 5s
       retries: 2
+    user: "999:1000"
+    read_only: true
+    security_opt:
+      - no-new-privileges:true
+    cap_drop:
+      - ALL
 
 volumes:
   nitter-redis:

nitter.example.conf

@@ -1,11 +1,11 @@
 [Server]
+hostname = "nitter.net"  # for generating links, change this to your own domain/ip
+title = "nitter"
 address = "0.0.0.0"
 port = 8080
 https = false  # disable to enable cookies when not using https
 httpMaxConnections = 100
 staticDir = "./public"
-title = "nitter"
-hostname = "nitter.net"
 
 [Cache]
 listMinutes = 240  # how long to cache list info (not the tweets, so keep it high)
@@ -13,9 +13,9 @@ rssMinutes = 10  # how long to cache rss queries
 redisHost = "localhost"  # Change to "nitter-redis" if using docker-compose
 redisPort = 6379
 redisPassword = ""
-redisConnections = 20  # connection pool size
+redisConnections = 20  # minimum open connections in pool
 redisMaxConnections = 30
-# max, new connections are opened when none are available, but if the pool size
+# new connections are opened when none are available, but if the pool size
 # goes above this, they're closed when released. don't worry about this unless
 # you receive tons of requests per second
 
@@ -23,23 +23,22 @@ redisMaxConnections = 30
 hmacKey = "secretkey"  # random key for cryptographic signing of video urls
 base64Media = false  # use base64 encoding for proxied media urls
 enableRSS = true  # set this to false to disable RSS feeds
-enableDebug = false  # enable request logs and debug endpoints
+enableDebug = false  # enable request logs and debug endpoints (/.tokens)
 proxy = ""  # http/https url, SOCKS proxies are not supported
 proxyAuth = ""
 tokenCount = 10
 # minimum amount of usable tokens. tokens are used to authorize API requests,
-# but they expire after ~1 hour, and have a limit of 187 requests.
-# the limit gets reset every 15 minutes, and the pool is filled up so there's
-# always at least $tokenCount usable tokens. again, only increase this if
-# you receive major bursts all the time
+# but they expire after ~1 hour, and have a limit of 500 requests per endpoint.
+# the limits reset every 15 minutes, and the pool is filled up so there's
+# always at least `tokenCount` usable tokens. only increase this if you receive
+# major bursts all the time and don't have a rate limiting setup via e.g. nginx
 
 # Change default preferences here, see src/prefs_impl.nim for a complete list
 [Preferences]
 theme = "Nitter"
 replaceTwitter = "nitter.net"
-replaceYouTube = "piped.kavin.rocks"
+replaceYouTube = "piped.video"
 replaceReddit = "teddit.net"
-replaceInstagram = ""
 proxyVideos = true
 hlsPlayback = false
 infiniteScroll = false

nitter.nimble

@@ -11,18 +11,18 @@ bin           = @["nitter"]
 # Dependencies
 
 requires "nim >= 1.4.8"
-requires "jester >= 0.5.0"
-requires "karax#5498909"
-requires "sass#e683aa1"
-requires "nimcrypto#a5742a9"
-requires "markdown#a661c26"
-requires "packedjson#d11d167"
-requires "supersnappy#2.1.1"
+requires "jester#baca3f"
+requires "karax#5cf360c"
+requires "sass#7dfdd03"
+requires "nimcrypto#4014ef9"
+requires "markdown#158efe3"
+requires "packedjson#9e6fbb6"
+requires "supersnappy#6c94198"
 requires "redpool#8b7c1db"
 requires "https://github.com/zedeus/redis#d0a0e6f"
-requires "zippy#0.7.3"
-requires "flatty#0.2.3"
-requires "jsony#d0e69bd"
+requires "zippy#ca5989a"
+requires "flatty#e668085"
+requires "jsony#ea811be"
 
 
 # Tasks

public/robots.txt

@@ -1,4 +1,5 @@
 User-agent: *
 Disallow: /
+Crawl-delay: 1
 User-agent: Twitterbot
 Disallow:

src/api.nim

@@ -4,123 +4,143 @@ import packedjson
 import types, query, formatters, consts, apiutils, parser
 import experimental/parser as newParser
 
-proc getGraphUser*(id: string): Future[User] {.async.} =
+proc getGraphUser*(username: string): Future[User] {.async.} =
+  if username.len == 0: return
+  let
+    variables = %*{"screen_name": username}
+    params = {"variables": $variables, "features": gqlFeatures}
+    js = await fetchRaw(graphUser ? params, Api.userScreenName)
+  result = parseGraphUser(js)
+
+proc getGraphUserById*(id: string): Future[User] {.async.} =
   if id.len == 0 or id.any(c => not c.isDigit): return
   let
-    variables = %*{"userId": id, "withSuperFollowsUserFields": true}
-    js = await fetchRaw(graphUser ? {"variables": $variables}, Api.userRestId)
+    variables = %*{"userId": id}
+    params = {"variables": $variables, "features": gqlFeatures}
+    js = await fetchRaw(graphUserById ? params, Api.userRestId)
   result = parseGraphUser(js)
 
+proc getGraphUserTweets*(id: string; kind: TimelineKind; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = userTweetsVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    (url, apiId) = case kind
+                   of TimelineKind.tweets: (graphUserTweets, Api.userTweets)
+                   of TimelineKind.replies: (graphUserTweetsAndReplies, Api.userTweetsAndReplies)
+                   of TimelineKind.media: (graphUserMedia, Api.userMedia)
+    js = await fetch(url ? params, apiId)
+  result = parseGraphTimeline(js, "user", after)
+
+proc getGraphListTweets*(id: string; after=""): Future[Timeline] {.async.} =
+  if id.len == 0: return
+  let
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = listTweetsVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphListTweets ? params, Api.listTweets)
+  result = parseGraphTimeline(js, "list", after)
+
 proc getGraphListBySlug*(name, list: string): Future[List] {.async.} =
   let
-    variables = %*{"screenName": name, "listSlug": list, "withHighlightedLabel": false}
-    url = graphListBySlug ? {"variables": $variables}
-  result = parseGraphList(await fetch(url, Api.listBySlug))
+    variables = %*{"screenName": name, "listSlug": list}
+    params = {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphList(await fetch(graphListBySlug ? params, Api.listBySlug))
 
 proc getGraphList*(id: string): Future[List] {.async.} =
   let
-    variables = %*{"listId": id, "withHighlightedLabel": false}
-    url = graphList ? {"variables": $variables}
-  result = parseGraphList(await fetch(url, Api.list))
+    variables = %*{"listId": id}
+    params = {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphList(await fetch(graphListById ? params, Api.list))
 
 proc getGraphListMembers*(list: List; after=""): Future[Result[User]] {.async.} =
   if list.id.len == 0: return
   var
     variables = %*{
       "listId": list.id,
-      "withSuperFollowsUserFields": false,
       "withBirdwatchPivots": false,
       "withDownvotePerspective": false,
       "withReactionsMetadata": false,
-      "withReactionsPerspective": false,
-      "withSuperFollowsTweetFields": false
+      "withReactionsPerspective": false
     }
   if after.len > 0:
     variables["cursor"] = % after
-  let url = graphListMembers ? {"variables": $variables}
+  let url = graphListMembers ? {"variables": $variables, "features": gqlFeatures}
   result = parseGraphListMembers(await fetchRaw(url, Api.listMembers), after)
 
-proc getListTimeline*(id: string; after=""): Future[Timeline] {.async.} =
+proc getGraphTweetResult*(id: string): Future[Tweet] {.async.} =
   if id.len == 0: return
   let
-    ps = genParams({"list_id": id, "ranking_mode": "reverse_chronological"}, after)
-    url = listTimeline ? ps
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+    variables = tweetResultVariables % id
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphTweetResult ? params, Api.tweetResult)
+  result = parseGraphTweetResult(js)
 
-proc getUser*(username: string): Future[User] {.async.} =
-  if username.len == 0: return
+proc getGraphTweet(id: string; after=""): Future[Conversation] {.async.} =
+  if id.len == 0: return
   let
-    ps = genParams({"screen_name": username})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json, username)
+    cursor = if after.len > 0: "\"cursor\":\"$1\"," % after else: ""
+    variables = tweetVariables % [id, cursor]
+    params = {"variables": variables, "features": gqlFeatures}
+    js = await fetch(graphTweet ? params, Api.tweetDetail)
+  result = parseGraphConversation(js, id)
 
-proc getUserById*(userId: string): Future[User] {.async.} =
-  if userId.len == 0: return
-  let
-    ps = genParams({"user_id": userId})
-    json = await fetchRaw(userShow ? ps, Api.userShow)
-  result = parseUser(json)
+proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
+  result = (await getGraphTweet(id, after)).replies
+  result.beginning = after.len == 0
 
-proc getTimeline*(id: string; after=""; replies=false): Future[Timeline] {.async.} =
-  if id.len == 0: return
-  let
-    ps = genParams({"userId": id, "include_tweet_replies": $replies}, after)
-    url = timeline / (id & ".json") ? ps
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
+  result = await getGraphTweet(id)
+  if after.len > 0:
+    result.replies = await getReplies(id, after)
 
-proc getMediaTimeline*(id: string; after=""): Future[Timeline] {.async.} =
-  if id.len == 0: return
-  let url = mediaTimeline / (id & ".json") ? genParams(cursor=after)
-  result = parseTimeline(await fetch(url, Api.timeline), after)
+proc getGraphSearch*(query: Query; after=""): Future[Result[Tweet]] {.async.} =
+  let q = genQueryParam(query)
+  if q.len == 0 or q == emptyQuery:
+    return Result[Tweet](query: query, beginning: true)
+
+  var
+    variables = %*{
+      "rawQuery": q,
+      "count": 20,
+      "product": "Latest",
+      "withDownvotePerspective": false,
+      "withReactionsMetadata": false,
+      "withReactionsPerspective": false
+    }
+  if after.len > 0:
+    variables["cursor"] = % after
+  let url = graphSearchTimeline ? {"variables": $variables, "features": gqlFeatures}
+  result = parseGraphSearch(await fetch(url, Api.search), after)
+  result.query = query
+
+proc getUserSearch*(query: Query; page="1"): Future[Result[User]] {.async.} =
+  if query.text.len == 0:
+    return Result[User](query: query, beginning: true)
+
+  var url = userSearch ? {
+    "q": query.text,
+    "skip_status": "1",
+    "count": "20",
+    "page": page
+  }
+
+  result = parseUsers(await fetchRaw(url, Api.userSearch))
+  result.query = query
+  if page.len == 0:
+    result.bottom = "2"
+  elif page.allCharsInSet(Digits):
+    result.bottom = $(parseInt(page) + 1)
 
 proc getPhotoRail*(name: string): Future[PhotoRail] {.async.} =
   if name.len == 0: return
   let
     ps = genParams({"screen_name": name, "trim_user": "true"},
-                   count="18", ext=false)
+                    count="18", ext=false)
     url = photoRail ? ps
   result = parsePhotoRail(await fetch(url, Api.timeline))
 
-proc getSearch*[T](query: Query; after=""): Future[Result[T]] {.async.} =
-  when T is User:
-    const
-      searchMode = ("result_filter", "user")
-      parse = parseUsers
-      fetchFunc = fetchRaw
-  else:
-    const
-      searchMode = ("tweet_search_mode", "live")
-      parse = parseTimeline
-      fetchFunc = fetch
-
-  let q = genQueryParam(query)
-  if q.len == 0 or q == emptyQuery:
-    return Result[T](beginning: true, query: query)
-
-  let url = search ? genParams(searchParams & @[("q", q), searchMode], after)
-  try:
-    result = parse(await fetchFunc(url, Api.search), after)
-    result.query = query
-  except InternalError:
-    return Result[T](beginning: true, query: query)
-
-proc getTweetImpl(id: string; after=""): Future[Conversation] {.async.} =
-  let url = tweet / (id & ".json") ? genParams(cursor=after)
-  result = parseConversation(await fetch(url, Api.tweet), id)
-
-proc getReplies*(id, after: string): Future[Result[Chain]] {.async.} =
-  result = (await getTweetImpl(id, after)).replies
-  result.beginning = after.len == 0
-
-proc getTweet*(id: string; after=""): Future[Conversation] {.async.} =
-  result = await getTweetImpl(id)
-  if after.len > 0:
-    result.replies = await getReplies(id, after)
-
-proc getStatus*(id: string): Future[Tweet] {.async.} =
-  let url = status / (id & ".json") ? genParams()
-  result = parseStatus(await fetch(url, Api.status))
-
 proc resolve*(url: string; prefs: Prefs): Future[string] {.async.} =
   let client = newAsyncHttpClient(maxRedirects=0)
   try:

src/apiutils.nim

@@ -17,13 +17,13 @@ proc genParams*(pars: openArray[(string, string)] = @[]; cursor="";
     result &= p
   if ext:
     result &= ("ext", "mediaStats")
-    result &= ("include_ext_alt_text", "true")
-    result &= ("include_ext_media_availability", "true")
+    result &= ("include_ext_alt_text", "1")
+    result &= ("include_ext_media_availability", "1")
   if count.len > 0:
     result &= ("count", count)
   if cursor.len > 0:
     # The raw cursor often has plus signs, which sometimes get turned into spaces,
-    # so we need to them back into a plus
+    # so we need to turn them back into a plus
     if " " in cursor:
       result &= ("cursor", cursor.replace(" ", "+"))
     else:
@@ -44,7 +44,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
   })
 
 template updateToken() =
-  if api != Api.search and resp.headers.hasKey(rlRemaining):
+  if resp.headers.hasKey(rlRemaining):
     let
       remaining = parseInt(resp.headers[rlRemaining])
       reset = parseInt(resp.headers[rlReset])
@@ -61,12 +61,15 @@ template fetchImpl(result, fetchBody) {.dirty.} =
   try:
     var resp: AsyncResponse
     pool.use(genHeaders(token)):
-      resp = await c.get($url)
-      result = await resp.body
+      template getContent =
+        resp = await c.get($url)
+        result = await resp.body
+
+      getContent()
 
       if resp.status == $Http503:
         badClient = true
-        raise newException(InternalError, result)
+        raise newException(BadClientError, "Bad client")
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":
@@ -82,6 +85,9 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       raise newException(InternalError, $url)
   except InternalError as e:
     raise e
+  except BadClientError as e:
+    release(token, used=true)
+    raise e
   except Exception as e:
     echo "error: ", e.name, ", msg: ", e.msg, ", token: ", token[], ", url: ", url
     if "length" notin e.msg and "descriptor" notin e.msg:
@@ -100,7 +106,7 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
     updateToken()
 
     let error = result.getError
-    if error in {invalidToken, forbidden, badToken}:
+    if error in {invalidToken, badToken}:
       echo "fetch error: ", result.getError
       release(token, invalid=true)
       raise rateLimitError()
@@ -115,7 +121,7 @@ proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
 
     if result.startsWith("{\"errors"):
       let errors = result.fromJson(Errors)
-      if errors in {invalidToken, forbidden, badToken}:
+      if errors in {invalidToken, badToken}:
         echo "fetch error: ", errors
         release(token, invalid=true)
         raise rateLimitError()

src/consts.nim

@@ -1,28 +1,28 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import uri, sequtils
+import uri, sequtils, strutils
 
 const
-  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAAPYXBAAAAAAACLXUNDekMxqa8h%2F40K4moUkGsoc%3DTYfbDKbT3jJPCEVnMYqilB28NHfOPqkca3qaAxGfsyKCs0wRbw"
+  auth* = "Bearer AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA"
 
   api = parseUri("https://api.twitter.com")
   activate* = $(api / "1.1/guest/activate.json")
 
-  userShow* = api / "1.1/users/show.json"
   photoRail* = api / "1.1/statuses/media_timeline.json"
-  status* = api / "1.1/statuses/show"
-  search* = api / "2/search/adaptive.json"
-
-  timelineApi = api / "2/timeline"
-  timeline* = timelineApi / "profile"
-  mediaTimeline* = timelineApi / "media"
-  listTimeline* = timelineApi / "list.json"
-  tweet* = timelineApi / "conversation"
+  userSearch* = api / "1.1/users/search.json"
 
   graphql = api / "graphql"
-  graphUser* = graphql / "I5nvpI91ljifos1Y3Lltyg/UserByRestId"
-  graphList* = graphql / "JADTh6cjebfgetzvF3tQvQ/List"
-  graphListBySlug* = graphql / "ErWsz9cObLel1BF-HjuBlA/ListBySlug"
-  graphListMembers* = graphql / "Ke6urWMeCV2UlKXGRy4sow/ListMembers"
+  graphUser* = graphql / "pVrmNaXcxPjisIvKtLDMEA/UserByScreenName"
+  graphUserById* = graphql / "1YAM811Q8Ry4XyPpJclURQ/UserByRestId"
+  graphUserTweets* = graphql / "WzJjibAcDa-oCjCcLOotcg/UserTweets"
+  graphUserTweetsAndReplies* = graphql / "fn9oRltM1N4thkh5CVusPg/UserTweetsAndReplies"
+  graphUserMedia* = graphql / "qQoeS7szGavsi8-ehD2AWg/UserMedia"
+  graphTweet* = graphql / "miKSMGb2R1SewIJv2-ablQ/TweetDetail"
+  graphTweetResult* = graphql / "0kc0a_7TTr3dvweZlMslsQ/TweetResultByRestId"
+  graphSearchTimeline* = graphql / "gkjsKepM6gl_HmFWoWKfgg/SearchTimeline"
+  graphListById* = graphql / "iTpgCtbdxrsJfyx0cFjHqg/ListByRestId"
+  graphListBySlug* = graphql / "-kmqNvm5Y-cVrfvBy6docg/ListBySlug"
+  graphListMembers* = graphql / "P4NpVZDqUD_7MEM84L-8nw/ListMembers"
+  graphListTweets* = graphql / "jZntL0oVJSdjhmPcdbw_eA/ListLatestTweetsTimeline"
 
   timelineParams* = {
     "include_profile_interstitial_type": "0",
@@ -33,27 +33,89 @@ const
     "include_mute_edge": "0",
     "include_can_dm": "0",
     "include_can_media_tag": "1",
+    "include_ext_is_blue_verified": "1",
     "skip_status": "1",
     "cards_platform": "Web-12",
     "include_cards": "1",
-    "include_composer_source": "false",
+    "include_composer_source": "0",
     "include_reply_count": "1",
     "tweet_mode": "extended",
-    "include_entities": "true",
-    "include_user_entities": "true",
-    "include_ext_media_color": "false",
-    "send_error_codes": "true",
-    "simple_quoted_tweet": "true",
-    "include_quote_count": "true"
+    "include_entities": "1",
+    "include_user_entities": "1",
+    "include_ext_media_color": "0",
+    "send_error_codes": "1",
+    "simple_quoted_tweet": "1",
+    "include_quote_count": "1"
   }.toSeq
 
-  searchParams* = {
-    "query_source": "typed_query",
-    "pc": "1",
-    "spelling_corrections": "1"
-  }.toSeq
-  ## top: nothing
-  ## latest: "tweet_search_mode: live"
-  ## user:   "result_filter: user"
-  ## photos: "result_filter: photos"
-  ## videos: "result_filter: videos"
+  gqlFeatures* = """{
+  "blue_business_profile_image_shape_enabled": false,
+  "creator_subscriptions_tweet_preview_api_enabled": true,
+  "freedom_of_speech_not_reach_fetch_enabled": false,
+  "graphql_is_translatable_rweb_tweet_is_translatable_enabled": false,
+  "highlights_tweets_tab_ui_enabled": false,
+  "interactive_text_enabled": false,
+  "longform_notetweets_consumption_enabled": true,
+  "longform_notetweets_inline_media_enabled": false,
+  "longform_notetweets_richtext_consumption_enabled": true,
+  "longform_notetweets_rich_text_read_enabled": false,
+  "responsive_web_edit_tweet_api_enabled": false,
+  "responsive_web_enhance_cards_enabled": false,
+  "responsive_web_graphql_exclude_directive_enabled": true,
+  "responsive_web_graphql_skip_user_profile_image_extensions_enabled": false,
+  "responsive_web_graphql_timeline_navigation_enabled": false,
+  "responsive_web_text_conversations_enabled": false,
+  "responsive_web_twitter_blue_verified_badge_is_enabled": true,
+  "rweb_lists_timeline_redesign_enabled": true,
+  "spaces_2022_h2_clipping": true,
+  "spaces_2022_h2_spaces_communities": true,
+  "standardized_nudges_misinfo": false,
+  "tweet_awards_web_tipping_enabled": false,
+  "tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled": false,
+  "tweetypie_unmention_optimization_enabled": false,
+  "verified_phone_label_enabled": false,
+  "vibe_api_enabled": false,
+  "view_counts_everywhere_api_enabled": false
+}""".replace(" ", "").replace("\n", "")
+
+  tweetVariables* = """{
+  "focalTweetId": "$1",
+  $2
+  "withBirdwatchNotes": false,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false
+}"""
+
+  tweetResultVariables* = """{
+  "tweetId": "$1",
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false,
+  "withCommunity": false
+}"""
+
+  userTweetsVariables* = """{
+  "userId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false,
+  "withV2Timeline": true
+}"""
+
+  listTweetsVariables* = """{
+  "listId": "$1", $2
+  "count": 20,
+  "includePromotedContent": false,
+  "withDownvotePerspective": false,
+  "withReactionsMetadata": false,
+  "withReactionsPerspective": false,
+  "withVoice": false
+}"""

src/experimental/parser.nim

@@ -1,2 +1,2 @@
-import parser/[user, graphql, timeline]
-export user, graphql, timeline
+import parser/[user, graphql]
+export user, graphql

src/experimental/parser/graphql.nim

@@ -1,11 +1,17 @@
+import options
 import jsony
 import user, ../types/[graphuser, graphlistmembers]
 from ../../types import User, Result, Query, QueryKind
 
 proc parseGraphUser*(json: string): User =
   let raw = json.fromJson(GraphUser)
+
+  if raw.data.user.result.reason.get("") == "Suspended":
+    return User(suspended: true)
+
   result = toUser raw.data.user.result.legacy
   result.id = raw.data.user.result.restId
+  result.verified = result.verified or raw.data.user.result.isBlueVerified
 
 proc parseGraphListMembers*(json, cursor: string): Result[User] =
   result = Result[User](

src/experimental/parser/timeline.nim

@@ -1,30 +0,0 @@
-import std/[strutils, tables]
-import jsony
-import user, ../types/timeline
-from ../../types import Result, User
-
-proc getId(id: string): string {.inline.} =
-  let start = id.rfind("-")
-  if start < 0: return id
-  id[start + 1 ..< id.len]
-
-proc parseUsers*(json: string; after=""): Result[User] =
-  result = Result[User](beginning: after.len == 0)
-
-  let raw = json.fromJson(Search)
-  if raw.timeline.instructions.len == 0:
-    return
-
-  for i in raw.timeline.instructions:
-    if i.addEntries.entries.len > 0:
-      for e in i.addEntries.entries:
-        let id = e.entryId.getId
-        if e.entryId.startsWith("user"):
-         if id in raw.globalObjects.users:
-           result.content.add toUser raw.globalObjects.users[id]
-        elif e.entryId.startsWith("cursor"):
-         let cursor = e.content.operation.cursor
-         if cursor.cursorType == "Top":
-           result.top = cursor.value
-         elif cursor.cursorType == "Bottom":
-           result.bottom = cursor.value

src/experimental/parser/unifiedcard.nim

@@ -66,6 +66,8 @@ proc parseMedia(component: Component; card: UnifiedCard; result: var Card) =
       durationMs: videoInfo.durationMillis,
       variants: videoInfo.variants
     )
+  of model3d:
+    result.title = "Unsupported 3D model ad"
 
 proc parseUnifiedCard*(json: string): Card =
   let card = json.fromJson(UnifiedCard)
@@ -82,6 +84,10 @@ proc parseUnifiedCard*(json: string): Card =
       component.parseMedia(card, result)
     of buttonGroup:
       discard
+    of ComponentType.hidden:
+      result.kind = CardKind.hidden
+    of ComponentType.unknown:
+      echo "ERROR: Unknown component type: ", json
 
     case component.kind
     of twitterListDetails:

src/experimental/parser/user.nim

@@ -2,7 +2,7 @@ import std/[algorithm, unicode, re, strutils, strformat, options, nre]
 import jsony
 import utils, slices
 import ../types/user as userType
-from ../../types import User, Error
+from ../../types import Result, User, Error
 
 let
   unRegex = re.re"(^|[^A-z0-9-_./?])@([A-z0-9_]{1,15})"
@@ -76,3 +76,12 @@ proc parseUser*(json: string; username=""): User =
     else: echo "[error - parseUser]: ", error
 
   result = toUser json.fromJson(RawUser)
+
+proc parseUsers*(json: string; after=""): Result[User] =
+  result = Result[User](beginning: after.len == 0)
+
+  # starting with '{' means it's an error
+  if json[0] == '[':
+    let raw = json.fromJson(seq[RawUser])
+    for user in raw:
+      result.content.add user.toUser

src/experimental/types/graphuser.nim

@@ -1,3 +1,4 @@
+import options
 import user
 
 type
@@ -10,3 +11,5 @@ type
   UserResult = object
     legacy*: RawUser
     restId*: string
+    isBlueVerified*: bool
+    reason*: Option[string]

src/experimental/types/unifiedcard.nim

@@ -17,6 +17,8 @@ type
     twitterListDetails
     communityDetails
     mediaWithDetailsHorizontal
+    hidden
+    unknown
 
   Component* = object
     kind*: ComponentType
@@ -47,7 +49,7 @@ type
     vanity*: string
 
   MediaType* = enum
-    photo, video
+    photo, video, model3d
 
   MediaEntity* = object
     kind*: MediaType
@@ -70,10 +72,37 @@ type
   Text = object
     content: string
 
-  HasTypeField = Component | Destination | MediaEntity | AppStoreData
+  TypeField = Component | Destination | MediaEntity | AppStoreData
 
 converter fromText*(text: Text): string = text.content
 
-proc renameHook*(v: var HasTypeField; fieldName: var string) =
+proc renameHook*(v: var TypeField; fieldName: var string) =
   if fieldName == "type":
     fieldName = "kind"
+
+proc enumHook*(s: string; v: var ComponentType) =
+  v = case s
+      of "details": details
+      of "media": media
+      of "swipeable_media": swipeableMedia
+      of "button_group": buttonGroup
+      of "app_store_details": appStoreDetails
+      of "twitter_list_details": twitterListDetails
+      of "community_details": communityDetails
+      of "media_with_details_horizontal": mediaWithDetailsHorizontal
+      of "commerce_drop_details": hidden
+      else: echo "ERROR: Unknown enum value (ComponentType): ", s; unknown
+
+proc enumHook*(s: string; v: var AppType) =
+  v = case s
+      of "android_app": androidApp
+      of "iphone_app": iPhoneApp
+      of "ipad_app": iPadApp
+      else: echo "ERROR: Unknown enum value (AppType): ", s; androidApp
+
+proc enumHook*(s: string; v: var MediaType) =
+  v = case s
+      of "video": video
+      of "photo": photo
+      of "model3d": model3d
+      else: echo "ERROR: Unknown enum value (MediaType): ", s; photo

src/formatters.nim

@@ -12,8 +12,7 @@ let
   twRegex = re"(?<=(?<!\S)https:\/\/|(?<=\s))(www\.|mobile\.)?twitter\.com"
   twLinkRegex = re"""<a href="https:\/\/twitter.com([^"]+)">twitter\.com(\S+)</a>"""
 
-  ytRegex = re"([A-z.]+\.)?youtu(be\.com|\.be)"
-  igRegex = re"(www\.)?instagram\.com"
+  ytRegex = re(r"([A-z.]+\.)?youtu(be\.com|\.be)", {reStudy, reIgnoreCase})
 
   rdRegex = re"(?<![.b])((www|np|new|amp|old)\.)?reddit.com"
   rdShortRegex = re"(?<![.b])redd\.it\/"
@@ -56,15 +55,13 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
 
   if prefs.replaceYouTube.len > 0 and "youtu" in result:
     result = result.replace(ytRegex, prefs.replaceYouTube)
-    if prefs.replaceYouTube in result:
-      result = result.replace("/c/", "/")
 
   if prefs.replaceTwitter.len > 0 and ("twitter.com" in body or tco in body):
-    result = result.replace(tco, &"{https}{prefs.replaceTwitter}/t.co")
+    result = result.replace(tco, https & prefs.replaceTwitter & "/t.co")
     result = result.replace(cards, prefs.replaceTwitter & "/cards")
     result = result.replace(twRegex, prefs.replaceTwitter)
     result = result.replacef(twLinkRegex, a(
-      prefs.replaceTwitter & "$2", href = &"{https}{prefs.replaceTwitter}$1"))
+      prefs.replaceTwitter & "$2", href = https & prefs.replaceTwitter & "$1"))
 
   if prefs.replaceReddit.len > 0 and ("reddit.com" in result or "redd.it" in result):
     result = result.replace(rdShortRegex, prefs.replaceReddit & "/comments/")
@@ -72,9 +69,6 @@ proc replaceUrls*(body: string; prefs: Prefs; absolute=""): string =
     if prefs.replaceReddit in result and "/gallery/" in result:
       result = result.replace("/gallery/", "/comments/")
 
-  if prefs.replaceInstagram.len > 0 and "instagram.com" in result:
-    result = result.replace(igRegex, prefs.replaceInstagram)
-
   if absolute.len > 0 and "href" in result:
     result = result.replace("href=\"/", &"href=\"{absolute}/")
 

src/http_pool.nim

@@ -42,5 +42,11 @@ template use*(pool: HttpPool; heads: HttpHeaders; body: untyped): untyped =
   except ProtocolError:
     # Twitter closed the connection, retry
     body
+  except BadClientError:
+    # Twitter returned 503, we need a new client
+    pool.release(c, true)
+    badClient = false
+    c = pool.acquire(heads)
+    body
   finally:
     pool.release(c, badClient)

src/nitter.nim

@@ -56,6 +56,7 @@ settings:
   port = Port(cfg.port)
   staticDir = cfg.staticDir
   bindAddr = cfg.address
+  reusePort = true
 
 routes:
   get "/":
@@ -84,19 +85,23 @@ routes:
     resp Http500, showError(
       &"An error occurred, please {link} with the URL you tried to visit.", cfg)
 
+  error BadClientError:
+    echo error.exc.name, ": ", error.exc.msg
+    resp Http500, showError("Network error occured, please try again.", cfg)
+
   error RateLimitError:
     const link = a("another instance", href = instancesUrl)
     resp Http429, showError(
       &"Instance has been rate limited.<br>Use {link} or try again later.", cfg)
 
-  extend unsupported, ""
-  extend preferences, ""
-  extend resolver, ""
   extend rss, ""
+  extend status, ""
   extend search, ""
   extend timeline, ""
-  extend list, ""
-  extend status, ""
   extend media, ""
+  extend list, ""
+  extend preferences, ""
+  extend resolver, ""
   extend embed, ""
   extend debug, ""
+  extend unsupported, ""

src/parser.nim

@@ -4,6 +4,8 @@ import packedjson, packedjson/deserialiser
 import types, parserutils, utils
 import experimental/parser/unifiedcard
 
+proc parseGraphTweet(js: JsonNode): Tweet
+
 proc parseUser(js: JsonNode; id=""): User =
   if js.isNull: return
   result = User(
@@ -19,13 +21,20 @@ proc parseUser(js: JsonNode; id=""): User =
     tweets: js{"statuses_count"}.getInt,
     likes: js{"favourites_count"}.getInt,
     media: js{"media_count"}.getInt,
-    verified: js{"verified"}.getBool,
+    verified: js{"verified"}.getBool or js{"ext_is_blue_verified"}.getBool,
     protected: js{"protected"}.getBool,
     joinDate: js{"created_at"}.getTime
   )
 
   result.expandUserEntities(js)
 
+proc parseGraphUser(js: JsonNode): User =
+  let user = ? js{"user_results", "result"}
+  result = parseUser(user{"legacy"})
+
+  if "is_blue_verified" in user:
+    result.verified = true
+
 proc parseGraphList*(js: JsonNode): List =
   if js.isNull: return
 
@@ -38,14 +47,13 @@ proc parseGraphList*(js: JsonNode): List =
   result = List(
     id: list{"id_str"}.getStr,
     name: list{"name"}.getStr,
-    username: list{"user", "legacy", "screen_name"}.getStr,
-    userId: list{"user", "rest_id"}.getStr,
+    username: list{"user_results", "result", "legacy", "screen_name"}.getStr,
+    userId: list{"user_results", "result", "rest_id"}.getStr,
     description: list{"description"}.getStr,
     members: list{"member_count"}.getInt,
-    banner: list{"custom_banner_media", "media_info", "url"}.getImageStr
+    banner: list{"custom_banner_media", "media_info", "original_img_url"}.getImageStr
   )
 
-
 proc parsePoll(js: JsonNode): Poll =
   let vals = js{"binding_values"}
   # name format is pollNchoice_*
@@ -73,8 +81,8 @@ proc parseGif(js: JsonNode): Gif =
 proc parseVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"media_url_https"}.getImageStr,
-    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr,
-    available: js{"ext_media_availability", "status"}.getStr == "available",
+    views: js{"ext", "mediaStats", "r", "ok", "viewCount"}.getStr($js{"mediaStats", "viewCount"}.getInt),
+    available: js{"ext_media_availability", "status"}.getStr.toLowerAscii == "available",
     title: js{"ext_alt_text"}.getStr,
     durationMs: js{"video_info", "duration_millis"}.getInt
     # playbackType: mp4
@@ -186,7 +194,7 @@ proc parseCard(js: JsonNode; urls: JsonNode): Card =
      result.url.len == 0 or result.url.startsWith("card://"):
     result.url = getPicUrl(result.image)
 
-proc parseTweet(js: JsonNode): Tweet =
+proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull()): Tweet =
   if js.isNull: return
   result = Tweet(
     id: js{"id_str"}.getId,
@@ -194,7 +202,6 @@ proc parseTweet(js: JsonNode): Tweet =
     replyId: js{"in_reply_to_status_id_str"}.getId,
     text: js{"full_text"}.getStr,
     time: js{"created_at"}.getTime,
-    source: getSource(js),
     hasThread: js{"self_thread"}.notNull,
     available: true,
     user: User(id: js{"user_id_str"}.getStr),
@@ -208,14 +215,26 @@ proc parseTweet(js: JsonNode): Tweet =
 
   result.expandTweetEntities(js)
 
+  # fix for pinned threads
+  if result.hasThread and result.threadId == 0:
+    result.threadId = js{"self_thread", "id_str"}.getId
+
   if js{"is_quote_status"}.getBool:
     result.quote = some Tweet(id: js{"quoted_status_id_str"}.getId)
 
+  # legacy
   with rt, js{"retweeted_status_id_str"}:
     result.retweet = some Tweet(id: rt.getId)
     return
 
-  with jsCard, js{"card"}:
+  # graphql
+  with rt, js{"retweeted_status_result", "result"}:
+    # needed due to weird edgecase where the actual tweet data isn't included
+    if "legacy" in rt:
+      result.retweet = some parseGraphTweet(rt)
+      return
+
+  if jsCard.kind != JNull:
     let name = jsCard{"name"}.getStr
     if "poll" in name:
       if "image" in name:
@@ -235,7 +254,10 @@ proc parseTweet(js: JsonNode): Tweet =
       of "video":
         result.video = some(parseVideo(m))
         with user, m{"additional_media_info", "source_user"}:
-          result.attribution = some(parseUser(user))
+          if user{"id"}.getInt > 0:
+            result.attribution = some(parseUser(user))
+          else:
+            result.attribution = some(parseGraphUser(user))
       of "animated_gif":
         result.gif = some(parseGif(m))
       else: discard
@@ -292,70 +314,11 @@ proc parseGlobalObjects(js: JsonNode): GlobalObjects =
     result.users[k] = parseUser(v, k)
 
   for k, v in tweets:
-    var tweet = parseTweet(v)
+    var tweet = parseTweet(v, v{"card"})
     if tweet.user.id in result.users:
       tweet.user = result.users[tweet.user.id]
     result.tweets[k] = tweet
 
-proc parseThread(js: JsonNode; global: GlobalObjects): tuple[thread: Chain, self: bool] =
-  result.thread = Chain()
-
-  let thread = js{"content", "item", "content", "conversationThread"}
-  with cursor, thread{"showMoreCursor"}:
-    result.thread.cursor = cursor{"value"}.getStr
-    result.thread.hasMore = true
-
-  for t in thread{"conversationComponents"}:
-    let content = t{"conversationTweetComponent", "tweet"}
-
-    if content{"displayType"}.getStr == "SelfThread":
-      result.self = true
-
-    var tweet = finalizeTweet(global, content{"id"}.getStr)
-    if not tweet.available:
-      tweet.tombstone = getTombstone(content{"tombstone"})
-    result.thread.content.add tweet
-
-proc parseConversation*(js: JsonNode; tweetId: string): Conversation =
-  result = Conversation(replies: Result[Chain](beginning: true))
-  let global = parseGlobalObjects(? js)
-
-  let instructions = ? js{"timeline", "instructions"}
-  if instructions.len == 0:
-    return
-
-  for e in instructions[0]{"addEntries", "entries"}:
-    let entry = e{"entryId"}.getStr
-    if "tweet" in entry or "tombstone" in entry:
-      let tweet = finalizeTweet(global, e.getEntryId)
-      if $tweet.id != tweetId:
-        result.before.content.add tweet
-      else:
-        result.tweet = tweet
-    elif "conversationThread" in entry:
-      let (thread, self) = parseThread(e, global)
-      if thread.content.len > 0:
-        if self:
-          result.after = thread
-        else:
-          result.replies.content.add thread
-    elif "cursor-showMore" in entry:
-      result.replies.bottom = e.getCursor
-    elif "cursor-bottom" in entry:
-      result.replies.bottom = e.getCursor
-
-proc parseStatus*(js: JsonNode): Tweet =
-  with e, js{"errors"}:
-    if e.getError == tweetNotFound:
-      return
-
-  result = parseTweet(js)
-  if not result.isNil:
-    result.user = parseUser(js{"user"})
-
-    with quote, js{"quoted_status"}:
-      result.quote = some parseStatus(js{"quoted_status"})
-
 proc parseInstructions[T](res: var Result[T]; global: GlobalObjects; js: JsonNode) =
   if js.kind != JArray or js.len == 0:
     return
@@ -396,7 +359,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
       result.top = e.getCursor
     elif "cursor-bottom" in entry:
       result.bottom = e.getCursor
-    elif entry.startsWith("sq-C"):
+    elif entry.startsWith("sq-cursor"):
       with cursor, e{"content", "operation", "cursor"}:
         if cursor{"cursorType"}.getStr == "Bottom":
           result.bottom = cursor{"value"}.getStr
@@ -406,7 +369,7 @@ proc parseTimeline*(js: JsonNode; after=""): Timeline =
 proc parsePhotoRail*(js: JsonNode): PhotoRail =
   for tweet in js:
     let
-      t = parseTweet(tweet)
+      t = parseTweet(tweet, js{"card"})
       url = if t.photos.len > 0: t.photos[0]
             elif t.video.isSome: get(t.video).thumb
             elif t.gif.isSome: get(t.gif).thumb
@@ -415,3 +378,141 @@ proc parsePhotoRail*(js: JsonNode): PhotoRail =
 
     if url.len == 0: continue
     result.add GalleryPhoto(url: url, tweetId: $t.id)
+
+proc parseGraphTweet(js: JsonNode): Tweet =
+  if js.kind == JNull:
+    return Tweet()
+
+  case js{"__typename"}.getStr
+  of "TweetUnavailable":
+    return Tweet()
+  of "TweetTombstone":
+    return Tweet(text: js{"tombstone", "text"}.getTombstone)
+  of "TweetPreviewDisplay":
+    return Tweet(text: "You're unable to view this Tweet because it's only available to the Subscribers of the account owner.")
+  of "TweetWithVisibilityResults":
+    return parseGraphTweet(js{"tweet"})
+
+  var jsCard = copy(js{"card", "legacy"})
+  if jsCard.kind != JNull:
+    var values = newJObject()
+    for val in jsCard["binding_values"]:
+      values[val["key"].getStr] = val["value"]
+    jsCard["binding_values"] = values
+
+  result = parseTweet(js{"legacy"}, jsCard)
+  result.user = parseGraphUser(js{"core"})
+
+  with noteTweet, js{"note_tweet", "note_tweet_results", "result"}:
+    result.expandNoteTweetEntities(noteTweet)
+
+  if result.quote.isSome:
+    result.quote = some(parseGraphTweet(js{"quoted_status_result", "result"}))
+
+proc parseGraphThread(js: JsonNode): tuple[thread: Chain; self: bool] =
+  let thread = js{"content", "items"}
+  for t in js{"content", "items"}:
+    let entryId = t{"entryId"}.getStr
+    if "cursor-showmore" in entryId:
+      let cursor = t{"item", "itemContent", "value"}
+      result.thread.cursor = cursor.getStr
+      result.thread.hasMore = true
+    elif "tweet" in entryId:
+      let tweet = parseGraphTweet(t{"item", "itemContent", "tweet_results", "result"})
+      result.thread.content.add tweet
+
+      if t{"item", "itemContent", "tweetDisplayType"}.getStr == "SelfThread":
+        result.self = true
+
+proc parseGraphTweetResult*(js: JsonNode): Tweet =
+  with tweet, js{"data", "tweetResult", "result"}:
+    result = parseGraphTweet(tweet)
+
+proc parseGraphConversation*(js: JsonNode; tweetId: string): Conversation =
+  result = Conversation(replies: Result[Chain](beginning: true))
+
+  let instructions = ? js{"data", "threaded_conversation_with_injections", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for e in instructions[0]{"entries"}:
+    let entryId = e{"entryId"}.getStr
+    # echo entryId
+    if entryId.startsWith("tweet"):
+      with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+        let tweet = parseGraphTweet(tweetResult)
+
+        if not tweet.available:
+          tweet.id = parseBiggestInt(entryId.getId())
+
+        if $tweet.id == tweetId:
+          result.tweet = tweet
+        else:
+          result.before.content.add tweet
+    elif entryId.startsWith("tombstone"):
+      let id = entryId.getId()
+      let tweet = Tweet(
+        id: parseBiggestInt(id),
+        available: false,
+        text: e{"content", "itemContent", "tombstoneInfo", "richText"}.getTombstone
+      )
+
+      if id == tweetId:
+        result.tweet = tweet
+      else:
+        result.before.content.add tweet
+    elif entryId.startsWith("conversationthread"):
+      let (thread, self) = parseGraphThread(e)
+      if self:
+        result.after = thread
+      else:
+        result.replies.content.add thread
+    elif entryId.startsWith("cursor-bottom"):
+      result.replies.bottom = e{"content", "itemContent", "value"}.getStr
+
+proc parseGraphTimeline*(js: JsonNode; root: string; after=""): Timeline =
+  result = Timeline(beginning: after.len == 0)
+
+  let instructions =
+    if root == "list": ? js{"data", "list", "tweets_timeline", "timeline", "instructions"}
+    else: ? js{"data", "user", "result", "timeline_v2", "timeline", "instructions"}
+
+  if instructions.len == 0:
+    return
+
+  for i in instructions:
+    if i{"type"}.getStr == "TimelineAddEntries":
+      for e in i{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+            let tweet = parseGraphTweet(tweetResult)
+            if not tweet.available:
+              tweet.id = parseBiggestInt(entryId.getId())
+            result.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          result.bottom = e{"content", "value"}.getStr
+
+proc parseGraphSearch*(js: JsonNode; after=""): Timeline =
+  result = Timeline(beginning: after.len == 0)
+
+  let instructions = js{"data", "search_by_raw_query", "search_timeline", "timeline", "instructions"}
+  if instructions.len == 0:
+    return
+
+  for instruction in instructions:
+    let typ = instruction{"type"}.getStr
+    if typ == "TimelineAddEntries":
+      for e in instructions[0]{"entries"}:
+        let entryId = e{"entryId"}.getStr
+        if entryId.startsWith("tweet"):
+          with tweetResult, e{"content", "itemContent", "tweet_results", "result"}:
+            let tweet = parseGraphTweet(tweetResult)
+            if not tweet.available:
+              tweet.id = parseBiggestInt(entryId.getId())
+            result.content.add tweet
+        elif entryId.startsWith("cursor-bottom"):
+          result.bottom = e{"content", "value"}.getStr
+    elif typ == "TimelineReplaceEntry":
+      if instruction{"entry_id_to_replace"}.getStr.startsWith("cursor-bottom"):
+        result.bottom = instruction{"entry", "content", "value"}.getStr

src/parserutils.nim

@@ -28,13 +28,13 @@ template `?`*(js: JsonNode): untyped =
   if j.isNull: return
   j
 
-template `with`*(ident, value, body): untyped =
-  block:
+template with*(ident, value, body): untyped =
+  if true:
     let ident {.inject.} = value
     if ident != nil: body
 
-template `with`*(ident; value: JsonNode; body): untyped =
-  block:
+template with*(ident; value: JsonNode; body): untyped =
+  if true:
     let ident {.inject.} = value
     if value.notNull: body
 
@@ -130,13 +130,9 @@ proc getBanner*(js: JsonNode): string =
       return
 
 proc getTombstone*(js: JsonNode): string =
-  result = js{"tombstoneInfo", "richText", "text"}.getStr
+  result = js{"text"}.getStr
   result.removeSuffix(" Learn more")
 
-proc getSource*(js: JsonNode): string =
-  let src = js{"source"}.getStr
-  result = src.substr(src.find('>') + 1, src.rfind('<') - 1)
-
 proc getMp4Resolution*(url: string): int =
   # parses the height out of a URL like this one:
   # https://video.twimg.com/ext_tw_video/<tweet-id>/pu/vid/720x1280/<random>.mp4
@@ -234,47 +230,37 @@ proc expandUserEntities*(user: var User; js: JsonNode) =
   user.bio = user.bio.replacef(unRegex, unReplace)
                      .replacef(htRegex, htReplace)
 
-proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
-  let
-    orig = tweet.text.toRunes
-    textRange = js{"display_text_range"}
-    textSlice = textRange{0}.getInt .. textRange{1}.getInt
-    hasQuote = js{"is_quote_status"}.getBool
-    hasCard = tweet.card.isSome
-
-  var replyTo = ""
-  if tweet.replyId != 0:
-    with reply, js{"in_reply_to_screen_name"}:
-      tweet.reply.add reply.getStr
-      replyTo = reply.getStr
-
-  let ent = ? js{"entities"}
+proc expandTextEntities(tweet: Tweet; entities: JsonNode; text: string; textSlice: Slice[int];
+                        replyTo=""; hasQuote=false) =
+  let hasCard = tweet.card.isSome
 
   var replacements = newSeq[ReplaceSlice]()
 
-  with urls, ent{"urls"}:
+  with urls, entities{"urls"}:
     for u in urls:
       let urlStr = u["url"].getStr
-      if urlStr.len == 0 or urlStr notin tweet.text:
+      if urlStr.len == 0 or urlStr notin text:
         continue
+
       replacements.extractUrls(u, textSlice.b, hideTwitter = hasQuote)
+
       if hasCard and u{"url"}.getStr == get(tweet.card).url:
         get(tweet.card).url = u{"expanded_url"}.getStr
 
-  with media, ent{"media"}:
+  with media, entities{"media"}:
     for m in media:
       replacements.extractUrls(m, textSlice.b, hideTwitter = true)
 
-  if "hashtags" in ent:
-    for hashtag in ent["hashtags"]:
+  if "hashtags" in entities:
+    for hashtag in entities["hashtags"]:
       replacements.extractHashtags(hashtag)
 
-  if "symbols" in ent:
-    for symbol in ent["symbols"]:
+  if "symbols" in entities:
+    for symbol in entities["symbols"]:
       replacements.extractHashtags(symbol)
 
-  if "user_mentions" in ent:
-    for mention in ent["user_mentions"]:
+  if "user_mentions" in entities:
+    for mention in entities["user_mentions"]:
       let
         name = mention{"screen_name"}.getStr
         slice = mention.extractSlice
@@ -291,5 +277,27 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
   replacements.deduplicate
   replacements.sort(cmp)
 
-  tweet.text = orig.replacedWith(replacements, textSlice)
-                   .strip(leading=false)
+  tweet.text = text.toRunes.replacedWith(replacements, textSlice).strip(leading=false)
+
+proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entities"}
+    hasQuote = js{"is_quote_status"}.getBool
+    textRange = js{"display_text_range"}
+    textSlice = textRange{0}.getInt .. textRange{1}.getInt
+
+  var replyTo = ""
+  if tweet.replyId != 0:
+    with reply, js{"in_reply_to_screen_name"}:
+      replyTo = reply.getStr
+      tweet.reply.add replyTo
+
+  tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo, hasQuote)
+
+proc expandNoteTweetEntities*(tweet: Tweet; js: JsonNode) =
+  let
+    entities = ? js{"entity_set"}
+    text = js{"text"}.getStr
+    textSlice = 0..text.runeLen
+
+  tweet.expandTextEntities(entities, text, textSlice)

src/prefs_impl.nim

@@ -83,7 +83,7 @@ genPrefs:
       "Enable mp4 video playback"
 
     hlsPlayback(checkbox, false):
-      "Enable hls video streaming (requires JavaScript)"
+      "Enable HLS video streaming (requires JavaScript)"
 
     proxyVideos(checkbox, true):
       "Proxy video streaming through the server (might be slow)"
@@ -107,10 +107,6 @@ genPrefs:
       "Reddit -> Teddit/Libreddit"
       placeholder: "Teddit hostname"
 
-    replaceInstagram(input, ""):
-      "Instagram -> Bibliogram"
-      placeholder: "Bibliogram hostname"
-
 iterator allPrefs*(): Pref =
   for k, v in prefList:
     for pref in v:

src/redis_cache.nim

@@ -118,11 +118,11 @@ proc getUserId*(username: string): Future[string] {.async.} =
   pool.withAcquire(r):
     result = await r.hGet(name.uidKey, name)
     if result == redisNil:
-      let user = await getUser(username)
+      let user = await getGraphUser(username)
       if user.suspended:
         return "suspended"
       else:
-        await cacheUserId(name, user.id)
+        await all(cacheUserId(name, user.id), cache(user))
         return user.id
 
 proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
@@ -130,8 +130,7 @@ proc getCachedUser*(username: string; fetch=true): Future[User] {.async.} =
   if prof != redisNil:
     prof.deserialize(User)
   elif fetch:
-    let userId = await getUserId(username)
-    result = await getGraphUser(userId)
+    result = await getGraphUser(username)
     await cache(result)
 
 proc getCachedUsername*(userId: string): Future[string] {.async.} =
@@ -142,9 +141,11 @@ proc getCachedUsername*(userId: string): Future[string] {.async.} =
   if username != redisNil:
     result = username
   else:
-    let user = await getUserById(userId)
+    let user = await getGraphUserById(userId)
     result = user.username
     await setEx(key, baseCacheTime, result)
+    if result.len > 0 and user.id.len > 0:
+      await all(cacheUserId(result, user.id), cache(user))
 
 proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
   if id == 0: return
@@ -152,8 +153,8 @@ proc getCachedTweet*(id: int64): Future[Tweet] {.async.} =
   if tweet != redisNil:
     tweet.deserialize(Tweet)
   else:
-    result = await getStatus($id)
-    if result.isNil:
+    result = await getGraphTweetResult($id)
+    if not result.isNil:
       await cache(result)
 
 proc getCachedPhotoRail*(name: string): Future[PhotoRail] {.async.} =

src/routes/embed.nim

@@ -25,7 +25,7 @@ proc createEmbedRouter*(cfg: Config) =
       if convo == nil or convo.tweet == nil:
         resp Http404
 
-      resp $renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
+      resp renderTweetEmbed(convo.tweet, path, prefs, cfg, request)
 
     get "/embed/Tweet.html":
       let id = @"id"

src/routes/list.nim

@@ -6,7 +6,6 @@ import jester
 import router_utils
 import ".."/[types, redis_cache, api]
 import ../views/[general, timeline, list]
-export getListTimeline, getGraphList
 
 template respList*(list, timeline, title, vnode: typed) =
   if list.id.len == 0 or list.name.len == 0:
@@ -39,7 +38,7 @@ proc createListRouter*(cfg: Config) =
       let
         prefs = cookiePrefs()
         list = await getCachedList(id=(@"id"))
-        timeline = await getListTimeline(list.id, getCursor())
+        timeline = await getGraphListTweets(list.id, getCursor())
         vnode = renderTimelineTweets(timeline, prefs, request.path)
       respList(list, timeline, list.title, vnode)
 

src/routes/rss.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, tables, times, hashes, uri
+import asyncdispatch, tables, times, hashes, uri
 
 import jester
 
@@ -10,6 +10,11 @@ include "../views/rss.nimf"
 
 export times, hashes
 
+proc redisKey*(page, name, cursor: string): string =
+  result = page & ":" & name
+  if cursor.len > 0:
+    result &= ":" & cursor
+
 proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.} =
   var profile: Profile
   let
@@ -23,7 +28,7 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
     var q = query
     q.fromUser = names
     profile = Profile(
-      tweets: await getSearch[Tweet](q, after),
+      tweets: await getGraphSearch(q, after),
       # this is kinda dumb
       user: User(
         username: name,
@@ -42,8 +47,8 @@ proc timelineRss*(req: Request; cfg: Config; query: Query): Future[Rss] {.async.
 template respRss*(rss, page) =
   if rss.cursor.len == 0:
     let info = case page
-               of "User": &""" "{@"name"}" """
-               of "List": &""" "{@"id"}" """
+               of "User": " \"" & @"name" & "\" "
+               of "List": " \"" & @"id" & "\" "
                else: " "
 
     resp Http404, showError(page & info & "not found", cfg)
@@ -67,13 +72,13 @@ proc createRssRouter*(cfg: Config) =
 
       let
         cursor = getCursor()
-        key = &"search:{hash(genQueryUrl(query))}:cursor"
+        key = redisKey("search", $hash(genQueryUrl(query)), cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "Search")
 
-      let tweets = await getSearch[Tweet](query, cursor)
+      let tweets = await getGraphSearch(query, cursor)
       rss.cursor = tweets.bottom
       rss.feed = renderSearchRss(tweets.content, query.text, genQueryUrl(query), cfg)
 
@@ -84,9 +89,8 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       let
-        cursor = getCursor()
         name = @"name"
-        key = &"twitter:{name}:{cursor}"
+        key = redisKey("twitter", name, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -101,18 +105,20 @@ proc createRssRouter*(cfg: Config) =
       cond cfg.enableRss
       cond '.' notin @"name"
       cond @"tab" in ["with_replies", "media", "search"]
-      let name = @"name"
-      let query =
-        case @"tab"
-        of "with_replies": getReplyQuery(name)
-        of "media": getMediaQuery(name)
-        of "search": initQuery(params(request), name=name)
-        else: Query(fromUser: @[name])
-
-      var key = &"""{@"tab"}:{@"name"}:"""
-      if @"tab" == "search":
-        key &= $hash(genQueryUrl(query)) & ":"
-      key &= getCursor()
+      let
+        name = @"name"
+        tab = @"tab"
+        query =
+          case tab
+          of "with_replies": getReplyQuery(name)
+          of "media": getMediaQuery(name)
+          of "search": initQuery(params(request), name=name)
+          else: Query(fromUser: @[name])
+
+      let searchKey = if tab != "search": ""
+                      else: ":" & $hash(genQueryUrl(query))
+
+      let key = redisKey(tab, name & searchKey, getCursor())
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
@@ -132,29 +138,28 @@ proc createRssRouter*(cfg: Config) =
         cursor = getCursor()
 
       if list.id.len == 0:
-        resp Http404, showError(&"""List "{@"slug"}" not found""", cfg)
+        resp Http404, showError("List \"" & @"slug" & "\" not found", cfg)
 
-      let url = &"/i/lists/{list.id}/rss"
+      let url = "/i/lists/" & list.id & "/rss"
       if cursor.len > 0:
-        redirect(&"{url}?cursor={encodeUrl(cursor, false)}")
+        redirect(url & "?cursor=" & encodeUrl(cursor, false))
       else:
         redirect(url)
 
     get "/i/lists/@id/rss":
       cond cfg.enableRss
       let
+        id = @"id"
         cursor = getCursor()
-        key =
-          if cursor.len == 0: "lists:" & @"id"
-          else: &"""lists:{@"id"}:{cursor}"""
+        key = redisKey("lists", id, cursor)
 
       var rss = await getCachedRss(key)
       if rss.cursor.len > 0:
         respRss(rss, "List")
 
       let
-        list = await getCachedList(id=(@"id"))
-        timeline = await getListTimeline(list.id, cursor)
+        list = await getCachedList(id=id)
+        timeline = await getGraphListTweets(list.id, cursor)
       rss.cursor = timeline.bottom
       rss.feed = renderListRss(timeline.content, list, cfg)
 

src/routes/search.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import strutils, strformat, uri
+import strutils, uri
 
 import jester
 
@@ -14,32 +14,38 @@ export search
 proc createSearchRouter*(cfg: Config) =
   router search:
     get "/search/?":
-      if @"q".len > 500:
+      let q = @"q"
+      if q.len > 500:
         resp Http400, showError("Search input too long.", cfg)
 
       let
         prefs = cookiePrefs()
         query = initQuery(params(request))
+        title = "Search" & (if q.len > 0: " (" & q & ")" else: "")
 
       case query.kind
       of users:
-        if "," in @"q":
-          redirect("/" & @"q")
-        let users = await getSearch[User](query, getCursor())
-        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs)
+        if "," in q:
+          redirect("/" & q)
+        var users: Result[User]
+        try:
+          users = await getUserSearch(query, getCursor())
+        except InternalError:
+          users = Result[User](beginning: true, query: query)
+        resp renderMain(renderUserSearch(users, prefs), request, cfg, prefs, title)
       of tweets:
         let
-          tweets = await getSearch[Tweet](query, getCursor())
+          tweets = await getGraphSearch(query, getCursor())
           rss = "/search/rss?" & genQueryUrl(query)
         resp renderMain(renderTweetSearch(tweets, prefs, getPath()),
-                        request, cfg, prefs, rss=rss)
+                        request, cfg, prefs, title, rss=rss)
       else:
         resp Http404, showError("Invalid search", cfg)
 
     get "/hashtag/@hash":
-      redirect(&"""/search?q={encodeUrl("#" & @"hash")}""")
+      redirect("/search?q=" & encodeUrl("#" & @"hash"))
 
     get "/opensearch":
       let url = getUrlPrefix(cfg) & "/search?q="
       resp Http200, {"Content-Type": "application/opensearchdescription+xml"},
-                    generateOpenSearchXML(cfg.title, cfg.hostname, url)
+                     generateOpenSearchXML(cfg.title, cfg.hostname, url)

src/routes/status.nim

@@ -16,17 +16,21 @@ proc createStatusRouter*(cfg: Config) =
   router status:
     get "/@name/status/@id/?":
       cond '.' notin @"name"
-      cond not @"id".any(c => not c.isDigit)
+      let id = @"id"
+
+      if id.len > 19 or id.any(c => not c.isDigit):
+        resp Http404, showError("Invalid tweet ID", cfg)
+
       let prefs = cookiePrefs()
 
       # used for the infinite scroll feature
       if @"scroll".len > 0:
-        let replies = await getReplies(@"id", getCursor())
+        let replies = await getReplies(id, getCursor())
         if replies.content.len == 0:
           resp Http404, ""
         resp $renderReplies(replies, prefs, getPath())
 
-      let conv = await getTweet(@"id", getCursor())
+      let conv = await getTweet(id, getCursor())
       if conv == nil:
         echo "nil conv"
 
@@ -72,3 +76,6 @@ proc createStatusRouter*(cfg: Config) =
 
     get "/i/web/status/@id":
       redirect("/i/status/" & @"id")
+      
+    get "/@name/thread/@id/?":
+      redirect("/$1/status/$2" % [@"name", @"id"])

src/routes/timeline.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import asyncdispatch, strutils, strformat, sequtils, uri, options, times
+import asyncdispatch, strutils, sequtils, uri, options, times
 import jester, karax/vdom
 
 import router_utils
@@ -47,10 +47,10 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
   let
     timeline =
       case query.kind
-      of posts: getTimeline(userId, after)
-      of replies: getTimeline(userId, after, replies=true)
-      of media: getMediaTimeline(userId, after)
-      else: getSearch[Tweet](query, after)
+      of posts: getGraphUserTweets(userId, TimelineKind.tweets, after)
+      of replies: getGraphUserTweets(userId, TimelineKind.replies, after)
+      of media: getGraphUserTweets(userId, TimelineKind.media, after)
+      else: getGraphSearch(query, after)
 
     rail =
       skipIf(skipRail or query.kind == media, @[]):
@@ -64,6 +64,7 @@ proc fetchProfile*(after: string; query: Query; skipRail=false;
     let tweet = await getCachedTweet(user.pinnedTweet)
     if not tweet.isNil:
       tweet.pinned = true
+      tweet.user = user
       pinned = some tweet
 
   result = Profile(
@@ -82,7 +83,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
                    rss, after: string): Future[string] {.async.} =
   if query.fromUser.len != 1:
     let
-      timeline = await getSearch[Tweet](query, after)
+      timeline = await getGraphSearch(query, after)
       html = renderTweetSearch(timeline, prefs, getPath())
     return renderMain(html, request, cfg, prefs, "Multi", rss=rss)
 
@@ -102,7 +103,7 @@ proc showTimeline*(request: Request; query: Query; cfg: Config; prefs: Prefs;
 template respTimeline*(timeline: typed) =
   let t = timeline
   if t.len == 0:
-    resp Http404, showError(&"""User "{@"name"}" not found""", cfg)
+    resp Http404, showError("User \"" & @"name" & "\" not found", cfg)
   resp t
 
 template respUserId*() =
@@ -123,7 +124,7 @@ proc createTimelineRouter*(cfg: Config) =
 
     get "/@name/?@tab?/?":
       cond '.' notin @"name"
-      cond @"name" notin ["pic", "gif", "video"]
+      cond @"name" notin ["pic", "gif", "video", "search", "settings", "login", "intent", "i"]
       cond @"tab" in ["with_replies", "media", "search", ""]
       let
         prefs = cookiePrefs()
@@ -137,7 +138,7 @@ proc createTimelineRouter*(cfg: Config) =
       # used for the infinite scroll feature
       if @"scroll".len > 0:
         if query.fromUser.len != 1:
-          var timeline = await getSearch[Tweet](query, after)
+          var timeline = await getGraphSearch(query, after)
           if timeline.content.len == 0: resp Http404
           timeline.beginning = true
           resp $renderTweetSearch(timeline, prefs, getPath())

src/sass/profile/card.scss

@@ -73,9 +73,9 @@
         }
     }
 
-    .profile-joindate, .profile-location, profile-website {
+    .profile-joindate, .profile-location, .profile-website {
         color: var(--fg_faded);
-        margin: 2px 0;
+        margin: 1px 0;
         width: 100%;
     }
 }

src/sass/tweet/_base.scss

@@ -100,6 +100,7 @@
 .avatar {
     &.round {
         border-radius: 50%;
+        -webkit-user-select: none;
     }
     
     &.mini {
@@ -137,7 +138,6 @@
     }
 }
 
-
 .attribution {
     display: flex;
     pointer-events: all;
@@ -200,6 +200,7 @@
 
 .tweet-stats {
     margin-bottom: -3px;
+    -webkit-user-select: none;
 }
 
 .tweet-stat {
@@ -231,6 +232,7 @@
     left: 0;
     top: 0;
     position: absolute;
+    -webkit-user-select: none;
 
     &:hover {
         background-color: var(--bg_hover);

src/sass/tweet/thread.scss

@@ -23,7 +23,6 @@
     font-size: 18px;
 }
 
-
 @media(max-width: 600px) {
     .main-tweet .tweet-content {
         font-size: 16px;

src/sass/tweet/video.scss

@@ -3,7 +3,7 @@
 
 video {
     max-height: 100%;
-    max-width: 100%;
+    width: 100%;
 }
 
 .gallery-video {

src/tokens.nim

@@ -1,8 +1,7 @@
 # SPDX-License-Identifier: AGPL-3.0-only
 import asyncdispatch, httpclient, times, sequtils, json, random
 import strutils, tables
-import zippy
-import types, consts, http_pool
+import types, consts
 
 const
   maxConcurrentReqs = 5  # max requests at a time per token, to avoid race conditions
@@ -11,11 +10,12 @@ const
   failDelay = initDuration(minutes=30)
 
 var
-  clientPool: HttpPool
   tokenPool: seq[Token]
   lastFailed: Time
   enableLogging = false
 
+let headers = newHttpHeaders({"authorization": auth})
+
 template log(str) =
   if enableLogging: echo "[tokens] ", str
 
@@ -41,9 +41,12 @@ proc getPoolJson*(): JsonNode =
       let
         maxReqs =
           case api
-          of Api.listMembers, Api.listBySlug, Api.list, Api.userRestId: 500
           of Api.timeline: 187
-          else: 180
+          of Api.listMembers, Api.listBySlug, Api.list, Api.listTweets,
+             Api.userTweets, Api.userTweetsAndReplies, Api.userMedia,
+             Api.userRestId, Api.userScreenName,
+             Api.tweetDetail, Api.tweetResult, Api.search: 500
+          of Api.userSearch: 900
         reqs = maxReqs - token.apis[api].remaining
 
       reqsPerApi[$api] = reqsPerApi.getOrDefault($api, 0) + reqs
@@ -64,18 +67,12 @@ proc fetchToken(): Future[Token] {.async.} =
   if getTime() - lastFailed < failDelay:
     raise rateLimitError()
 
-  let headers = newHttpHeaders({
-    "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
-    "accept-encoding": "gzip",
-    "accept-language": "en-US,en;q=0.5",
-    "connection": "keep-alive",
-    "authorization": auth
-  })
+  let client = newAsyncHttpClient(headers=headers)
 
   try:
     let
-      resp = clientPool.use(headers): await c.postContent(activate)
-      tokNode = parseJson(uncompress(resp))["guest_token"]
+      resp = await client.postContent(activate)
+      tokNode = parseJson(resp)["guest_token"]
       tok = tokNode.getStr($(tokNode.getInt))
       time = getTime()
 
@@ -85,6 +82,8 @@ proc fetchToken(): Future[Token] {.async.} =
     if "Try again" notin e.msg:
       echo "[tokens] fetching tokens paused, resuming in 30 minutes"
       lastFailed = getTime()
+  finally:
+    client.close()
 
 proc expired(token: Token): bool =
   let time = getTime()
@@ -157,7 +156,6 @@ proc poolTokens*(amount: int) {.async.} =
       tokenPool.add newToken
 
 proc initTokenPool*(cfg: Config) {.async.} =
-  clientPool = HttpPool()
   enableLogging = cfg.enableDebug
 
   while true:

src/types.nim

@@ -7,17 +7,28 @@ genPrefsType()
 type
   RateLimitError* = object of CatchableError
   InternalError* = object of CatchableError
+  BadClientError* = object of CatchableError
+
+  TimelineKind* {.pure.} = enum
+    tweets
+    replies
+    media
 
   Api* {.pure.} = enum
-    userShow
+    tweetDetail
+    tweetResult
     timeline
     search
-    tweet
+    userSearch
     list
     listBySlug
     listMembers
+    listTweets
     userRestId
-    status
+    userScreenName
+    userTweets
+    userTweetsAndReplies
+    userMedia
 
   RateLimit* = object
     remaining*: int
@@ -34,17 +45,22 @@ type
     null = 0
     noUserMatches = 17
     protectedUser = 22
+    missingParams = 25
     couldntAuth = 32
     doesntExist = 34
+    invalidParam = 47
     userNotFound = 50
     suspended = 63
     rateLimited = 88
     invalidToken = 89
     listIdOrSlug = 112
     tweetNotFound = 144
+    tweetNotAuthorized = 179
     forbidden = 200
     badToken = 239
     noCsrf = 353
+    tweetUnavailable = 421
+    tweetCensored = 422
 
   User* = object
     id*: string
@@ -145,6 +161,7 @@ type
     imageDirectMessage = "image_direct_message"
     audiospace = "audiospace"
     newsletterPublication = "newsletter_publication"
+    hidden
     unknown
     
   Card* = object
@@ -175,6 +192,7 @@ type
     available*: bool
     tombstone*: string
     location*: string
+    # Unused, needed for backwards compat
     source*: string
     stats*: TweetStats
     retweet*: Option[Tweet]

src/views/embed.nim

@@ -15,7 +15,8 @@ proc renderVideoEmbed*(tweet: Tweet; cfg: Config; req: Request): string =
   let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req, video=vidUrl, images=(@[thumb]))
 
-    tdiv(class="embed-video"):
-      renderVideo(get(tweet.video), prefs, "")
+    body:
+      tdiv(class="embed-video"):
+        renderVideo(get(tweet.video), prefs, "")
 
   result = doctype & $node

src/views/general.nim

@@ -81,7 +81,7 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
 
     title:
       if titleText.len > 0:
-        text &"{titleText}|{cfg.title}"
+        text titleText & " | " & cfg.title
       else:
         text cfg.title
 
@@ -98,9 +98,8 @@ proc renderHead*(prefs: Prefs; cfg: Config; req: Request; titleText=""; desc="";
       link(rel="preload", type="image/png", href=bannerUrl, `as`="image")
 
     for url in images:
-      let suffix = if "400x400" in url or url.endsWith("placeholder.png"): ""
-                   else: "?name=small"
-      let preloadUrl = getPicUrl(url & suffix)
+      let preloadUrl = if "400x400" in url: getPicUrl(url)
+                       else: getSmallPic(url)
       link(rel="preload", type="image/png", href=preloadUrl, `as`="image")
 
       let image = getUrlPrefix(cfg) & getPicUrl(url)

src/views/profile.nim

@@ -50,7 +50,7 @@ proc renderUserCard*(user: User; prefs: Prefs): VNode =
           span:
             let url = replaceUrls(user.website, prefs)
             icon "link"
-            a(href=url): text shortLink(url)
+            a(href=url): text url.shortLink
 
       tdiv(class="profile-joindate"):
         span(title=getJoinDateFull(user)):
@@ -108,7 +108,7 @@ proc renderProfile*(profile: var Profile; prefs: Prefs; path: string): VNode =
         renderBanner(profile.user.banner)
 
     let sticky = if prefs.stickyProfile: " sticky" else: ""
-    tdiv(class=(&"profile-tab{sticky}")):
+    tdiv(class=("profile-tab" & sticky)):
       renderUserCard(profile.user, prefs)
       if profile.photoRail.len > 0:
         renderPhotoRail(profile)

src/views/renderutils.nim

@@ -3,6 +3,14 @@ import strutils, strformat
 import karax/[karaxdsl, vdom, vstyles]
 import ".."/[types, utils]
 
+const smallWebp* = "?name=small&format=webp"
+
+proc getSmallPic*(url: string): string =
+  result = url
+  if "?" notin url and not url.endsWith("placeholder.png"):
+    result &= smallWebp
+  result = getPicUrl(result)
+
 proc icon*(icon: string; text=""; title=""; class=""; href=""): VNode =
   var c = "icon-" & icon
   if class.len > 0: c = &"{c} {class}"
@@ -51,29 +59,23 @@ proc buttonReferer*(action, text, path: string; class=""; `method`="post"): VNod
 proc genCheckbox*(pref, label: string; state: bool): VNode =
   buildHtml(label(class="pref-group checkbox-container")):
     text label
-    if state: input(name=pref, `type`="checkbox", checked="")
-    else: input(name=pref, `type`="checkbox")
+    input(name=pref, `type`="checkbox", checked=state)
     span(class="checkbox")
 
-proc genInput*(pref, label, state, placeholder: string; class=""): VNode =
+proc genInput*(pref, label, state, placeholder: string; class=""; autofocus=true): VNode =
   let p = placeholder
   buildHtml(tdiv(class=("pref-group pref-input " & class))):
     if label.len > 0:
       label(`for`=pref): text label
-    if state.len == 0:
-      input(name=pref, `type`="text", placeholder=p, value=state, autofocus="")
-    else:
-      input(name=pref, `type`="text", placeholder=p, value=state)
+    input(name=pref, `type`="text", placeholder=p, value=state, autofocus=(autofocus and state.len == 0))
 
 proc genSelect*(pref, label, state: string; options: seq[string]): VNode =
   buildHtml(tdiv(class="pref-group pref-input")):
     label(`for`=pref): text label
     select(name=pref):
       for opt in options:
-        if opt == state:
-          option(value=opt, selected=""): text opt
-        else:
-          option(value=opt): text opt
+        option(value=opt, selected=(opt == state)):
+          text opt
 
 proc genDate*(pref, state: string): VNode =
   buildHtml(span(class="date-input")):
@@ -85,12 +87,9 @@ proc genImg*(url: string; class=""): VNode =
     img(src=getPicUrl(url), class=class, alt="", loading="lazy", decoding="async")
 
 proc getTabClass*(query: Query; tab: QueryKind): string =
-  result = "tab-item"
-  if query.kind == tab:
-    result &= " active"
+  if query.kind == tab: "tab-item active"
+  else: "tab-item"
 
 proc getAvatarClass*(prefs: Prefs): string =
-  if prefs.squareAvatars:
-    "avatar"
-  else:
-    "avatar round"
+  if prefs.squareAvatars: "avatar"
+  else: "avatar round"

src/views/search.nim

@@ -63,12 +63,10 @@ proc renderSearchPanel*(query: Query): VNode =
     hiddenField("f", "tweets")
     genInput("q", "", query.text, "Enter search...", class="pref-inline")
     button(`type`="submit"): icon "search"
-    if isPanelOpen(query):
-      input(id="search-panel-toggle", `type`="checkbox", checked="")
-    else:
-      input(id="search-panel-toggle", `type`="checkbox")
-    label(`for`="search-panel-toggle"):
-      icon "down"
+
+    input(id="search-panel-toggle", `type`="checkbox", checked=isPanelOpen(query))
+    label(`for`="search-panel-toggle"): icon "down"
+
     tdiv(class="search-panel"):
       for f in @["filter", "exclude"]:
         span(class="search-title"): text capitalize(f)
@@ -88,7 +86,7 @@ proc renderSearchPanel*(query: Query): VNode =
             genDate("until", query.until)
         tdiv:
           span(class="search-title"): text "Near"
-          genInput("near", "", query.near, placeholder="Location...")
+          genInput("near", "", query.near, "Location...", autofocus=false)
 
 proc renderTweetSearch*(results: Result[Tweet]; prefs: Prefs; path: string;
                         pinned=none(Tweet)): VNode =

src/views/tweet.nim

@@ -7,11 +7,7 @@ import renderutils
 import ".."/[types, utils, formatters]
 import general
 
-proc getSmallPic(url: string): string =
-  result = url
-  if "?" notin url and not url.endsWith("placeholder.png"):
-    result &= "?name=small"
-  result = getPicUrl(result)
+const doctype = "<!DOCTYPE html>\n"
 
 proc renderMiniAvatar(user: User; prefs: Prefs): VNode =
   let url = getPicUrl(user.getUserPic("_mini"))
@@ -57,9 +53,8 @@ proc renderAlbum(tweet: Tweet): VNode =
           tdiv(class="attachment image"):
             let
               named = "name=" in photo
-              orig = photo
-              small = if named: photo else: photo & "?name=small"
-            a(href=getOrigPicUrl(orig), class="still-image", target="_blank"):
+              small = if named: photo else: photo & smallWebp
+            a(href=getOrigPicUrl(photo), class="still-image", target="_blank"):
               genImg(small)
 
 proc isPlaybackEnabled(prefs: Prefs; playbackType: VideoType): bool =
@@ -106,12 +101,10 @@ proc renderVideo*(video: Video; prefs: Prefs; path: string): VNode =
                      else: vidUrl
           case playbackType
           of mp4:
-            if prefs.muteVideos:
-              video(src=source, poster=thumb, controls="", preload="none", muted=""):
-            else:
-              video(src=source, poster=thumb, controls="", preload="none"):
+            video(poster=thumb, controls="", muted=prefs.muteVideos):
+              source(src=source, `type`="video/mp4")
           of m3u8, vmap:
-            video(poster=thumb, data-url=source, data-autoload="false")
+            video(poster=thumb, data-url=source, data-autoload="false", muted=prefs.muteVideos)
             verbatim "<div class=\"video-overlay\" onclick=\"playVideo(this)\">"
             tdiv(class="overlay-circle"): span(class="overlay-triangle")
             verbatim "</div>"
@@ -132,12 +125,9 @@ proc renderGif(gif: Gif; prefs: Prefs): VNode =
   buildHtml(tdiv(class="attachments media-gif")):
     tdiv(class="gallery-gif", style={maxHeight: "unset"}):
       tdiv(class="attachment"):
-        let thumb = getSmallPic(gif.thumb)
-        let url = getPicUrl(gif.url)
-        if prefs.autoplayGifs:
-          video(src=url, class="gif", poster=thumb, controls="", muted="", loop="", playsinline="", autoplay="")
-        else:
-          video(src=url, class="gif", poster=thumb, controls="", muted="", loop="", playsinline="")
+        video(class="gif", poster=getSmallPic(gif.thumb), autoplay=prefs.autoplayGifs,
+              controls="", muted="", loop=""):
+          source(src=getPicUrl(gif.url), `type`="video/mp4")
 
 proc renderPoll(poll: Poll): VNode =
   buildHtml(tdiv(class="poll")):
@@ -331,7 +321,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
       if tweet.attribution.isSome:
         renderAttribution(tweet.attribution.get(), prefs)
 
-      if tweet.card.isSome:
+      if tweet.card.isSome and tweet.card.get().kind != hidden:
         renderCard(tweet.card.get(), prefs, path)
 
       if tweet.photos.len > 0:
@@ -350,7 +340,7 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         renderQuote(tweet.quote.get(), prefs, path)
 
       if mainTweet:
-        p(class="tweet-published"): text &"{getTime(tweet)} · {tweet.source}"
+        p(class="tweet-published"): text &"{getTime(tweet)}"
 
       if tweet.mediaTags.len > 0:
         renderMediaTags(tweet.mediaTags)
@@ -362,7 +352,12 @@ proc renderTweet*(tweet: Tweet; prefs: Prefs; path: string; class=""; index=0;
         a(class="show-thread", href=("/i/status/" & $tweet.threadId)):
           text "Show this thread"
 
-proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): VNode =
-  buildHtml(tdiv(class="tweet-embed")):
+proc renderTweetEmbed*(tweet: Tweet; path: string; prefs: Prefs; cfg: Config; req: Request): string =
+  let node = buildHtml(html(lang="en")):
     renderHead(prefs, cfg, req)
-    renderTweet(tweet, prefs, path, mainTweet=true)
+
+    body:
+      tdiv(class="tweet-embed"):
+        renderTweet(tweet, prefs, path, mainTweet=true)
+
+  result = doctype & $node

tests/requirements.txt

@@ -0,0 +1 @@
+seleniumbase

tests/test_card.py

@@ -3,11 +3,6 @@ from parameterized import parameterized
 
 
 card = [
-    ['Thom_Wolf/status/1122466524860702729',
-     'facebookresearch/fairseq',
-     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
-     'github.com', True],
-
     ['nim_lang/status/1136652293510717440',
      'Version 0.20.0 released',
      'We are very proud to announce Nim version 0.20. This is a massive release, both literally and figuratively. It contains more than 1,000 commits and it marks our release candidate for version 1.0!',
@@ -25,6 +20,11 @@ card = [
 ]
 
 no_thumb = [
+    ['Thom_Wolf/status/1122466524860702729',
+     'facebookresearch/fairseq',
+     'Facebook AI Research Sequence-to-Sequence Toolkit written in Python. - GitHub - facebookresearch/fairseq: Facebook AI Research Sequence-to-Sequence Toolkit written in Python.',
+     'github.com'],
+
     ['brent_p/status/1088857328680488961',
      'Hts Nim Sugar',
      'hts-nim is a library that allows one to use htslib via the nim programming language. Nim is a garbage-collected language that compiles to C and often has similar performance. I have become very...',
@@ -35,14 +35,9 @@ no_thumb = [
      'A sample of a Qt app written using mostly nim. Contribute to sinkingsugar/nimqt-example development by creating an account on GitHub.',
      'github.com'],
 
-    ['mobile_test/status/490378953744318464',
-     'Nantasket Beach',
-     'Explore this photo titled Nantasket Beach by Ben Sandofsky (@sandofsky) on 500px',
-     '500px.com'],
-
     ['nim_lang/status/1082989146040340480',
      'Nim in 2018: A short recap',
-     'Posted in r/programming by u/miran1',
+     'Posted by u/miran1 - 36 votes and 46 comments',
      'reddit.com']
 ]
 
@@ -76,7 +71,7 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('_img', self.get_image_url(c.image + ' img'))
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
         if len(description) > 0:
             self.assert_text(description, c.description)
         if large:
@@ -99,7 +94,7 @@ class CardTest(BaseTestCase):
         c = Card(Conversation.main + " ")
         self.assert_text(title, c.title)
         self.assert_text(destination, c.destination)
-        self.assertIn('_img', self.get_image_url(c.image + ' img'))
+        self.assertIn('/pic/', self.get_image_url(c.image + ' img'))
         self.assert_element_visible('.card-overlay')
         if len(description) > 0:
             self.assert_text(description, c.description)

tests/test_profile.py

@@ -17,11 +17,6 @@ protected = [
 
 invalid = [['thisprofiledoesntexist'], ['%']]
 
-banner_color = [
-    ['nim_lang', '22, 25, 32'],
-    ['rustlang', '35, 31, 32']
-]
-
 banner_image = [
     ['mobile_test', 'profile_banners%2F82135242%2F1384108037%2F1500x500']
 ]
@@ -74,12 +69,6 @@ class ProfileTest(BaseTestCase):
         self.open_nitter('user')
         self.assert_text('User "user" has been suspended')
 
-    @parameterized.expand(banner_color)
-    def test_banner_color(self, username, color):
-        self.open_nitter(username)
-        banner = self.find_element(Profile.banner + ' a')
-        self.assertIn(color, banner.value_of_css_property('background-color'))
-
     @parameterized.expand(banner_image)
     def test_banner_image(self, username, url):
         self.open_nitter(username)

tests/test_quote.py

@@ -3,7 +3,7 @@ from parameterized import parameterized
 
 text = [
     ['elonmusk/status/1138136540096319488',
-     'Trev Page', '@Model3Owners',
+     'TREV PAGE', '@Model3Owners',
      """As of March 58.4% of new car sales in Norway are electric. 
 
 What are we doing wrong? reuters.com/article/us-norwa…"""],