Refactor fetch helper, fix list pagination

src/apiutils.nim

@@ -1,6 +1,8 @@
 import httpclient, asyncdispatch, options, times, strutils, uri
 import packedjson
-import types, agents, tokens, consts, parserutils
+import types, tokens, consts, parserutils
+
+const rl = "x-rate-limit-"
 
 proc genParams*(pars: openarray[(string, string)] = @[];
                 cursor=""): seq[(string, string)] =
@@ -10,55 +12,43 @@ proc genParams*(pars: openarray[(string, string)] = @[];
   if cursor.len > 0:
     result &= ("cursor", cursor)
 
-proc genHeaders*(token: Token): HttpHeaders =
+proc genHeaders*(token: Token = nil): HttpHeaders =
   result = newHttpHeaders({
-    "DNT": "1",
     "authorization": auth,
     "content-type": "application/json",
-    "user-agent": getAgent(),
     "x-guest-token": if token == nil: "" else: token.tok,
     "x-twitter-active-user": "yes",
     "authority": "api.twitter.com",
     "accept-language": "en-US,en;q=0.9",
     "accept": "*/*",
+    "DNT": "1"
   })
 
-proc fetch*(url: Uri; retried=false; oldApi=false): Future[JsonNode] {.async.} =
+proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
   var
-    token = await getToken()
-    keepToken = true
-    proxy: Proxy = when defined(proxy): newProxy(prox) else: nil
-    client = newAsyncHttpClient(proxy=proxy, headers=genHeaders(token))
+    token = if oldApi: nil else: await getToken()
+    client = newAsyncHttpClient(headers=genHeaders(token))
 
   try:
     let
       resp = await client.get($url)
       body = await resp.body
 
-    const rl = "x-rate-limit-"
+    if not body.startsWith('{'):
+      echo resp.status, ": ", body
+      result = newJNull()
+    else:
+      result = parseJson(body)
+
     if not oldApi and resp.headers.hasKey(rl & "limit"):
       token.remaining = parseInt(resp.headers[rl & "remaining"])
       token.reset = fromUnix(parseInt(resp.headers[rl & "reset"]))
 
-    if resp.status != $Http200:
-      if "Bad guest token" in body:
-        keepToken = false
-        return newJNull()
-      elif not body.startsWith('{'):
-        echo resp.status, " ", body
-        return newJNull()
-
-    result = parseJson(body)
-
-    if result{"errors"}.notNull and result.getError == forbidden:
-      keepToken = false
-      echo "bad token"
-  except:
+    if result.getError notin {invalidToken, forbidden, badToken}:
+      token.release()
+  except Exception:
     echo "error: ", url
     result = newJNull()
   finally:
-    if keepToken:
-      token.release()
-
     try: client.close()
     except: discard

src/routes/list.nim

@@ -35,7 +35,7 @@ proc createListRouter*(cfg: Config) =
       let
         prefs = cookiePrefs()
         list = await getCachedList(@"name", @"list")
-        members = await getListMembers(list)
+        members = await getListMembers(list, getCursor())
       respList(list, members, renderTimelineUsers(members, prefs, request.path))
 
     get "/i/lists/@id/?":

src/types.nim

@@ -9,7 +9,6 @@ type
     remaining*: int
     reset*: Time
     init*: Time
-    # agent*: string
 
   Error* = enum
     null = 0
@@ -18,9 +17,11 @@ type
     doesntExist = 34
     userNotFound = 50
     suspended = 63
+    rateLimited = 88
     invalidToken = 89
     listIdOrSlug = 112
     forbidden = 200
+    badToken = 239
     noCsrf = 353
 
   Profile* = object