Track rate limits, reset after 24 hours

src/apiutils.nim

@@ -1,5 +1,5 @@
 # SPDX-License-Identifier: AGPL-3.0-only
-import httpclient, asyncdispatch, options, strutils, uri, times, math
+import httpclient, asyncdispatch, options, strutils, uri, times, math, tables
 import jsony, packedjson, zippy, oauth1
 import types, tokens, consts, parserutils, http_pool
 import experimental/types/common
@@ -129,6 +129,16 @@ proc fetch*(url: Uri; api: Api): Future[JsonNode] {.async.} =
       release(account, invalid=true)
       raise rateLimitError()
 
+    if body.startsWith("{\"errors"):
+      let errors = body.fromJson(Errors)
+      if errors in {invalidToken, badToken}:
+        echo "fetch error: ", errors
+        release(account, invalid=true)
+        raise rateLimitError()
+      elif errors in {rateLimited}:
+        account.apis[api].limited = true
+        echo "rate limited, api: ", $api, ", reqs left: ", account.apis[api].remaining, ", id: ", account.id
+
 proc fetchRaw*(url: Uri; api: Api): Future[string] {.async.} =
   fetchImpl result:
     if not (result.startsWith('{') or result.startsWith('[')):

src/tokens.nim

@@ -3,7 +3,9 @@ import asyncdispatch, times, json, random, strutils, tables
 import types
 
 # max requests at a time per account to avoid race conditions
-const maxConcurrentReqs = 5
+const
+  maxConcurrentReqs = 5
+  dayInSeconds = 24 * 60 * 60
 
 var
   accountPool: seq[GuestAccount]
@@ -19,7 +21,7 @@ proc getPoolJson*(): JsonNode =
     totalPending = 0
     reqsPerApi: Table[string, int]
 
-  let now = epochTime()
+  let now = epochTime().int
 
   for account in accountPool:
     totalPending.inc(account.pending)
@@ -29,10 +31,17 @@ proc getPoolJson*(): JsonNode =
     }
 
     for api in account.apis.keys:
-      if (now.int - account.apis[api].reset) / 60 > 15:
-        continue
+      let obj = %*{}
+      if account.apis[api].limited:
+        obj["limited"] = %true
+
+      if account.apis[api].reset > now.int:
+        obj["remaining"] = %account.apis[api].remaining
+
+      list[account.id]["apis"][$api] = obj
 
-      list[account.id]["apis"][$api] = %account.apis[api].remaining
+      if "remaining" notin obj:
+        continue
 
       let
         maxReqs =
@@ -65,7 +74,12 @@ proc isLimited(account: GuestAccount; api: Api): bool =
 
   if api in account.apis:
     let limit = account.apis[api]
-    return (limit.remaining <= 10 and limit.reset > epochTime().int)
+
+    if limit.limited and (epochTime().int - limit.limitedAt) > dayInSeconds:
+      account.apis[api].limited = false
+      echo "account limit reset, api: ", api, ", id: ", account.id
+
+    return limit.limited or (limit.remaining <= 10 and limit.reset > epochTime().int)
   else:
     return false
 

src/types.nim

@@ -32,6 +32,8 @@ type
   RateLimit* = object
     remaining*: int
     reset*: int
+    limited*: bool
+    limitedAt*: int
 
   GuestAccount* = ref object
     id*: string