Use dynamic rate limits from API responses

src/apiutils.nim

@@ -7,6 +7,7 @@ import experimental/types/common
 const
   rlRemaining = "x-rate-limit-remaining"
   rlReset = "x-rate-limit-reset"
+  rlLimit = "x-rate-limit-limit"
   errorsToSkip = {doesntExist, tweetNotFound, timeout, unauthorized, badRequest}
 
 var pool: HttpPool
@@ -83,7 +84,8 @@ template fetchImpl(result, fetchBody) {.dirty.} =
       let
         remaining = parseInt(resp.headers[rlRemaining])
         reset = parseInt(resp.headers[rlReset])
-      session.setRateLimit(api, remaining, reset)
+        limit = parseInt(resp.headers[rlLimit])
+      session.setRateLimit(api, remaining, reset, limit)
 
     if result.len > 0:
       if resp.headers.getOrDefault("content-encoding") == "gzip":

src/auth.nim

@@ -57,7 +57,8 @@ proc getSessionPoolHealth*(): JsonNode =
     for api in session.apis.keys:
       let
         apiStatus = session.apis[api]
-        reqs = apiMaxReqs[api] - apiStatus.remaining
+        limit = if apiStatus.limit > 0: apiStatus.limit else: apiMaxReqs.getOrDefault(api, 0)
+        reqs = limit - apiStatus.remaining
 
       # no requests made with this session and endpoint since the limit reset
       if apiStatus.reset < now:
@@ -172,17 +173,17 @@ proc setLimited*(session: Session; api: Api) =
   session.limitedAt = epochTime().int
   log "rate limited by api: ", api, ", reqs left: ", session.apis[api].remaining, ", id: ", session.id
 
-proc setRateLimit*(session: Session; api: Api; remaining, reset: int) =
+proc setRateLimit*(session: Session; api: Api; remaining, reset, limit: int) =
   # avoid undefined behavior in race conditions
   if api in session.apis:
-    let limit = session.apis[api]
-    if limit.reset >= reset and limit.remaining < remaining:
+    let rateLimit = session.apis[api]
+    if rateLimit.reset >= reset and rateLimit.remaining < remaining:
       return
-    if limit.reset == reset and limit.remaining >= remaining:
+    if rateLimit.reset == reset and rateLimit.remaining >= remaining:
       session.apis[api].remaining = remaining
       return
 
-  session.apis[api] = RateLimit(remaining: remaining, reset: reset)
+  session.apis[api] = RateLimit(limit: limit, remaining: remaining, reset: reset)
 
 proc initSessionPool*(cfg: Config; path: string) =
   enableLogging = cfg.enableDebug

src/types.nim

@@ -28,6 +28,7 @@ type
     userMedia
 
   RateLimit* = object
+    limit*: int
     remaining*: int
     reset*: int