Use gzip for API calls to lower bandwidth and RAM

nitter.nimble

@@ -22,6 +22,7 @@ requires "supersnappy#2.1.1"
 requires "redpool#f880f49"
 requires "https://github.com/zedeus/redis#d0a0e6f"
 requires "https://github.com/disruptek/frosty#0.3.1"
+requires "zippy#0.7.3"
 
 
 # Tasks

src/apiutils.nim

@@ -1,5 +1,5 @@
 import httpclient, asyncdispatch, options, times, strutils, uri
-import packedjson
+import packedjson, zippy
 import types, tokens, consts, parserutils, http_pool
 
 const rl = "x-rate-limit-"
@@ -26,6 +26,7 @@ proc genHeaders*(token: Token = nil): HttpHeaders =
     "x-guest-token": if token == nil: "" else: token.tok,
     "x-twitter-active-user": "yes",
     "authority": "api.twitter.com",
+    "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.9",
     "accept": "*/*",
     "DNT": "1"
@@ -44,7 +45,7 @@ proc fetch*(url: Uri; oldApi=false): Future[JsonNode] {.async.} =
     var resp: AsyncResponse
     let body = pool.use(headers):
       resp = await c.get($url)
-      await resp.body
+      uncompress(await resp.body)
 
     if body.startsWith('{') or body.startsWith('['):
       result = parseJson(body)

src/http_pool.nim

@@ -7,7 +7,8 @@ type
 var maxConns {.threadvar.}: int
 
 let keepAlive* = newHttpHeaders({
-  "Connection": "Keep-Alive"
+  "connection": "Keep-Alive",
+  "accept-encoding": "gzip"
 })
 
 proc setMaxHttpConns*(n: int) =

src/tokens.nim

@@ -1,5 +1,6 @@
 import asyncdispatch, httpclient, times, sequtils, json, math, random
 import strutils, strformat
+import zippy
 import types, agents, consts, http_pool
 
 const
@@ -28,6 +29,7 @@ proc fetchToken(): Future[Token] {.async.} =
 
   let headers = newHttpHeaders({
     "accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
+    "accept-encoding": "gzip",
     "accept-language": "en-US,en;q=0.5",
     "connection": "keep-alive",
     "user-agent": getAgent(),
@@ -41,7 +43,7 @@ proc fetchToken(): Future[Token] {.async.} =
 
   try:
     resp = clientPool.use(headers): await c.postContent(activate)
-    tokNode = parseJson(resp)["guest_token"]
+    tokNode = parseJson(uncompress(resp))["guest_token"]
     tok = tokNode.getStr($(tokNode.getInt))
 
     let time = getTime()