Fix entity expansion crash, simplify URL handling

Fixes #1409

src/parser.nim

@@ -241,11 +241,6 @@ proc parseLegacyMediaEntities(js: JsonNode; result: var Tweet) =
         ))
       else: discard
 
-      with url, m{"url"}:
-        if result.text.endsWith(url.getStr):
-          result.text.removeSuffix(url.getStr)
-          result.text = result.text.strip()
-
 proc parseMediaEntities(js: JsonNode; result: var Tweet) =
   with mediaEntities, js{"media_entities"}:
     var parsedMedia: MediaEntities
@@ -286,23 +281,9 @@ proc parseMediaEntities(js: JsonNode; result: var Tweet) =
           ))
         else: discard
 
-      if "expanded_url" in mediaEntity:
-        let expandedUrl = js.getExpandedUrl
-        if result.text.endsWith(expandedUrl):
-          result.text.removeSuffix(expandedUrl)
-          result.text = result.text.strip()
-
     if mediaEntities.len > 0 and parsedMedia.len == mediaEntities.len:
       result.media = parsedMedia
 
-  # Remove media URLs from text
-  with mediaList, js{"legacy", "entities", "media"}:
-    for url in mediaList:
-      let expandedUrl = url.getExpandedUrl
-      if result.text.endsWith(expandedUrl):
-        result.text.removeSuffix(expandedUrl)
-        result.text = result.text.strip()
-
 proc parsePromoVideo(js: JsonNode): Video =
   result = Video(
     thumb: js{"player_image_large"}.getImageVal,
@@ -469,8 +450,8 @@ proc parseTweet(js: JsonNode; jsCard: JsonNode = newJNull();
     elif name.len > 0 and jsCard{"binding_values"}.notNull:
       result.card = some parseCard(jsCard, js{"entities", "urls"})
 
-  parseLegacyMediaEntities(js, result)
   result.expandTweetEntities(js)
+  parseLegacyMediaEntities(js, result)
 
   with jsWithheld, js{"withheld_in_countries"}:
     let withheldInCountries: seq[string] =

src/parserutils.nim

@@ -319,7 +319,6 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
     textSlice = textRange{0}.getInt .. textRange{1}.getInt
     hasQuote = js{"is_quote_status"}.getBool
     hasJobCard = tweet.card.isSome and get(tweet.card).kind == jobDetails
-    hasAttribution = tweet.attribution.isSome
 
   var replyTo = ""
   if tweet.replyId != 0:
@@ -328,7 +327,7 @@ proc expandTweetEntities*(tweet: Tweet; js: JsonNode) =
       tweet.reply.add replyTo
 
   tweet.expandTextEntities(entities, tweet.text, textSlice, replyTo,
-                           hasQuote or hasJobCard or hasAttribution)
+                           hasQuote or hasJobCard)
 
 proc expandTextEntitiesV2(tweet: Tweet; js: JsonNode; text: string; textSlice: Slice[int];
                           hasRedundantLink=false) =