set the encoding consistently to be 'utf-8'

Author: BurnzZ

scrapy_zyte_api/handler.py

@@ -32,7 +32,6 @@ def __init__(
         self._stats = crawler.stats
         self._job_id = crawler.settings.get("JOB")
         self._session = create_session()
-        self._encoding = "utf-8"
 
     @classmethod
     def from_crawler(cls, crawler):
@@ -89,9 +88,7 @@ async def _download_request(
         if api_response.get("browserHtml"):
             # Using TextResponse because browserHtml always returns a browser-rendered page
             # even when requesting files (like images)
-            return ZyteAPITextResponse.from_api_response(
-                api_response, request=request, encoding=self._encoding
-            )
+            return ZyteAPITextResponse.from_api_response(api_response, request=request)
         else:
             return ZyteAPIResponse.from_api_response(api_response, request=request)
 

scrapy_zyte_api/responses.py

@@ -23,14 +23,11 @@ def _prepare_headers(init_headers: Optional[List[Dict[str, str]]]):
 
 class ZyteAPITextResponse(ZyteAPIMixin, TextResponse):
     @classmethod
-    def from_api_response(
-        cls, api_response: Dict, *, request: Request = None, encoding: str = "utf-8"
-    ):
+    def from_api_response(cls, api_response: Dict, *, request: Request = None):
         return cls(
             url=api_response["url"],
             status=200,
-            body=api_response["browserHtml"].encode(encoding),
-            encoding=encoding,
+            body=api_response["browserHtml"].encode("utf-8"),
             request=request,
             flags=["zyte-api"],
             headers=cls._prepare_headers(api_response.get("httpResponseHeaders")),