diff --git a/pipreqs/pipreqs.py b/pipreqs/pipreqs.py index dceda96..3f94a0c 100644 --- a/pipreqs/pipreqs.py +++ b/pipreqs/pipreqs.py @@ -38,6 +38,7 @@ | e.g. Flask --scan-notebooks Look for imports in jupyter notebook files. """ +import asyncio from contextlib import contextmanager import os import sys @@ -46,10 +47,9 @@ import ast import traceback from docopt import docopt -import requests +import httpx from yarg import json2package from yarg.exceptions import HTTPError - from pipreqs import __version__ REGEXP = [re.compile(r"^import (.+)$"), re.compile(r"^from ((?!\.+).*?) import (?:.*)$")] @@ -227,37 +227,50 @@ def output_requirements(imports, symbol): generate_requirements_file("-", imports, symbol) -def get_imports_info(imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): - result = [] +async def _get_response(client, url): + try: + response = await client.get(url) + if response.status_code == 200: + if hasattr(response.content, "decode"): + data = json2package(response.content.decode()) + else: + data = json2package(response.content) + + return data + elif response.status_code >= 300: + raise HTTPError(status_code=response.status_code, url=url) + except HTTPError as e: + logging.error( + 'Failed to get package information for "%s" from PyPI: %s', + url, + e, + ) + return None - for item in imports: - try: + +async def get_imports_info(imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): + requests = [] + + async with httpx.AsyncClient( + base_url=pypi_server, + headers={"User-Agent": "pipreqs/{}".format(__version__)}, + proxy=proxy, + timeout=60.0, + follow_redirects=True + ) as client: + for item in imports: logging.warning( 'Import named "%s" not found locally. ' "Trying to resolve it at the PyPI server.", item, ) - response = requests.get("{0}{1}/json".format(pypi_server, item), proxies=proxy) - if response.status_code == 200: - if hasattr(response.content, "decode"): - data = json2package(response.content.decode()) - else: - data = json2package(response.content) - elif response.status_code >= 300: - raise HTTPError(status_code=response.status_code, reason=response.reason) - except HTTPError: - logging.warning('Package "%s" does not exist or network problems', item) - continue - logging.warning( - 'Import named "%s" was resolved to "%s:%s" package (%s).\n' - "Please, verify manually the final list of requirements.txt " - "to avoid possible dependency confusions.", - item, - data.name, - data.latest_release_id, - data.pypi_url, - ) - result.append({"name": item, "version": data.latest_release_id}) - return result + requests.append(asyncio.create_task(_get_response(client, f"{item}/json"))) + + responses = await asyncio.gather(*requests) + + return [ + {"name": data.name, "version": data.latest_release_id} + for data in responses if data is not None + ] def get_locally_installed_packages(encoding="utf-8"): @@ -500,7 +513,7 @@ def handle_scan_noteboooks(): raise NbconvertNotInstalled() -def init(args): +async def init(args): global scan_noteboooks encoding = args.get("--encoding") extra_ignore_dirs = args.get("--ignore") @@ -572,7 +585,7 @@ def init(args): x.lower() not in [x["name"] for x in local] ] - imports = local + get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) + imports = local + await get_imports_info(difference, proxy=proxy, pypi_server=pypi_server) # sort imports based on lowercase name of package, similar to `pip freeze`. imports = sorted(imports, key=lambda x: x["name"].lower()) @@ -609,7 +622,7 @@ def main(): # pragma: no cover logging.basicConfig(level=log_level, format="%(levelname)s: %(message)s") try: - init(args) + asyncio.run(init(args)) except KeyboardInterrupt: sys.exit(0) diff --git a/poetry.lock b/poetry.lock index f6be611..e499575 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.2 and should not be changed by hand. [[package]] name = "alabaster" @@ -12,6 +12,29 @@ files = [ {file = "alabaster-0.7.16.tar.gz", hash = "sha256:75a8b99c28a5dad50dd7f8ccdd447a121ddb3892da9e53d1ca5cca3106d58d65"}, ] +[[package]] +name = "anyio" +version = "4.9.0" +description = "High level compatibility layer for multiple asynchronous event loop implementations" +optional = false +python-versions = ">=3.9" +groups = ["main"] +files = [ + {file = "anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c"}, + {file = "anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028"}, +] + +[package.dependencies] +exceptiongroup = {version = ">=1.0.2", markers = "python_version < \"3.11\""} +idna = ">=2.8" +sniffio = ">=1.1" +typing_extensions = {version = ">=4.5", markers = "python_version < \"3.13\""} + +[package.extras] +doc = ["Sphinx (>=8.2,<9.0)", "packaging", "sphinx-autodoc-typehints (>=1.2.0)", "sphinx_rtd_theme"] +test = ["anyio[trio]", "blockbuster (>=1.5.23)", "coverage[toml] (>=7)", "exceptiongroup (>=1.2.0)", "hypothesis (>=4.0)", "psutil (>=5.9)", "pytest (>=7.0)", "trustme", "truststore (>=0.9.1) ; python_version >= \"3.10\"", "uvloop (>=0.21) ; platform_python_implementation == \"CPython\" and platform_system != \"Windows\" and python_version < \"3.14\""] +trio = ["trio (>=0.26.1)"] + [[package]] name = "appnope" version = "0.1.4" @@ -506,6 +529,25 @@ files = [ {file = "docutils-0.21.2.tar.gz", hash = "sha256:3a6b18732edf182daa3cd12775bbb338cf5691468f91eeeb109deff6ebfa986f"}, ] +[[package]] +name = "exceptiongroup" +version = "1.3.0" +description = "Backport of PEP 654 (exception groups)" +optional = false +python-versions = ">=3.7" +groups = ["main"] +markers = "python_version < \"3.11\"" +files = [ + {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, + {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, +] + +[package.dependencies] +typing-extensions = {version = ">=4.6.0", markers = "python_version < \"3.13\""} + +[package.extras] +test = ["pytest (>=6)"] + [[package]] name = "executing" version = "2.2.0" @@ -570,6 +612,65 @@ mccabe = ">=0.7.0,<0.8.0" pycodestyle = ">=2.11.0,<2.12.0" pyflakes = ">=3.1.0,<3.2.0" +[[package]] +name = "h11" +version = "0.16.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86"}, + {file = "h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1"}, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +description = "A minimal low-level HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55"}, + {file = "httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8"}, +] + +[package.dependencies] +certifi = "*" +h11 = ">=0.16" + +[package.extras] +asyncio = ["anyio (>=4.0,<5.0)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +trio = ["trio (>=0.22.0,<1.0)"] + +[[package]] +name = "httpx" +version = "0.28.1" +description = "The next generation HTTP client." +optional = false +python-versions = ">=3.8" +groups = ["main"] +files = [ + {file = "httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad"}, + {file = "httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc"}, +] + +[package.dependencies] +anyio = "*" +certifi = "*" +httpcore = "==1.*" +idna = "*" + +[package.extras] +brotli = ["brotli ; platform_python_implementation == \"CPython\"", "brotlicffi ; platform_python_implementation != \"CPython\""] +cli = ["click (==8.*)", "pygments (==2.*)", "rich (>=10,<14)"] +http2 = ["h2 (>=3,<5)"] +socks = ["socksio (==1.*)"] +zstd = ["zstandard (>=0.18.0)"] + [[package]] name = "idna" version = "3.10" @@ -604,11 +705,11 @@ description = "Read metadata from Python packages" optional = false python-versions = ">=3.8" groups = ["main", "dev"] +markers = "python_version == \"3.9\"" files = [ {file = "importlib_metadata-8.5.0-py3-none-any.whl", hash = "sha256:45e54197d28b7a7f1559e60b95e7c567032b602131fbd588f1497f47880aa68b"}, {file = "importlib_metadata-8.5.0.tar.gz", hash = "sha256:71522656f0abace1d072b9e5481a48f07c138e00f079c38c8f883823f9c26bd7"}, ] -markers = {main = "python_version < \"3.10\"", dev = "python_version == \"3.9\""} [package.dependencies] zipp = ">=3.20" @@ -1534,6 +1635,18 @@ files = [ {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, ] +[[package]] +name = "sniffio" +version = "1.3.1" +description = "Sniff out which async library your code is running under" +optional = false +python-versions = ">=3.7" +groups = ["main"] +files = [ + {file = "sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2"}, + {file = "sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc"}, +] + [[package]] name = "snowballstemmer" version = "2.2.0" @@ -1940,11 +2053,11 @@ description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.8" groups = ["main", "dev"] +markers = "python_version == \"3.9\"" files = [ {file = "zipp-3.20.2-py3-none-any.whl", hash = "sha256:a817ac80d6cf4b23bf7f2828b7cabf326f15a001bea8b1f9b49631780ba28350"}, {file = "zipp-3.20.2.tar.gz", hash = "sha256:bc9eb26f4506fda01b81bcde0ca78103b6e62f991b381fec825435c836edbc29"}, ] -markers = {main = "python_version < \"3.10\"", dev = "python_version == \"3.9\""} [package.extras] check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] @@ -1959,5 +2072,5 @@ dev = ["coverage", "flake8", "sphinx", "tox"] [metadata] lock-version = "2.1" -python-versions = ">=3.8.1,<3.14" -content-hash = "a28090833ea5fbee5586a3e60380a739249c241e7dd159d62597bee3feee4957" +python-versions = ">=3.9, <3.14" +content-hash = "16016e0d7a3063f276125331ebc6e29954dae07fb89ede0af276584654baa669" diff --git a/pyproject.toml b/pyproject.toml index 9685b7d..ee3b6da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -31,6 +31,7 @@ dependencies = [ "docopt>=0.6.2", "nbconvert>=7.11.0", "ipython>=8.12.3", + "httpx (>=0.28.1,<0.29.0)", ] [project.optional-dependencies] dev = [