Improve environment variable handling. Allow modules to declare environment variables as well, and create an environment variable index.

Author: felixfontein

changelogs/fragments/73-env-vars.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - "Use correct markup (``envvar`` role) for environment variables. Allow modules to declare environment variables. Compile an index of all environment variables used by plugins and modules. (https://github.com/ansible-community/antsibull-docs/pull/73)."

src/antsibull_docs/cli/doc_commands/stable.py

@@ -37,6 +37,7 @@
     load_all_collection_routing,
     remove_redirect_duplicates,
 )
+from ...env_variables import load_ansible_config, collect_referenced_environment_variables
 from ...schemas.docs import DOCS_SCHEMAS
 from ...utils.collection_name_transformer import CollectionNameTransformer
 from ...write_docs import (
@@ -47,6 +48,7 @@
     output_indexes,
     output_plugin_indexes,
     output_extra_docs,
+    output_environment_variables,
 )
 
 if t.TYPE_CHECKING:
@@ -337,12 +339,16 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
     app_ctx = app_context.app_ctx.get()
 
     # Get the info from the plugins
-    plugin_info, collection_metadata = asyncio_run(get_ansible_plugin_info(
+    plugin_info, full_collection_metadata = asyncio_run(get_ansible_plugin_info(
         venv, collection_dir, collection_names=collection_names))
     flog.notice('Finished parsing info from plugins and collections')
     # flog.fields(plugin_info=plugin_info).debug('Plugin data')
     # flog.fields(
-    #     collection_metadata=collection_metadata).debug('Collection metadata')
+    #     collection_metadata=full_collection_metadata).debug('Collection metadata')
+
+    collection_metadata = dict(full_collection_metadata)
+    if collection_names is not None and 'ansible.builtin' not in collection_names:
+        del collection_metadata['ansible.builtin']
 
     # Load collection routing information
     collection_routing = asyncio_run(load_all_collection_routing(collection_metadata))
@@ -363,7 +369,7 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
         {name: data.path for name, data in collection_metadata.items()}))
     flog.debug('Finished getting collection extra docs data')
 
-    # Load collection extra docs data
+    # Load collection links data
     link_data = asyncio_run(load_collections_links(
         {name: data.path for name, data in collection_metadata.items()}))
     flog.debug('Finished getting collection link data')
@@ -384,6 +390,10 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
                     print(f"{plugin_name} {plugin_type}: {textwrap.indent(error, '    ').lstrip()}")
         return 1
 
+    # Handle environment variables
+    ansible_config = load_ansible_config(full_collection_metadata['ansible.builtin'])
+    referenced_env_vars = collect_referenced_environment_variables(new_plugin_info, ansible_config)
+
     collection_namespaces = get_collection_namespaces(collection_to_plugin_info.keys())
 
     collection_url = CollectionNameTransformer(
@@ -444,7 +454,14 @@ def generate_docs_for_all_collections(venv: t.Union[VenvRunner, FakeVenvRunner],
 
     asyncio_run(output_extra_docs(dest_dir, extra_docs_data,
                                   squash_hierarchy=squash_hierarchy))
-    flog.debug('Finished writing extra extra docs docs')
+    flog.debug('Finished writing extra docs')
+
+    if referenced_env_vars:
+        asyncio_run(output_environment_variables(dest_dir, referenced_env_vars,
+                                                 squash_hierarchy=squash_hierarchy))
+        flog.debug('Finished writing environment variables')
+    else:
+        flog.debug('Skipping environment variables (as there are none)')
     return 0
 
 

src/antsibull_docs/data/docsite/list_of_env_variables.rst.j2

@@ -0,0 +1,39 @@
+{#
+  Copyright (c) Ansible Project
+  GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+  SPDX-License-Identifier: GPL-3.0-or-later
+#}
+
+:orphan:
+
+.. _list_of_collection_env_vars:
+
+Index of all Collection Environment Variables
+=============================================
+
+The following index documents all environment variables declared by plugins and modules in collections.
+Environment variables used by the ansible-core configuation are documented in :ref:`ansible_configuration_settings`.
+{# TODO: use label `ansible_configuration_env_vars` once the ansible-core PR is merged #}
+
+{% for _, env_var in env_variables | dictsort %}
+.. envvar:: @{ env_var.name }@
+
+{%   for paragraph in env_var.description or [] %}
+    @{ paragraph | replace('\n', '\n    ') | rst_ify | indent(4) }@
+
+{%   endfor %}
+    *Used by:*
+{% set plugins_ = [] %}
+{%   for plugin_type, plugins in env_var.plugins.items() %}
+{%     for plugin_name in plugins %}
+{%       set _ = plugins_.append((plugin_name, plugin_type)) %}
+{%     endfor %}
+{%   endfor %}
+{%   for plugin_name, plugin_type in plugins_ | unique | sort %}
+    :ref:`@{ plugin_name | rst_escape }@ {% if plugin_type == 'module' %}module{% else %}@{ plugin_type }@ plugin{% endif %} <ansible_collections.@{ plugin_name }@_@{ plugin_type }@>`
+{%-    if not loop.last -%}
+,
+{%     endif -%}
+{%-  endfor %}
+
+{% endfor %}

src/antsibull_docs/data/docsite/macros/parameters.rst.j2

@@ -99,12 +99,16 @@
       :ansible-option-default-bold:`Default:` :ansible-option-default:`@{ value['default'] | antsibull_to_json | rst_escape(escape_ending_whitespace=true) | indent(6, blank=true) }@`
 {%   endif %}
 {#   Configuration #}
-{%   if plugin_type != 'module' and plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
+{%   if plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
 
       .. rst-class:: ansible-option-line
 
       :ansible-option-configuration:`Configuration:`
 
+{%     if plugin_type == 'module' and value['env'] %}
+      The below environment variable{% if value['env'] | length > 1 %}s{% endif %} will be used on the host that executes this module.
+
+{%     endif %}
 {%     if value['ini'] %}
       - INI {% if value['ini'] | length == 1 %}entry{% else %}entries{% endif %}:
 {%       for ini in value['ini'] %}
@@ -125,30 +129,30 @@
 {%       endfor %}
 {%     endif %}
 {%     for env in value['env'] %}
-      - Environment variable: @{ env['name'] | rst_escape }@
+      - Environment variable: :envvar:`@{ env['name'] | rst_escape(escape_ending_whitespace=true) }@`
 {%       if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
 
         :ansible-option-versionadded:`added in @{ version_added_rst(env['version_added'], env['version_added_collection'] or collection) }@`
 {%       endif %}
 @{       deprecates_rst(env['deprecated'], collection, 8) }@
 {%     endfor %}
-{%     for myvar in value['vars'] %}
+{%     for myvar in value['vars'] | default([]) %}
       - Variable: @{ myvar['name'] | rst_escape }@
 {%       if myvar['version_added'] is still_relevant(collection=myvar['version_added_collection'] or collection) %}
 
         :ansible-option-versionadded:`added in @{ version_added_rst(myvar['version_added'], myvar['version_added_collection'] or collection) }@`
 {%       endif %}
 @{       deprecates_rst(myvar['deprecated'], collection, 8) }@
 {%     endfor %}
-{%     for kw in value['keyword'] %}
+{%     for kw in value['keyword'] | default([]) %}
       - Keyword: @{ kw['name'] | rst_escape }@
 {%       if kw['version_added'] is still_relevant(collection=kw['version_added_collection'] or collection) %}
 
         :ansible-option-versionadded:`added in @{ version_added_rst(kw['version_added'], kw['version_added_collection'] or collection) }@`
 {%       endif %}
 @{       deprecates_rst(kw['deprecated'], collection, 8) }@
 {%     endfor %}
-{%     for mycli in value['cli'] %}
+{%     for mycli in value['cli'] | default([]) %}
       - CLI argument: @{ mycli['option'] | rst_escape }@
 {%       if mycli['version_added'] is still_relevant(collection=mycli['version_added_collection'] or collection) %}
 
@@ -228,7 +232,7 @@
       <p class="ansible-option-line"><span class="ansible-option-default-bold">Default:</span> <code class="ansible-value literal notranslate ansible-option-default">@{ value['default'] | antsibull_to_json | escape | indent(6, blank=true) }@</code></p>
 {%   endif %}
 {#   Configuration #}
-{%   if plugin_type != 'module' and plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
+{%   if plugin_type != 'role' and (value['ini'] or value['env'] or value['vars'] or value['cli']) %}
       <p class="ansible-option-line"><span class="ansible-option-configuration">Configuration:</span></p>
       <ul class="simple">
 {%     if value['ini'] %}
@@ -250,14 +254,14 @@
 {%     endif %}
 {%     for env in value['env'] %}
       <li>
-        <p>Environment variable: @{ env['name'] | escape }@</p>
+        <p>Environment variable: <code class="xref std std-envvar literal notranslate">@{ env['name'] | escape }@</code></p>
 {%       if env['version_added'] is still_relevant(collection=env['version_added_collection'] or collection) %}
         <p><span class="ansible-option-versionadded">added in @{ version_added_html(env['version_added'], env['version_added_collection'] or collection) }@</span></p>
 {%       endif %}
 @{       deprecates_html(env['deprecated'], collection) }@
       </li>
 {%     endfor %}
-{%     for myvar in value['vars'] %}
+{%     for myvar in value['vars'] | default([]) %}
       <li>
         <p>Variable: @{ myvar['name'] | escape }@</p>
 {%       if myvar['version_added'] is still_relevant(collection=myvar['version_added_collection'] or collection) %}
@@ -266,7 +270,7 @@
 @{       deprecates_html(myvar['deprecated'], collection) }@
       </li>
 {%     endfor %}
-{%     for kw in value['keyword'] %}
+{%     for kw in value['keyword'] | default([]) %}
       <li>
         <p>Keyword: @{ kw['name'] | escape }@</p>
 {%       if kw['version_added'] is still_relevant(collection=kw['version_added_collection'] or collection) %}
@@ -275,7 +279,7 @@
 @{       deprecates_html(kw['deprecated'], collection) }@
       </li>
 {%     endfor %}
-{%     for mycli in value['cli'] %}
+{%     for mycli in value['cli'] | default([]) %}
       <li>
         <p>CLI argument: @{ mycli['option'] | escape }@</p>
 {%       if mycli['version_added'] is still_relevant(collection=mycli['version_added_collection'] or collection) %}

src/antsibull_docs/env_variables.py

@@ -0,0 +1,124 @@
+# Author: Felix Fontein <[email protected]>
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or
+# https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+# SPDX-FileCopyrightText: 2022, Ansible Project
+"""Environment variable handling."""
+
+import os
+import os.path
+import typing as t
+
+from antsibull_core import yaml
+
+from .docs_parsing import AnsibleCollectionMetadata
+
+
+class EnvironmentVariableInfo:
+    name: str
+    description: t.Optional[t.List[str]]
+    plugins: t.Dict[str, t.List[str]]  # maps plugin_type to lists of plugin FQCNs
+
+    def __init__(self,
+                 name: str,
+                 description: t.Optional[t.List[str]] = None,
+                 plugins: t.Optional[t.Dict[str, t.List[str]]] = None):
+        self.name = name
+        self.description = description
+        self.plugins = plugins or {}
+
+    def __repr__(self):
+        return f'E({self.name}, description={repr(self.description)}, plugins={self.plugins})'
+
+
+def load_ansible_config(ansible_builtin_metadata: AnsibleCollectionMetadata
+                        ) -> t.Mapping[str, t.Mapping[str, t.Any]]:
+    """
+    Load Ansible base configuration (``lib/ansible/config/base.yml``).
+
+    :arg ansible_builtin_metadata: Metadata for the ansible.builtin collection.
+    :returns: A Mapping of configuration options to information on these options.
+    """
+    return yaml.load_yaml_file(os.path.join(ansible_builtin_metadata.path, 'config', 'base.yml'))
+
+
+def _find_env_vars(options: t.Mapping[str, t.Mapping[str, t.Any]]
+                   ) -> t.Generator[t.Tuple[str, t.Optional[t.List[str]]], None, None]:
+    for _, option_data in options.items():
+        if isinstance(option_data.get('env'), list):
+            description = option_data.get('description')
+            if isinstance(description, str):
+                description = [description]
+            if isinstance(description, list):
+                description = [str(desc) for desc in description]
+            else:
+                description = None
+            for env_var in option_data['env']:
+                if isinstance(env_var.get('name'), str):
+                    yield (env_var['name'], description)
+        if isinstance(option_data.get('suboptions'), dict):
+            yield from _find_env_vars(option_data['suboptions'])
+
+
+def _collect_env_vars_and_descriptions(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
+                                       core_envs: t.Set[str],
+                                       ) -> t.Tuple[t.Mapping[str, EnvironmentVariableInfo],
+                                                    t.Mapping[str, t.List[t.List[str]]]]:
+    other_variables: t.Dict[str, EnvironmentVariableInfo] = {}
+    other_variable_description: t.Dict[str, t.List[t.List[str]]] = {}
+    for plugin_type, plugins in plugin_info.items():
+        for plugin_name, plugin_data in plugins.items():
+            plugin_options: t.Mapping[str, t.Mapping[str, t.Any]] = (
+                (plugin_data.get('doc') or {}).get('options') or {}
+            )
+            for env_var, env_var_description in _find_env_vars(plugin_options):
+                if env_var in core_envs:
+                    continue
+                if env_var not in other_variables:
+                    other_variables[env_var] = EnvironmentVariableInfo(env_var)
+                    other_variable_description[env_var] = []
+                if plugin_type not in other_variables[env_var].plugins:
+                    other_variables[env_var].plugins[plugin_type] = []
+                other_variables[env_var].plugins[plugin_type].append(plugin_name)
+                if env_var_description is not None:
+                    other_variable_description[env_var].append(env_var_description)
+    return other_variables, other_variable_description
+
+
+def _augment_env_var_descriptions(other_variables: t.Mapping[str, EnvironmentVariableInfo],
+                                  other_variable_description: t.Mapping[str, t.List[t.List[str]]],
+                                  ) -> None:
+    for variable, variable_info in other_variables.items():
+        if other_variable_description[variable]:
+            value: t.Optional[t.List[str]] = other_variable_description[variable][0]
+            for other_value in other_variable_description[variable]:
+                if value != other_value:
+                    value = [
+                        'See the documentations for the options where this environment variable'
+                        ' is used.'
+                    ]
+                    break
+            variable_info.description = value
+
+
+def collect_referenced_environment_variables(plugin_info: t.Mapping[str, t.Mapping[str, t.Any]],
+                                             ansible_config: t.Mapping[str, t.Mapping[str, t.Any]],
+                                             ) -> t.Mapping[str, EnvironmentVariableInfo]:
+    """
+    Collect referenced environment variables that are not defined in the ansible-core
+    configuration.
+
+    :arg plugin_info: Mapping of plugin type to a mapping of plugin name to plugin record.
+    :arg ansible_config: The Ansible base configuration (``lib/ansible/config/base.yml``).
+    :returns: A Mapping of environment variable name to an environment variable infomation object.
+    """
+    core_envs = {'ANSIBLE_CONFIG'}
+    for config in ansible_config.values():
+        if config.get('env'):
+            for env in config['env']:
+                core_envs.add(env['name'])
+
+    other_variables, other_variable_description = _collect_env_vars_and_descriptions(
+        plugin_info, core_envs)
+    _augment_env_var_descriptions(other_variables, other_variable_description)
+    return other_variables

src/antsibull_docs/schemas/docs/module.py

@@ -10,10 +10,11 @@
 import pydantic as p
 
 from .base import BaseModel, DocSchema, OptionsSchema
-from .plugin import PluginExamplesSchema, PluginMetadataSchema, PluginReturnSchema
+from .plugin import OptionEnvSchema, PluginExamplesSchema, PluginMetadataSchema, PluginReturnSchema
 
 
 class InnerModuleOptionsSchema(OptionsSchema):
+    env: t.List[OptionEnvSchema] = []
     suboptions: t.Dict[str, 'InnerModuleOptionsSchema'] = {}
 
     @p.root_validator(pre=True)
@@ -29,6 +30,7 @@ def allow_description_to_be_optional(cls, values):
 
 
 class ModuleOptionsSchema(OptionsSchema):
+    env: t.List[OptionEnvSchema] = []
     suboptions: t.Dict[str, 'InnerModuleOptionsSchema'] = {}