spark_on_k8s.utils.env

Implementation reference. These symbols describe how the project works; their presence here does not make them supported application imports. Use the documented entry points and support policy.

Parse SPARK_ON_K8S_* environment variables with errors that name the variable.

ConfigurationError

Bases: ValueError

An environment variable holds a value that cannot be used.

Source code in spark_on_k8s/utils/env.py

class ConfigurationError(ValueError):
    """An environment variable holds a value that cannot be used."""

env_client_config(name, configuration_type)

A Kubernetes client configuration from a JSON object of its constructor arguments.

Source code in spark_on_k8s/utils/env.py

def env_client_config(name: str, configuration_type: type) -> Any:
    """A Kubernetes client configuration from a JSON object of its constructor arguments."""
    if not getenv(name):
        return None
    try:
        return configuration_type(**env_json(name, {}, dict))
    except TypeError:
        raise ConfigurationError(
            f"{name} names an argument the Kubernetes client configuration does not accept"
        ) from None

env_json(name, default, kind)

A JSON object or array. The value is never echoed: some of these variables hold secrets.

Source code in spark_on_k8s/utils/env.py

def env_json(name: str, default: Any, kind: type) -> Any:
    """A JSON object or array. The value is never echoed: some of these variables hold secrets."""
    raw = getenv(name)
    if raw is None:
        return default
    expected = "object" if kind is dict else "array"
    try:
        value = json.loads(raw)
    except ValueError:
        raise ConfigurationError(f"{name} must be a JSON {expected}") from None
    if not isinstance(value, kind):
        raise ConfigurationError(f"{name} must be a JSON {expected}")
    return value

env_optional_int(name)

An integer, or None when the variable is unset or empty.

Source code in spark_on_k8s/utils/env.py

def env_optional_int(name: str) -> int | None:
    """An integer, or None when the variable is unset or empty."""
    return env_int(name, None) if getenv(name) else None