Add include-optional directive.

CHANGES.md

@@ -4,6 +4,12 @@
 
 <!-- Add future changes here -->
 
+- Add `include-optional` for INI configuration files that may be absent, skipping
+  missing local files and HTTP 404 responses while preserving mandatory includes.
+  This allows projects to define an optional include for client-specific
+  customizations.
+- Fix HTTP error logging during INI inclusion and preserve the original exception.
+- Fix relative INI includes from URLs with a directory path.
 
 ## 5.4.1 (2026-08-04)
 

README.md

@@ -198,6 +198,32 @@ If an included file is an HTTP-URL, it is loaded from there.
 
 If the included file is a relative path, it is loaded relative to the parent's directory or URL.
 
+Missing files and HTTP errors stop configuration loading. Use `include-optional` for files that may be absent.
+
+Default: empty
+
+##### `include-optional`
+
+Optional includes allow projects to define a client-specific configuration file
+via a file name defined by a convention - e.g. to make customer- or
+developer-specific adaptions which should not be checked into a repository.
+
+Include one or more optional INI files, one per line:
+
+```ini
+[settings]
+include-optional =
+    mx-custom.ini
+```
+
+Missing local files and HTTP 404 responses are ignored. Other errors, such as HTTP 403/500,
+connection failures, or invalid INI content, still stop configuration loading.
+Mandatory `include` entries inside an existing optional file remain mandatory.
+
+Paths and URLs are resolved in the same way as `include`, and optional files may include other files.
+For each file, mandatory includes are read first, then optional includes in their listed order,
+then the file itself. Later settings override earlier settings, so the main file takes precedence.
+
 Default: empty
 
 ##### `directory`

src/mxdev/including.py

@@ -1,8 +1,10 @@
+from .logging import logger
 from configparser import ConfigParser
 from configparser import ExtendedInterpolation
 from pathlib import Path
 from urllib import parse
 from urllib import request
+from urllib.error import HTTPError
 
 import os
 import tempfile
@@ -12,14 +14,14 @@ def resolve_dependencies(
     file_or_url: str | Path,
     tmpdir: str,
     http_parent=None,
+    optional: bool = False,
 ) -> list[Path]:
     """Resolve dependencies of a file or url
 
-    The result is a list of Path objects, starting with the
-    given file_or_url and followed by all file_or_urls referenced from it.
+    Return included paths before their parent, so parent settings take precedence.
 
-    The file_or_url is assumed to be a ini file or url to such, with an option key "include"
-    under the "[settings]" section.
+    Follow "include" and "include-optional" under the "[settings]" section.
+    If optional, skip a missing file or HTTP 404 for this input only.
     """
     if isinstance(file_or_url, str):
         if http_parent:
@@ -29,40 +31,59 @@ def resolve_dependencies(
         # Windows drive letters are single characters, URL schemes are longer
         is_url = parsed.scheme and len(parsed.scheme) > 1
         if is_url:
-            with request.urlopen(str(file_or_url)) as fio:
-                tf = tempfile.NamedTemporaryFile(
-                    suffix=".ini",
-                    dir=str(tmpdir),
-                    delete=False,
-                )
-                tf.write(fio.read())
-                tf.flush()
-                file = Path(tf.name)
-            parts = list(parsed)
-            parts[2] = str(Path(parts[2]).parent)
-            http_parent = parse.urlunparse(parts)
+            try:
+                with request.urlopen(str(file_or_url)) as fio:
+                    with tempfile.NamedTemporaryFile(
+                        suffix=".ini",
+                        dir=str(tmpdir),
+                        delete=False,
+                    ) as tf:
+                        tf.write(fio.read())
+                        file = Path(tf.name)
+            except HTTPError as e:
+                if optional and e.code == 404:
+                    logger.info("Skipping missing optional include: %s", file_or_url)
+                    return []
+                logger.error("Error %s for URL: %s", e.code, e.url)
+                raise
+            http_parent = parse.urljoin(str(file_or_url), ".")
         else:
             file = Path(file_or_url)
     else:
         file = file_or_url
-    if not file.exists():
-        raise FileNotFoundError(file)
     cfg = ConfigParser()
-    cfg.read(file)
-    if not ("settings" in cfg and "include" in cfg["settings"]):
+    try:
+        with file.open() as fio:
+            cfg.read_file(fio)
+    except FileNotFoundError:
+        if not optional:
+            raise
+        logger.info("Skipping missing optional include: %s", file_or_url)
+        return []
+    if "settings" not in cfg:
         return [file]
     file_list = []
-    for include in cfg["settings"]["include"].split("\n"):
-        include = include.strip()
-        if not include:
-            continue
-        # Check if it's a real URL scheme (not a Windows drive letter)
-        parsed_include = parse.urlparse(include)
-        is_include_url = parsed_include.scheme and len(parsed_include.scheme) > 1
-        if http_parent or is_include_url:
-            file_list += resolve_dependencies(include, tmpdir, http_parent)
-        else:
-            file_list += resolve_dependencies(file.parent / include, tmpdir)
+    for directive in ("include", "include-optional"):
+        for include in cfg["settings"].get(directive, "").splitlines():
+            include = include.strip()
+            if not include:
+                continue
+            # Check if it's a real URL scheme (not a Windows drive letter)
+            parsed_include = parse.urlparse(include)
+            is_include_url = parsed_include.scheme and len(parsed_include.scheme) > 1
+            if http_parent or is_include_url:
+                file_list += resolve_dependencies(
+                    file_or_url=include,
+                    tmpdir=tmpdir,
+                    http_parent=http_parent,
+                    optional=directive == "include-optional",
+                )
+            else:
+                file_list += resolve_dependencies(
+                    file_or_url=file.parent / include,
+                    tmpdir=tmpdir,
+                    optional=directive == "include-optional",
+                )
 
     file_list.append(file)
     return file_list
@@ -80,6 +101,9 @@ def read_with_included(file_or_url: str | Path) -> ConfigParser:
     cfg.optionxform = str  # type: ignore
     cfg["settings"]["directory"] = os.getcwd()
     with tempfile.TemporaryDirectory() as tmpdir:
-        resolved = resolve_dependencies(file_or_url, tmpdir)
+        resolved = resolve_dependencies(
+            file_or_url=file_or_url,
+            tmpdir=tmpdir,
+        )
         cfg.read(resolved)
     return cfg

tests/test_including.py

@@ -2,6 +2,88 @@
 import pytest
 
 
+def test_optional_includes_and_precedence(tmp_path):
+    from mxdev.including import read_with_included
+
+    (tmp_path / "required.ini").write_text("[settings]\nvalue = required\n")
+    (tmp_path / "nested.ini").write_text("[settings]\nnested = yes\n")
+    (tmp_path / "custom.ini").write_text(
+        "[settings]\ninclude = nested.ini\ninclude-optional = absent.ini\nvalue = optional\nmain = optional\n"
+    )
+    main = tmp_path / "mx.ini"
+    main.write_text(
+        "[settings]\ninclude = required.ini\ninclude-optional =\n    missing.ini\n    custom.ini\nmain = main\n"
+    )
+    cfg = read_with_included(main)
+    assert cfg["settings"]["value"] == "optional"
+    assert cfg["settings"]["main"] == "main"
+    assert cfg["settings"]["nested"] == "yes"
+
+
+def test_optional_include_preserves_required_nested_failure(tmp_path):
+    from mxdev.including import read_with_included
+
+    main = tmp_path / "mx.ini"
+    main.write_text("[settings]\ninclude-optional = custom.ini\n")
+    (tmp_path / "custom.ini").write_text("[settings]\ninclude = missing.ini\n")
+    with pytest.raises(FileNotFoundError):
+        read_with_included(main)
+
+
+@pytest.mark.parametrize("directive", ["include", "include-optional"])
+@pytest.mark.parametrize("status", [404, 403, 500])
+def test_include_http_errors(tmp_path, httpretty, directive, status):
+    from mxdev.including import read_with_included
+    from urllib.error import HTTPError
+
+    url = "http://example.com/custom.ini"
+    httpretty.register_uri(httpretty.GET, url, status=status)
+    main = tmp_path / "mx.ini"
+    main.write_text(f"[settings]\n{directive} = {url}\nvalue = main\n")
+    if directive == "include-optional" and status == 404:
+        assert read_with_included(main)["settings"]["value"] == "main"
+    else:
+        with pytest.raises(HTTPError) as exc:
+            read_with_included(main)
+        assert exc.value.code == status
+
+
+@pytest.mark.parametrize("directive", ["include", "include-optional"])
+def test_http_relative_include(tmp_path, httpretty, directive):
+    from mxdev.including import read_with_included
+
+    url = "http://example.com/config/mx.ini"
+    httpretty.register_uri(httpretty.GET, url, body=f"[settings]\n{directive} = custom.ini\n")
+    httpretty.register_uri(httpretty.GET, "http://example.com/config/custom.ini", body="[settings]\ncustom = yes\n")
+    assert read_with_included(url)["settings"]["custom"] == "yes"
+
+
+def test_optional_include_connection_error(tmp_path, monkeypatch):
+    from mxdev.including import read_with_included
+    from urllib.error import URLError
+
+    main = tmp_path / "mx.ini"
+    main.write_text("[settings]\ninclude-optional = https://example.com/custom.ini\n")
+
+    def fail(url):
+        raise URLError("Connection failed")
+
+    monkeypatch.setattr("mxdev.including.request.urlopen", fail)
+    with pytest.raises(URLError, match="Connection failed"):
+        read_with_included(main)
+
+
+def test_optional_include_invalid_ini(tmp_path):
+    from configparser import MissingSectionHeaderError
+    from mxdev.including import read_with_included
+
+    main = tmp_path / "mx.ini"
+    main.write_text("[settings]\ninclude-optional = custom.ini\n")
+    (tmp_path / "custom.ini").write_text("invalid ini\n")
+    with pytest.raises(MissingSectionHeaderError):
+        read_with_included(main)
+
+
 def test_resolve_dependencies_files():
     from mxdev.including import resolve_dependencies