diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 24c5de5..7af809c 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -11,3 +11,7 @@ updates: directory: "/" schedule: interval: "weekly" + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" diff --git a/.github/workflows/github_release_on_release_branch_merge.yml b/.github/workflows/github_release_on_release_branch_merge.yml index 49829b9..607834b 100644 --- a/.github/workflows/github_release_on_release_branch_merge.yml +++ b/.github/workflows/github_release_on_release_branch_merge.yml @@ -26,12 +26,12 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout merge commit - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.event.pull_request.merge_commit_sha }} - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.12" diff --git a/.github/workflows/proxy_integration_tests.yml b/.github/workflows/proxy_integration_tests.yml index a47659c..048fd8c 100644 --- a/.github/workflows/proxy_integration_tests.yml +++ b/.github/workflows/proxy_integration_tests.yml @@ -12,16 +12,40 @@ permissions: contents: read jobs: + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 + with: + persist-credentials: false + + - name: Set up Python + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 + with: + python-version: "3.x" + + - name: Install system dependencies (pycurl) + run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + + - name: Install package and test dependencies + run: | + python -m pip install --upgrade pip + pip install -e . + pip install urllib3 requests aiohttp httpx pycurl + + - name: Run header security unit tests + run: python test_header_security.py + integration: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: persist-credentials: false - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.x" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 0bc9391..0f62f3d 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -30,7 +30,7 @@ jobs: outputs: publish: ${{ steps.decide.outputs.publish }} steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 if: github.event_name == 'workflow_run' with: ref: main @@ -64,12 +64,12 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6 + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 with: ref: ${{ github.event_name == 'workflow_run' && 'main' || github.event_name == 'release' && github.ref || 'main' }} - name: Set up Python - uses: actions/setup-python@v6 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 with: python-version: "3.x" @@ -82,7 +82,7 @@ jobs: run: python -m build - name: Store distribution packages - uses: actions/upload-artifact@v7 + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a with: name: python-package-distributions path: dist/ @@ -102,13 +102,13 @@ jobs: steps: - name: Download distribution packages - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: python-package-distributions path: dist/ - name: Publish to PyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 publish-to-testpypi: name: Publish to TestPyPI @@ -122,12 +122,12 @@ jobs: steps: - name: Download distribution packages - uses: actions/download-artifact@v8 + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c with: name: python-package-distributions path: dist/ - name: Publish to TestPyPI - uses: pypa/gh-action-pypi-publish@release/v1 + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 with: repository-url: https://test.pypi.org/legacy/ diff --git a/docs/aiohttp.rst b/docs/aiohttp.rst index a3e0503..e571ae5 100644 --- a/docs/aiohttp.rst +++ b/docs/aiohttp.rst @@ -284,7 +284,7 @@ When establishing an HTTPS connection through a proxy, the connector: * Creates a CONNECT request to the proxy server * Includes any custom proxy headers you've specified * Captures the proxy's response headers (e.g., ``X-ProxyMesh-IP``) -* Stores them so they can be merged into the final response +* Stores them so safe CONNECT values can be merged into the final response without overwriting origin headers. Hop-by-hop and security-sensitive names are omitted. The full CONNECT set is on ``response.proxy_headers``. You typically don't need to use this class directly - it's automatically configured when using ``ProxyClientSession``. diff --git a/docs/httpx.rst b/docs/httpx.rst index 970d430..e33076d 100644 --- a/docs/httpx.rst +++ b/docs/httpx.rst @@ -375,7 +375,7 @@ The extension classes work by intercepting the CONNECT request/response cycle du 2. **CONNECT Response**: The proxy responds with a CONNECT response (status 200) that may include proxy information headers in the response (e.g., ``X-ProxyMesh-IP: 192.168.1.1``) -3. **Header Merging**: These proxy response headers are captured during the tunnel establishment and stored. When the actual HTTP request is made through the tunnel, the proxy response headers are merged into the final HTTP response headers using ``merge_headers()`` +3. **Header Merging**: Safe CONNECT headers are stored and later merged into the origin response with ``merge_headers()``. Origin headers are never overwritten, and hop-by-hop or security-sensitive names such as ``Set-Cookie`` or ``Location`` from CONNECT are not copied. 4. **Access**: Your application can then access both the target server's response headers and the proxy's response headers from the same response object diff --git a/docs/requests.rst b/docs/requests.rst index a089262..f6328ce 100644 --- a/docs/requests.rst +++ b/docs/requests.rst @@ -46,6 +46,8 @@ This section shows you how to quickly get up and running with proxy headers in r That's it! The ``requests_adapter`` module handles sending your custom headers to the proxy and makes proxy response headers available in the response. +CONNECT response headers are copied onto ``response.headers`` except hop-by-hop and security-sensitive names (``Set-Cookie``, ``Location``, ``Content-Type``, and similar). Existing origin headers are never overwritten. The full CONNECT set is on ``response.proxy_headers``. + Using Proxies with requests --------------------------- diff --git a/docs/urllib3.rst b/docs/urllib3.rst index c7613ad..63449a5 100644 --- a/docs/urllib3.rst +++ b/docs/urllib3.rst @@ -45,6 +45,8 @@ This section shows you how to quickly get up and running with proxy headers in u That's it! The ``ProxyHeaderManager`` handles sending your custom headers to the proxy and makes proxy response headers available in the response. +CONNECT response headers are copied onto ``response.headers`` except hop-by-hop and security-sensitive names (``Set-Cookie``, ``Location``, ``Content-Type``, and similar). Existing origin headers are never overwritten. The full CONNECT set is on ``response.proxy_headers``. + Using Proxies with urllib3 -------------------------- diff --git a/python_proxy_headers/aiohttp_proxy.py b/python_proxy_headers/aiohttp_proxy.py index 551b4ee..af94f04 100644 --- a/python_proxy_headers/aiohttp_proxy.py +++ b/python_proxy_headers/aiohttp_proxy.py @@ -6,6 +6,7 @@ from aiohttp.helpers import reify from aiohttp import hdrs from multidict import CIMultiDict, CIMultiDictProxy +from .header_utils import merge_proxy_response_headers class ProxyTCPConnector(TCPConnector): async def _create_proxy_connection(self, req: ClientRequest, traces, timeout): @@ -143,13 +144,21 @@ async def send(self, conn): return resp class ProxyClientResponse(ClientResponse): + @reify + def proxy_headers(self): + """Headers from the proxy CONNECT response, not merged into origin headers.""" + raw = getattr(self, "_proxy_headers", None) + if raw: + return CIMultiDictProxy(CIMultiDict(raw)) + return CIMultiDictProxy(CIMultiDict()) + @reify def headers(self): proxy_headers = getattr(self, '_proxy_headers', None) if proxy_headers: headers = CIMultiDict(self._headers) - headers.extend(proxy_headers) + merge_proxy_response_headers(headers, proxy_headers) return CIMultiDictProxy(headers) else: return self._headers diff --git a/python_proxy_headers/autoscraper_proxy.py b/python_proxy_headers/autoscraper_proxy.py index 09e1ba0..4c75ff2 100644 --- a/python_proxy_headers/autoscraper_proxy.py +++ b/python_proxy_headers/autoscraper_proxy.py @@ -35,6 +35,7 @@ "Install it with: pip install autoscraper" ) +from .header_utils import validate_headers from .requests_adapter import ProxySession @@ -71,7 +72,7 @@ def __init__( stack_list: Optional[List] = None ): super().__init__(stack_list=stack_list) - self._proxy_headers = proxy_headers or {} + self._proxy_headers = validate_headers(proxy_headers) self._session: Optional[ProxySession] = None def _get_session(self) -> ProxySession: @@ -90,7 +91,7 @@ def set_proxy_headers(self, proxy_headers: Dict[str, str]): Args: proxy_headers: New proxy headers to use """ - self._proxy_headers = proxy_headers + self._proxy_headers = validate_headers(proxy_headers) if self._session is not None: self._session.close() self._session = None diff --git a/python_proxy_headers/cloudscraper_proxy.py b/python_proxy_headers/cloudscraper_proxy.py index b2f547f..d8de536 100644 --- a/python_proxy_headers/cloudscraper_proxy.py +++ b/python_proxy_headers/cloudscraper_proxy.py @@ -27,6 +27,7 @@ "Install it with: pip install cloudscraper" ) +from .header_utils import validate_headers from .urllib3_proxy_manager import proxy_from_url @@ -41,8 +42,13 @@ class CipherSuiteProxyHeaderAdapter(CipherSuiteAdapter): """ def __init__(self, proxy_headers: Optional[Dict[str, str]] = None, **kwargs): - self._proxy_headers = proxy_headers or {} + self._proxy_headers = validate_headers(proxy_headers) super().__init__(**kwargs) + + def build_response(self, req, resp): + response = super().build_response(req, resp) + response.proxy_headers = getattr(resp, "proxy_headers", {}) or {} + return response def proxy_manager_for(self, proxy, **proxy_kwargs): """ @@ -103,7 +109,7 @@ class ProxyCloudScraper(cloudscraper.CloudScraper): """ def __init__(self, proxy_headers: Optional[Dict[str, str]] = None, **kwargs): - self._proxy_headers = proxy_headers or {} + self._proxy_headers = validate_headers(proxy_headers) # Call parent init super().__init__(**kwargs) @@ -142,7 +148,7 @@ def set_proxy_headers(self, proxy_headers: Dict[str, str]): Args: proxy_headers: New proxy headers to use """ - self._proxy_headers = proxy_headers + self._proxy_headers = validate_headers(proxy_headers) # Remount adapters with new headers self.mount( diff --git a/python_proxy_headers/header_utils.py b/python_proxy_headers/header_utils.py new file mode 100644 index 0000000..eb4d69d --- /dev/null +++ b/python_proxy_headers/header_utils.py @@ -0,0 +1,165 @@ +"""Validation and safe merging for proxy CONNECT headers. + +CONNECT requests are raw HTTP and must not interpolate CR, LF, or NUL. +CONNECT response headers are not origin HTTPS headers: hop-by-hop and +security-sensitive names are not copied onto the origin response, and +existing origin headers are never overwritten. +""" + +from __future__ import annotations + +from typing import Any, Dict, List, Mapping, MutableMapping, Optional, Sequence, Tuple, Union + +RawHeaders = Union[Mapping[Any, Any], Sequence[Tuple[Any, Any]], None] + +# Never copy these from a CONNECT response onto the origin response. +BLOCKED_PROXY_RESPONSE_HEADERS = frozenset({ + "age", + "authorization", + "cache-control", + "clear-site-data", + "connection", + "content-disposition", + "content-encoding", + "content-language", + "content-length", + "content-location", + "content-range", + "content-security-policy", + "content-security-policy-report-only", + "content-type", + "cookie", + "date", + "etag", + "expires", + "host", + "keep-alive", + "last-modified", + "link", + "location", + "pragma", + "proxy-agent", + "proxy-authenticate", + "proxy-authorization", + "proxy-connection", + "refresh", + "server", + "set-cookie", + "set-cookie2", + "strict-transport-security", + "te", + "trailer", + "transfer-encoding", + "upgrade", + "vary", + "via", + "warning", + "www-authenticate", + "x-content-type-options", + "x-frame-options", + "x-xss-protection", +}) + + +def _as_str(value: Any) -> str: + if isinstance(value, bytes): + return value.decode("latin-1") + return str(value) + + +def validate_header_name(name: Any) -> str: + """Return a header name, or raise ValueError if it is not a single token.""" + name_s = _as_str(name) + if not name_s: + raise ValueError("Header name must not be empty") + for char in name_s: + code = ord(char) + if char in "()<>@,;:\\\"/[]?={} \t:" or code <= 32 or code == 127: + raise ValueError(f"Invalid header name {name_s!r}") + return name_s + + +def validate_header_value(value: Any) -> str: + """Return a header value, or raise ValueError if it contains CR, LF, or NUL.""" + value_s = _as_str(value) + if any(char in value_s for char in "\r\n\x00"): + raise ValueError(f"Invalid header value {value_s!r}") + return value_s + + +def validate_headers(headers: Optional[Mapping[Any, Any]]) -> Dict[str, str]: + """Validate a mapping of CONNECT request headers. + + Raises: + ValueError: If any name or value contains CR, LF, NUL, or other + characters illegal in a single CONNECT header line. + """ + if not headers: + return {} + validated: Dict[str, str] = {} + for name, value in headers.items(): + validated[validate_header_name(name)] = validate_header_value(value) + return validated + + +def _header_items(headers: RawHeaders) -> List[Tuple[Any, Any]]: + if not headers: + return [] + if isinstance(headers, (list, tuple)): + return list(headers) + if hasattr(headers, "items"): + return list(headers.items()) + return [] + + +def origin_has_header(origin_headers: RawHeaders, name: str) -> bool: + """Return True if origin_headers already contains ``name`` (case-insensitive).""" + lowered = _as_str(name).lower() + for key, _value in _header_items(origin_headers): + if _as_str(key).lower() == lowered: + return True + return False + + +def is_mergeable_proxy_header(name: Any) -> bool: + """Return True if a CONNECT response header may be copied onto origin headers.""" + lowered = _as_str(name).lower() + if lowered in BLOCKED_PROXY_RESPONSE_HEADERS: + return False + if lowered.startswith("access-control-"): + return False + return True + + +def snapshot_headers(headers: RawHeaders) -> Dict[str, str]: + """Copy headers into a plain dict of strings.""" + snapshot: Dict[str, str] = {} + for name, value in _header_items(headers): + snapshot[_as_str(name)] = _as_str(value) + return snapshot + + +def merge_proxy_response_headers( + origin_headers: MutableMapping[Any, Any], + proxy_headers: RawHeaders, +) -> None: + """Copy safe CONNECT headers onto origin headers without overwriting.""" + for name, value in _header_items(proxy_headers): + name_s = _as_str(name) + if not is_mergeable_proxy_header(name_s): + continue + if origin_has_header(origin_headers, name_s): + continue + origin_headers[name_s] = _as_str(value) + + +def filter_connect_headers( + origin_headers: RawHeaders, + connect_headers: RawHeaders, +) -> List[Tuple[Any, Any]]: + """Return CONNECT header pairs that are safe to merge into origin headers.""" + extra: List[Tuple[Any, Any]] = [] + for name, value in _header_items(connect_headers): + if is_mergeable_proxy_header(name) and not origin_has_header(origin_headers, name): + extra.append((name, value)) + return extra diff --git a/python_proxy_headers/httpx_proxy.py b/python_proxy_headers/httpx_proxy.py index d61e7f5..430b5e3 100644 --- a/python_proxy_headers/httpx_proxy.py +++ b/python_proxy_headers/httpx_proxy.py @@ -9,6 +9,7 @@ from httpcore._trace import Trace from httpx import AsyncHTTPTransport, HTTPTransport, Client from httpx._config import DEFAULT_LIMITS, DEFAULT_TIMEOUT_CONFIG, Proxy, create_ssl_context +from .header_utils import filter_connect_headers class ProxyTunnelHTTPConnection(TunnelHTTPConnection): # Unfortunately the only way to get connect_response.headers into the Response @@ -90,10 +91,16 @@ def handle_request(self, request): keepalive_expiry=self._keepalive_expiry, ) + self._connect_response_headers = connect_response.headers self._connected = True - # this is the only modification + # Merge safe CONNECT headers without overwriting origin values response = self._connection.handle_request(request) - response.headers = merge_headers(response.headers, connect_response.headers) + response.headers = merge_headers( + response.headers, + filter_connect_headers( + response.headers, getattr(self, "_connect_response_headers", ()) + ), + ) return response class AsyncProxyTunnelHTTPConnection(AsyncTunnelHTTPConnection): @@ -174,10 +181,16 @@ async def handle_async_request(self, request): keepalive_expiry=self._keepalive_expiry, ) + self._connect_response_headers = connect_response.headers self._connected = True - # this is the only modification + # Merge safe CONNECT headers without overwriting origin values response = await self._connection.handle_async_request(request) - response.headers = merge_headers(response.headers, connect_response.headers) + response.headers = merge_headers( + response.headers, + filter_connect_headers( + response.headers, getattr(self, "_connect_response_headers", ()) + ), + ) return response class HTTPProxyHeaders(HTTPProxy): diff --git a/python_proxy_headers/pycurl_proxy.py b/python_proxy_headers/pycurl_proxy.py index ca60d73..d703a54 100644 --- a/python_proxy_headers/pycurl_proxy.py +++ b/python_proxy_headers/pycurl_proxy.py @@ -38,6 +38,8 @@ from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple +from .header_utils import validate_headers + try: import pycurl except ImportError: @@ -67,7 +69,8 @@ def set_proxy_headers(curl, headers: Dict[str, str]) -> None: """ if not headers: return - + + headers = validate_headers(headers) header_list = [f"{k}: {v}" for k, v in headers.items()] # Set CURLOPT_PROXYHEADER diff --git a/python_proxy_headers/requests_adapter.py b/python_proxy_headers/requests_adapter.py index 40e291b..870a606 100644 --- a/python_proxy_headers/requests_adapter.py +++ b/python_proxy_headers/requests_adapter.py @@ -1,11 +1,17 @@ from requests.adapters import HTTPAdapter from requests.sessions import Session +from .header_utils import validate_headers from .urllib3_proxy_manager import proxy_from_url class HTTPProxyHeaderAdapter(HTTPAdapter): def __init__(self, proxy_headers=None): super().__init__() - self._proxy_headers = proxy_headers or {} + self._proxy_headers = validate_headers(proxy_headers) + + def build_response(self, req, resp): + response = super().build_response(req, resp) + response.proxy_headers = getattr(resp, "proxy_headers", {}) or {} + return response def proxy_manager_for(self, proxy, **proxy_kwargs): """Return urllib3 ProxyManager for the given proxy. diff --git a/python_proxy_headers/urllib3_proxy_manager.py b/python_proxy_headers/urllib3_proxy_manager.py index 11948c6..718bfcd 100644 --- a/python_proxy_headers/urllib3_proxy_manager.py +++ b/python_proxy_headers/urllib3_proxy_manager.py @@ -6,6 +6,14 @@ from urllib3.util.request import make_headers from urllib3.util.url import parse_url +from .header_utils import ( + merge_proxy_response_headers, + snapshot_headers, + validate_header_name, + validate_header_value, + validate_headers, +) + if sys.version_info < (3, 12, 0): ##################################### ### copied from python3.12 source ### @@ -77,6 +85,8 @@ def _tunnel(self): self._http_vsn_str.encode("ascii")) headers = [connect] for header, value in self._tunnel_headers.items(): + validate_header_name(header) + validate_header_value(value) headers.append(f"{header}: {value}\r\n".encode("latin-1")) headers.append(b"\r\n") # Making a single send() call instead of one per line encourages @@ -125,7 +135,9 @@ def _prepare_proxy(self, conn): def urlopen(self, *args, **kwargs): response = super().urlopen(*args, **kwargs) - response.headers.update(self._proxy_response_headers) + proxy_headers = getattr(self, "_proxy_response_headers", None) + response.proxy_headers = snapshot_headers(proxy_headers) + merge_proxy_response_headers(response.headers, proxy_headers) return response class ProxyHeaderManager(ProxyManager): @@ -137,15 +149,14 @@ def __init__(self, *args, **kwargs): if proxy_url is None and args: proxy_url = args[0] proxy_headers = kwargs.get("proxy_headers") - merged = dict(proxy_headers or {}) + merged = validate_headers(dict(proxy_headers or {})) if isinstance(proxy_url, str): parsed = parse_url(proxy_url) if parsed.auth and not any( k.lower() == "proxy-authorization" for k in merged ): merged.update(make_headers(proxy_basic_auth=parsed.auth)) - if merged != dict(proxy_headers or {}): - kwargs["proxy_headers"] = merged + kwargs["proxy_headers"] = merged super().__init__(*args, **kwargs) self.pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSProxyConnectionPool} diff --git a/test_header_security.py b/test_header_security.py new file mode 100644 index 0000000..6be189f --- /dev/null +++ b/test_header_security.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Unit tests for CONNECT header validation and safe origin-header merging.""" + +from __future__ import annotations + +import os +import socket +import ssl +import subprocess +import tempfile +import threading +import time +import unittest + +from python_proxy_headers.header_utils import ( + filter_connect_headers, + is_mergeable_proxy_header, + merge_proxy_response_headers, + validate_header_name, + validate_header_value, + validate_headers, +) + + +class ValidateHeadersTests(unittest.TestCase): + def test_accepts_normal_proxy_header(self) -> None: + self.assertEqual( + validate_headers({"X-ProxyMesh-Country": "US"}), + {"X-ProxyMesh-Country": "US"}, + ) + + def test_rejects_crlf_in_value(self) -> None: + with self.assertRaises(ValueError): + validate_header_value("US\r\nX-Injected: pwned") + + def test_rejects_crlf_in_name(self) -> None: + with self.assertRaises(ValueError): + validate_header_name("X-Foo: bar\r\nX-Injected") + + def test_rejects_nul(self) -> None: + with self.assertRaises(ValueError): + validate_headers({"X-ProxyMesh-Country": "US\x00evil"}) + + def test_proxy_from_url_rejects_crlf(self) -> None: + from python_proxy_headers.urllib3_proxy_manager import proxy_from_url + + with self.assertRaises(ValueError): + proxy_from_url( + "http://127.0.0.1:9", + proxy_headers={ + "X-ProxyMesh-Country": "US\r\nProxy-Authorization: Basic ZXZpbDpldmls" + }, + ) + + def test_requests_adapter_rejects_crlf(self) -> None: + from python_proxy_headers.requests_adapter import ProxySession + + with self.assertRaises(ValueError): + ProxySession(proxy_headers={"X-ProxyMesh-Country": "US\r\nX-Injected: pwned"}) + + def test_pycurl_rejects_crlf(self) -> None: + try: + import pycurl + from python_proxy_headers.pycurl_proxy import set_proxy_headers + except ImportError: + self.skipTest("pycurl is not installed") + curl = pycurl.Curl() + try: + with self.assertRaises(ValueError): + set_proxy_headers( + curl, {"X-ProxyMesh-Country": "US\r\nX-Injected: pwned"} + ) + finally: + curl.close() + + +class MergeProxyHeadersTests(unittest.TestCase): + def test_merges_custom_proxy_header(self) -> None: + origin = {"Content-Type": "text/plain"} + merge_proxy_response_headers(origin, {"X-Custom-Exit-IP": "203.0.113.10"}) + self.assertEqual(origin["X-Custom-Exit-IP"], "203.0.113.10") + self.assertEqual(origin["Content-Type"], "text/plain") + + def test_does_not_overwrite_origin(self) -> None: + origin = {"X-Custom-Exit-IP": "origin-value"} + merge_proxy_response_headers(origin, {"X-Custom-Exit-IP": "proxy-value"}) + self.assertEqual(origin["X-Custom-Exit-IP"], "origin-value") + + def test_blocks_set_cookie_and_location(self) -> None: + origin = {"Content-Type": "text/plain"} + merge_proxy_response_headers( + origin, + { + "Set-Cookie": "session=attacker", + "Location": "https://evil.example/", + "Content-Type": "text/html", + "Proxy-Connection": "Keep-Alive", + "X-Custom-Exit-IP": "203.0.113.10", + }, + ) + self.assertEqual(origin["Content-Type"], "text/plain") + self.assertNotIn("Set-Cookie", origin) + self.assertNotIn("Location", origin) + self.assertNotIn("Proxy-Connection", origin) + self.assertEqual(origin["X-Custom-Exit-IP"], "203.0.113.10") + + def test_is_mergeable_skips_sensitive_headers(self) -> None: + self.assertTrue(is_mergeable_proxy_header("X-Custom-Exit-IP")) + self.assertTrue(is_mergeable_proxy_header("X-ProxyMesh-IP")) + self.assertFalse(is_mergeable_proxy_header("Set-Cookie")) + self.assertFalse(is_mergeable_proxy_header("Location")) + self.assertFalse(is_mergeable_proxy_header("Proxy-Connection")) + + def test_filter_connect_headers_preserves_bytes(self) -> None: + origin = [(b"content-type", b"text/plain")] + connect = [ + (b"set-cookie", b"session=attacker"), + (b"x-custom-exit-ip", b"203.0.113.10"), + ] + extra = filter_connect_headers(origin, connect) + self.assertEqual(extra, [(b"x-custom-exit-ip", b"203.0.113.10")]) + + +class _LocalHttpsProxy: + """Minimal CONNECT proxy plus TLS origin for merge/injection tests.""" + + def __init__(self, connect_headers: bytes) -> None: + self.connect_headers = connect_headers + self.captured_connect = b"" + self._tmpdir = tempfile.mkdtemp() + self._key = os.path.join(self._tmpdir, "key.pem") + self._cert = os.path.join(self._tmpdir, "cert.pem") + subprocess.check_call( + [ + "openssl", + "req", + "-x509", + "-newkey", + "rsa:2048", + "-keyout", + self._key, + "-out", + self._cert, + "-days", + "1", + "-nodes", + "-subj", + "/CN=127.0.0.1", + ], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + ) + self._ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self._ctx.load_cert_chain(self._cert, self._key) + self._origin = socket.socket() + self._origin.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._origin.bind(("127.0.0.1", 0)) + self.origin_port = self._origin.getsockname()[1] + self._proxy = socket.socket() + self._proxy.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._proxy.bind(("127.0.0.1", 0)) + self.proxy_port = self._proxy.getsockname()[1] + self._threads = [ + threading.Thread(target=self._run_origin, daemon=True), + threading.Thread(target=self._run_proxy, daemon=True), + ] + + def start(self) -> None: + for thread in self._threads: + thread.start() + time.sleep(0.05) + + def close(self) -> None: + for sock in (self._origin, self._proxy): + try: + sock.close() + except OSError: + pass + + def _run_origin(self) -> None: + self._origin.listen(5) + self._origin.settimeout(8) + try: + conn, _ = self._origin.accept() + except (socket.timeout, OSError): + return + try: + tls = self._ctx.wrap_socket(conn, server_side=True) + tls.settimeout(3) + data = b"" + while b"\r\n\r\n" not in data: + chunk = tls.recv(4096) + if not chunk: + break + data += chunk + body = b"origin-body" + tls.sendall( + b"HTTP/1.1 200 OK\r\n" + b"Content-Type: text/plain\r\n" + b"Set-Cookie: origin=safe\r\n" + b"X-Origin: real\r\n" + b"Content-Length: " + str(len(body)).encode() + b"\r\n" + b"\r\n" + body + ) + tls.close() + except Exception: + try: + conn.close() + except OSError: + pass + + def _run_proxy(self) -> None: + self._proxy.listen(5) + self._proxy.settimeout(8) + try: + conn, _ = self._proxy.accept() + except (socket.timeout, OSError): + return + conn.settimeout(5) + data = b"" + try: + while b"\r\n\r\n" not in data: + chunk = conn.recv(4096) + if not chunk: + break + data += chunk + except OSError: + pass + self.captured_connect = data + try: + conn.sendall(b"HTTP/1.1 200 Connection Established\r\n" + self.connect_headers + b"\r\n") + except OSError: + conn.close() + return + try: + origin = socket.create_connection(("127.0.0.1", self.origin_port), timeout=5) + except OSError: + conn.close() + return + conn.setblocking(False) + origin.setblocking(False) + deadline = time.time() + 4 + while time.time() < deadline: + moved = False + for src, dst in ((conn, origin), (origin, conn)): + try: + chunk = src.recv(8192) + except BlockingIOError: + chunk = None + except OSError: + chunk = b"" + if chunk: + try: + dst.sendall(chunk) + moved = True + except OSError: + deadline = 0 + break + elif chunk == b"": + deadline = 0 + break + if not moved: + time.sleep(0.01) + for sock in (conn, origin): + try: + sock.close() + except OSError: + pass + + +class ConnectMergeIntegrationTests(unittest.TestCase): + def test_urllib3_does_not_copy_hostile_connect_headers(self) -> None: + import urllib3 + from python_proxy_headers.requests_adapter import ProxySession + + urllib3.disable_warnings() + helper = _LocalHttpsProxy( + b"Set-Cookie: session=attacker\r\n" + b"Location: https://evil.example/\r\n" + b"Content-Type: text/html\r\n" + b"X-Origin: spoofed\r\n" + b"X-Custom-Exit-IP: 203.0.113.10\r\n" + ) + helper.start() + try: + with ProxySession() as session: + session.verify = False + session.proxies = {"https": f"http://127.0.0.1:{helper.proxy_port}"} + response = session.get( + f"https://127.0.0.1:{helper.origin_port}/", timeout=5 + ) + self.assertEqual(response.status_code, 200) + self.assertEqual(response.text, "origin-body") + self.assertEqual(response.headers.get("Content-Type"), "text/plain") + self.assertEqual(response.headers.get("X-Origin"), "real") + self.assertNotEqual(response.headers.get("Location"), "https://evil.example/") + self.assertNotIn("session=attacker", response.headers.get("Set-Cookie", "")) + self.assertEqual(response.headers.get("X-Custom-Exit-IP"), "203.0.113.10") + self.assertEqual(response.cookies.get("origin"), "safe") + self.assertNotIn("session", response.cookies) + proxy_headers = {k.lower(): v for k, v in (response.proxy_headers or {}).items()} + self.assertEqual(proxy_headers.get("x-custom-exit-ip"), "203.0.113.10") + self.assertEqual(proxy_headers.get("set-cookie"), "session=attacker") + finally: + helper.close() + + def test_urllib3_does_not_send_injected_connect_header(self) -> None: + from python_proxy_headers.urllib3_proxy_manager import proxy_from_url + + helper = _LocalHttpsProxy(b"") + helper.start() + try: + with self.assertRaises(ValueError): + proxy_from_url( + f"http://127.0.0.1:{helper.proxy_port}", + proxy_headers={"X-ProxyMesh-Country": "US\r\nX-Injected: pwned"}, + timeout=2.0, + retries=False, + ) + self.assertEqual(helper.captured_connect, b"") + finally: + helper.close() + + +if __name__ == "__main__": + unittest.main()