diff --git a/.github/workflows/dependency-testing.yaml b/.github/workflows/dependency-testing.yaml
index b07ca4c9f..bca85b137 100644
--- a/.github/workflows/dependency-testing.yaml
+++ b/.github/workflows/dependency-testing.yaml
@@ -5,6 +5,7 @@ on:
branches: [main]
paths:
- 'pyproject.toml'
+ - 'packages/**'
- 'requirements/**'
- 'validmind/**'
- 'tests/**'
@@ -12,6 +13,7 @@ on:
branches: ['*']
paths:
- 'pyproject.toml'
+ - 'packages/**'
- 'requirements/**'
- 'validmind/**'
- 'tests/**'
@@ -68,9 +70,9 @@ jobs:
curl -LsSf https://astral.sh/uv/install.sh | sh
echo "$HOME/.cargo/bin" >> $GITHUB_PATH
- - name: Build wheel and sdist
+ - name: Build wheels and sdists
run: |
- python -m build
+ uv build --all-packages
- name: Install and test via pip artifacts (${{ matrix.deps-type }})
run: |
@@ -82,7 +84,9 @@ jobs:
if [[ "${{ matrix.deps-type }}" == "core" ]]; then
# Exercise the unbounded core dependency set separately from the
# constrained optional extras (for example, latest scikit-learn).
- WHEEL=$(ls dist/*.whl | head -n 1)
+ WHEEL=$(ls dist/validmind-*.whl | head -n 1)
+ CORE_WHEEL=$(ls dist/validmind_tracking_core*.whl | head -n 1)
+ python -m pip install "${CORE_WHEEL}"
# Jinja2 is currently imported by the library but is only present
# transitively through optional extras. Install it as test tooling
# so this leg can exercise the otherwise unbounded core set.
@@ -95,7 +99,9 @@ jobs:
| cat
elif [[ "${{ matrix.deps-type }}" == "default" ]]; then
# Install only from built artifacts (let pip resolve deps normally)
- WHEEL=$(ls dist/*.whl | head -n 1)
+ WHEEL=$(ls dist/validmind-*.whl | head -n 1)
+ CORE_WHEEL=$(ls dist/validmind_tracking_core*.whl | head -n 1)
+ python -m pip install "${CORE_WHEEL}"
python -m pip install "${WHEEL}[all]"
pip check
python -m tests.test_unit_tests | cat
@@ -105,7 +111,9 @@ jobs:
uv pip compile pyproject.toml -p "${{ matrix.python-version }}" --all-extras --no-emit-index-url --no-annotate --no-strip-markers --output-file "$OUT" --upgrade
# Install constraints then the wheel without reinstalling deps
pip install -r "$OUT"
- WHEEL=$(ls dist/*.whl | head -n 1)
+ WHEEL=$(ls dist/validmind-*.whl | head -n 1)
+ CORE_WHEEL=$(ls dist/validmind_tracking_core*.whl | head -n 1)
+ pip install "${CORE_WHEEL}" --no-deps
python -m pip install "${WHEEL}[all]" --no-deps
pip check
python -m tests.test_unit_tests | cat
@@ -146,6 +154,7 @@ jobs:
fi
python -m pip install --upgrade pip
pip install -r "$FREEZE_FILE"
+ python -m pip install packages/validmind-tracking-core --no-deps
python -m pip install . --no-deps
python -m tests.test_unit_tests | cat
diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml
index 347733f6d..c3f3422de 100644
--- a/.github/workflows/integration.yaml
+++ b/.github/workflows/integration.yaml
@@ -44,7 +44,7 @@ jobs:
sudo apt install r-base r-base-dev
- name: Build the package
- run: uv build
+ run: uv build --all-packages
- name: Remove Build Environment
run: rm -rf .venv
@@ -52,9 +52,11 @@ jobs:
- name: 'Setup Virtual Environment for [all]'
run: python -m venv all-venv
- # This proves that the [all] install target works
+ # This proves that the [all] install target works
- name: 'Install Built Package for [all]'
- run: all-venv/bin/pip install --no-cache-dir "$(ls dist/validmind*.whl | head -n 1)[all]"
+ run: |
+ all-venv/bin/pip install --no-cache-dir "$(ls dist/validmind_tracking_core*.whl | head -n 1)"
+ all-venv/bin/pip install --no-cache-dir "$(ls dist/validmind-*.whl | head -n 1)[all]"
- name: Install Additional Dependencies
run: all-venv/bin/pip install nbformat papermill jupyter
diff --git a/.github/workflows/pypi-metrics.yaml b/.github/workflows/pypi-metrics.yaml
new file mode 100644
index 000000000..5088b201e
--- /dev/null
+++ b/.github/workflows/pypi-metrics.yaml
@@ -0,0 +1,37 @@
+# Publish the lightweight metric SDK independently from the full library.
+
+name: Publish Metrics SDK to PyPI
+
+on:
+ push:
+ tags:
+ - 'metrics-v*.*.*'
+
+permissions:
+ contents: read
+
+jobs:
+ publish:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+
+ - name: Build metrics SDK
+ run: uv build --package validmind-metrics --out-dir dist
+
+ - name: Publish metrics SDK
+ env:
+ UV_PUBLISH_USERNAME: __token__
+ UV_PUBLISH_PASSWORD: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }}
+ run: uv publish dist/validmind_metrics-*
diff --git a/.github/workflows/pypi-tracking-core.yaml b/.github/workflows/pypi-tracking-core.yaml
new file mode 100644
index 000000000..9b31f94ce
--- /dev/null
+++ b/.github/workflows/pypi-tracking-core.yaml
@@ -0,0 +1,37 @@
+# Publish the dependency-light tracking core independently from the full library.
+
+name: Publish Tracking Core to PyPI
+
+on:
+ push:
+ tags:
+ - 'tracking-core-v*.*.*'
+
+permissions:
+ contents: read
+
+jobs:
+ publish:
+ runs-on: ubuntu-latest
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - name: Set up Python 3.11
+ uses: actions/setup-python@v5
+ with:
+ python-version: '3.11'
+
+ - name: Install uv
+ uses: astral-sh/setup-uv@v5
+ with:
+ enable-cache: true
+
+ - name: Build tracking core
+ run: uv build --package validmind-tracking-core --out-dir dist
+
+ - name: Publish tracking core
+ env:
+ UV_PUBLISH_USERNAME: __token__
+ UV_PUBLISH_PASSWORD: ${{ secrets.POETRY_PYPI_TOKEN_PYPI }}
+ run: uv publish dist/validmind_tracking_core-*
diff --git a/.github/workflows/pypi.yaml b/.github/workflows/pypi.yaml
index 96b5bf2c5..4d8dd5aff 100644
--- a/.github/workflows/pypi.yaml
+++ b/.github/workflows/pypi.yaml
@@ -1,4 +1,5 @@
-# This workflow pushes the ValidMind Library package to PyPI when a tag is created
+# This workflow pushes the ValidMind Library package to PyPI when a tag is created.
+# The validmind-tracking-core package must be published first; the job checks this.
name: Publish to PyPI
@@ -33,6 +34,15 @@ jobs:
- name: Update twine
run: uv pip install --upgrade twine
+ - name: Check validmind-tracking-core is on PyPI
+ # validmind depends on validmind-tracking-core; publishing before that
+ # version exists would ship an uninstallable wheel. Publish it first by
+ # pushing a tracking-core-v* tag.
+ run: |
+ SPEC=$(python -c "import tomllib; print(next(d for d in tomllib.load(open('pyproject.toml','rb'))['project']['dependencies'] if d.startswith('validmind-tracking-core')).replace(' ', ''))")
+ echo "Resolving ${SPEC} from PyPI"
+ python -m pip download --no-deps --dest /tmp/tracking-core-check "${SPEC}"
+
- name: Publish to PyPI
env:
UV_PUBLISH_USERNAME: __token__
diff --git a/Makefile b/Makefile
index 2c64e0b94..d612eaeac 100644
--- a/Makefile
+++ b/Makefile
@@ -7,13 +7,13 @@ __check_defined = \
$(error Undefined $1$(if $2, ($2))))
format:
- uv run black validmind
- uv run isort validmind
+ uv run black validmind packages
+ uv run isort validmind packages
lint:
# don't check max line length for now since black already takes care of it
# and flake8 is too strict where it doesn't need to be
- uv run flake8 validmind --config .flake8
+ uv run flake8 validmind packages --config .flake8
install:
uv sync --all-extras --group dev
@@ -28,8 +28,13 @@ ifdef ONLY
uv run python -m unittest $(ONLY)
else
uv run python -m unittest discover tests
+ $(MAKE) test-packages
endif
+test-packages:
+ uv run --package validmind-tracking-core python -m unittest discover packages/validmind-tracking-core/tests
+ uv run --package validmind-metrics python -m unittest discover packages/validmind-metrics/tests
+
test-unit:
uv run python -m unittest "tests.test_unit_tests"
diff --git a/packages/validmind-metrics/LICENSE b/packages/validmind-metrics/LICENSE
new file mode 100644
index 000000000..a555e8aec
--- /dev/null
+++ b/packages/validmind-metrics/LICENSE
@@ -0,0 +1,687 @@
+ DUAL LICENSE NOTICE
+
+This software is dual-licensed under the GNU Affero General Public License
+version 3 (AGPL-3.0) and the ValidMind Commercial License. Users may choose
+to use the software under either of these licenses, subject to their
+respective terms and conditions.
+
+ COMMERCIAL LICENSE INQUIRY
+
+If your organization has policies regarding the use of software licensed
+under the GNU Affero General Public License, or if you are interested in
+applications beyond the scope of open-source licenses, a commercial license
+may be more appropriate. The ValidMind Commercial License offers an
+alternative to the AGPL, providing additional flexibility and benefits
+suited for commercial use.
+
+For organizations looking for a license that allows for proprietary
+development, customization, or other uses not covered under the AGPL, the
+ValidMind Commercial License is designed to meet these needs. This license
+is particularly beneficial for those seeking to integrate ValidMind software
+into their own products or services without the requirement to disclose
+proprietary source code.
+
+For inquiries regarding the licensing of this software, please contact
+ValidMind at info@validmind.com.
+
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+.
diff --git a/packages/validmind-metrics/README.md b/packages/validmind-metrics/README.md
new file mode 100644
index 000000000..cf77def18
--- /dev/null
+++ b/packages/validmind-metrics/README.md
@@ -0,0 +1,74 @@
+# ValidMind Metrics
+
+`validmind-metrics` is a lightweight client for sending unit metrics to the
+ValidMind Platform. It supports API-key authentication and OIDC device-flow
+authentication without installing or importing the full `validmind` library.
+
+## Module-level API
+
+```python
+import validmind_metrics
+
+validmind_metrics.init() # optional; reads the VM_* environment variables below
+validmind_metrics.log_metric("accuracy", 0.95)
+await validmind_metrics.alog_metric("accuracy", 0.95) # from async code
+```
+
+If `init()` is not called, the first `log_metric` / `alog_metric` call creates
+the default client from the environment. `init(**kwargs)` accepts the same
+arguments as `MetricsClient`.
+
+## Explicit client
+
+```python
+from validmind_metrics import MetricsClient
+
+client = MetricsClient(
+ api_host="https://app.validmind.ai/api/v1/tracking",
+ model="model-cuid",
+ api_key="api-key",
+ api_secret="api-secret",
+)
+
+client.log_metric("accuracy", 0.95)
+```
+
+## Environment variables
+
+| Variable | Meaning |
+| --- | --- |
+| `VM_API_HOST` / `VM_API_URL` | Tracking API URL |
+| `VM_API_MODEL` | Model CUID |
+| `VM_API_KEY`, `VM_API_SECRET` | API-key credentials |
+| `VM_OIDC_ISSUER`, `VM_OIDC_CLIENT_ID` | OIDC credentials (instead of an API key) |
+| `VM_OIDC_SCOPE`, `VM_OIDC_AUDIENCE` | Optional OIDC scope and audience |
+| `VM_API_TIMEOUT` | Request timeout in seconds (default 30) |
+
+Explicit arguments take precedence over environment variables.
+
+## Authentication in services
+
+Use API-key credentials for long-running services and HTTP handlers.
+
+OIDC uses the same `issuer`, `client_id`, optional `scope`, and optional
+`audience` settings as the full library, and caches tokens in
+`~/.validmind/credentials.json`. The client is non-interactive by default: with
+no usable cached token it raises `TrackingAuthError` rather than waiting for a
+device login. To log in, run once from a terminal with
+`init(interactive=True)` (or `MetricsClient(..., interactive=True)`); later
+processes reuse and refresh the cached token.
+
+## Async code
+
+`await alog_metric(...)` runs the blocking HTTP request in the event loop's
+default thread-pool executor, so the loop itself is not blocked. Each call
+holds one executor thread for a full round trip (up to `VM_API_TIMEOUT`); there
+is no background queue. Calling the synchronous `log_metric` from a coroutine
+blocks the loop for the duration of the request.
+
+## Errors
+
+All SDK errors derive from `TrackingError`: `TrackingConfigurationError`,
+`TrackingAuthError`, `TrackingConnectionError` (network failure or timeout),
+and `TrackingAPIError` (the API rejected the request). Invalid metric
+arguments raise `ValueError`.
diff --git a/packages/validmind-metrics/pyproject.toml b/packages/validmind-metrics/pyproject.toml
new file mode 100644
index 000000000..07ce12e37
--- /dev/null
+++ b/packages/validmind-metrics/pyproject.toml
@@ -0,0 +1,45 @@
+[build-system]
+requires = ["hatchling>=1.26.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "validmind-metrics"
+version = "0.1.0"
+description = "Lightweight metric logging client for the ValidMind Platform"
+readme = "README.md"
+requires-python = ">=3.9,<3.15"
+license = { file = "LICENSE" }
+authors = [
+ { name = "Andres Rodriguez", email = "andres@validmind.ai" },
+ { name = "Juan Martinez", email = "juan@validmind.ai" },
+ { name = "Anil Sorathiya", email = "anil@validmind.ai" },
+ { name = "Luis Pallares", email = "luis@validmind.ai" },
+ { name = "John Walz", email = "john@validmind.ai" },
+]
+classifiers = [
+ "Programming Language :: Python :: 3",
+ "Operating System :: OS Independent",
+ "Typing :: Typed",
+]
+dependencies = [
+ "validmind-tracking-core>=0.1.0,<0.2.0",
+]
+
+[tool.uv.sources]
+validmind-tracking-core = { workspace = true }
+
+[project.urls]
+Homepage = "https://validmind.com"
+Documentation = "https://docs.validmind.ai"
+Repository = "https://github.com/validmind/validmind-library"
+
+[tool.hatch.build.targets.sdist]
+include = [
+ "/src",
+ "/README.md",
+ "/LICENSE",
+ "/pyproject.toml",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/validmind_metrics"]
diff --git a/packages/validmind-metrics/src/validmind_metrics/__init__.py b/packages/validmind-metrics/src/validmind_metrics/__init__.py
new file mode 100644
index 000000000..df5f3e899
--- /dev/null
+++ b/packages/validmind-metrics/src/validmind_metrics/__init__.py
@@ -0,0 +1,98 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Lightweight metric logging client for the ValidMind Platform."""
+
+import threading
+from typing import Any, Dict, List, Optional
+
+from validmind_tracking_core import (
+ MetricsClient,
+ TrackingAPIError,
+ TrackingAuthError,
+ TrackingConfigurationError,
+ TrackingConnectionError,
+ TrackingError,
+)
+
+__version__ = "0.1.0"
+
+_client: Optional[MetricsClient] = None
+_client_lock = threading.Lock()
+
+
+def init(**kwargs: Any) -> MetricsClient:
+ """Create and retain the default metric client.
+
+ Accepts the same keyword arguments as ``MetricsClient``; anything not passed
+ is read from the ``VM_*`` environment variables.
+ """
+ global _client
+ kwargs.setdefault("client_version", __version__)
+ with _client_lock:
+ _client = MetricsClient(**kwargs)
+ return _client
+
+
+def _get_client() -> MetricsClient:
+ global _client
+ with _client_lock:
+ if _client is None:
+ _client = MetricsClient(client_version=__version__)
+ return _client
+
+
+def log_metric(
+ key: str,
+ value: Any,
+ inputs: Optional[List[str]] = None,
+ params: Optional[Dict[str, Any]] = None,
+ recorded_at: Optional[str] = None,
+ thresholds: Optional[Dict[str, Any]] = None,
+ passed: Optional[bool] = None,
+) -> Dict[str, Any]:
+ """Log one metric using the default client."""
+ return _get_client().log_metric(
+ key,
+ value,
+ inputs=inputs,
+ params=params,
+ recorded_at=recorded_at,
+ thresholds=thresholds,
+ passed=passed,
+ )
+
+
+async def alog_metric(
+ key: str,
+ value: Any,
+ inputs: Optional[List[str]] = None,
+ params: Optional[Dict[str, Any]] = None,
+ recorded_at: Optional[str] = None,
+ thresholds: Optional[Dict[str, Any]] = None,
+ passed: Optional[bool] = None,
+) -> Dict[str, Any]:
+ """Log one metric without blocking the current event loop."""
+ return await _get_client().alog_metric(
+ key,
+ value,
+ inputs=inputs,
+ params=params,
+ recorded_at=recorded_at,
+ thresholds=thresholds,
+ passed=passed,
+ )
+
+
+__all__ = [
+ "MetricsClient",
+ "TrackingAPIError",
+ "TrackingAuthError",
+ "TrackingConfigurationError",
+ "TrackingConnectionError",
+ "TrackingError",
+ "alog_metric",
+ "init",
+ "log_metric",
+]
diff --git a/packages/validmind-metrics/src/validmind_metrics/py.typed b/packages/validmind-metrics/src/validmind_metrics/py.typed
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/validmind-metrics/tests/test_metrics.py b/packages/validmind-metrics/tests/test_metrics.py
new file mode 100644
index 000000000..01c404da3
--- /dev/null
+++ b/packages/validmind-metrics/tests/test_metrics.py
@@ -0,0 +1,144 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import asyncio
+import json
+import subprocess
+import sys
+import unittest
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest.mock import Mock, patch
+
+from validmind_metrics import MetricsClient
+from validmind_tracking_core.credentials_store import upsert_cached_entry
+
+
+class TestMetricsClient(unittest.TestCase):
+ def _response(self, body=None, status_code=200, text=""):
+ response = Mock()
+ response.status_code = status_code
+ response.text = text
+ response.json.return_value = body or {"ok": True}
+ return response
+
+ @patch("validmind_tracking_core.metrics.requests.post")
+ def test_log_metric_uses_api_key_headers_and_schema(self, mock_post):
+ mock_post.return_value = self._response({"metric_id": "metric-1"})
+ client = MetricsClient(
+ api_host="https://tracking.example/api/v1/tracking",
+ model="model-1",
+ api_key="key",
+ api_secret="secret",
+ monitoring=True,
+ document="monitoring",
+ client_version="test-client/1.0",
+ )
+
+ result = client.log_metric(
+ "accuracy",
+ 0.95,
+ inputs=["dataset-1"],
+ params={"average": "macro"},
+ recorded_at="2026-08-27T00:00:00Z",
+ thresholds={"minimum": 0.9},
+ passed=True,
+ )
+
+ self.assertEqual(result, {"metric_id": "metric-1"})
+ mock_post.assert_called_once()
+ url = mock_post.call_args.args[0]
+ kwargs = mock_post.call_args.kwargs
+ self.assertEqual(
+ url, "https://tracking.example/api/v1/tracking/log_unit_metric"
+ )
+ self.assertEqual(
+ kwargs["headers"],
+ {
+ "X-MODEL-CUID": "model-1",
+ "X-MONITORING": "True",
+ "X-LIBRARY-VERSION": "test-client/1.0",
+ "X-DOCUMENT-TYPE": "monitoring",
+ "X-API-KEY": "key",
+ "X-API-SECRET": "secret",
+ },
+ )
+ self.assertEqual(
+ json.loads(kwargs["data"]),
+ {
+ "key": "accuracy",
+ "value": 0.95,
+ "inputs": ["dataset-1"],
+ "params": {"average": "macro"},
+ "recorded_at": "2026-08-27T00:00:00Z",
+ "thresholds": {"minimum": 0.9},
+ "passed": True,
+ },
+ )
+
+ @patch("validmind_tracking_core.metrics.requests.post")
+ def test_cached_oidc_token_is_used(self, mock_post):
+ mock_post.return_value = self._response({"metric_id": "metric-2"})
+ with TemporaryDirectory() as temp_dir:
+ credentials_path = Path(temp_dir) / "credentials.json"
+ upsert_cached_entry(
+ "https://issuer.example",
+ "client-1",
+ {
+ "access_token": "cached-token",
+ "refresh_token": "refresh-token",
+ "expires_at": (
+ datetime.now(timezone.utc) + timedelta(hours=1)
+ ).isoformat(),
+ },
+ path=credentials_path,
+ )
+
+ client = MetricsClient(
+ api_host="https://tracking.example/api/v1/tracking",
+ model="model-1",
+ issuer="https://issuer.example/",
+ client_id="client-1",
+ credentials_path=credentials_path,
+ )
+ client.log_metric("accuracy", 0.95)
+
+ headers = mock_post.call_args.kwargs["headers"]
+ self.assertEqual(headers["Authorization"], "Bearer cached-token")
+ self.assertNotIn("X-API-KEY", headers)
+
+ @patch("validmind_tracking_core.metrics.requests.post")
+ def test_async_metric_does_not_require_nested_event_loop(self, mock_post):
+ mock_post.return_value = self._response({"ok": True})
+ client = MetricsClient(
+ api_host="https://tracking.example/api/v1/tracking",
+ model="model-1",
+ api_key="key",
+ api_secret="secret",
+ )
+
+ async def handler():
+ return await client.alog_metric("accuracy", 0.95)
+
+ self.assertEqual(asyncio.run(handler()), {"ok": True})
+
+ def test_import_isolated_from_full_library(self):
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-c",
+ "import sys; import validmind_metrics; "
+ "assert 'validmind' not in sys.modules; "
+ "assert 'aiohttp' not in sys.modules",
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ self.assertEqual(result.stderr, "")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/packages/validmind-tracking-core/LICENSE b/packages/validmind-tracking-core/LICENSE
new file mode 100644
index 000000000..a555e8aec
--- /dev/null
+++ b/packages/validmind-tracking-core/LICENSE
@@ -0,0 +1,687 @@
+ DUAL LICENSE NOTICE
+
+This software is dual-licensed under the GNU Affero General Public License
+version 3 (AGPL-3.0) and the ValidMind Commercial License. Users may choose
+to use the software under either of these licenses, subject to their
+respective terms and conditions.
+
+ COMMERCIAL LICENSE INQUIRY
+
+If your organization has policies regarding the use of software licensed
+under the GNU Affero General Public License, or if you are interested in
+applications beyond the scope of open-source licenses, a commercial license
+may be more appropriate. The ValidMind Commercial License offers an
+alternative to the AGPL, providing additional flexibility and benefits
+suited for commercial use.
+
+For organizations looking for a license that allows for proprietary
+development, customization, or other uses not covered under the AGPL, the
+ValidMind Commercial License is designed to meet these needs. This license
+is particularly beneficial for those seeking to integrate ValidMind software
+into their own products or services without the requirement to disclose
+proprietary source code.
+
+For inquiries regarding the licensing of this software, please contact
+ValidMind at info@validmind.com.
+
+ GNU AFFERO GENERAL PUBLIC LICENSE
+ Version 3, 19 November 2007
+
+ Copyright (C) 2007 Free Software Foundation, Inc.
+ Everyone is permitted to copy and distribute verbatim copies
+ of this license document, but changing it is not allowed.
+
+ Preamble
+
+ The GNU Affero General Public License is a free, copyleft license for
+software and other kinds of works, specifically designed to ensure
+cooperation with the community in the case of network server software.
+
+ The licenses for most software and other practical works are designed
+to take away your freedom to share and change the works. By contrast,
+our General Public Licenses are intended to guarantee your freedom to
+share and change all versions of a program--to make sure it remains free
+software for all its users.
+
+ When we speak of free software, we are referring to freedom, not
+price. Our General Public Licenses are designed to make sure that you
+have the freedom to distribute copies of free software (and charge for
+them if you wish), that you receive source code or can get it if you
+want it, that you can change the software or use pieces of it in new
+free programs, and that you know you can do these things.
+
+ Developers that use our General Public Licenses protect your rights
+with two steps: (1) assert copyright on the software, and (2) offer
+you this License which gives you legal permission to copy, distribute
+and/or modify the software.
+
+ A secondary benefit of defending all users' freedom is that
+improvements made in alternate versions of the program, if they
+receive widespread use, become available for other developers to
+incorporate. Many developers of free software are heartened and
+encouraged by the resulting cooperation. However, in the case of
+software used on network servers, this result may fail to come about.
+The GNU General Public License permits making a modified version and
+letting the public access it on a server without ever releasing its
+source code to the public.
+
+ The GNU Affero General Public License is designed specifically to
+ensure that, in such cases, the modified source code becomes available
+to the community. It requires the operator of a network server to
+provide the source code of the modified version running there to the
+users of that server. Therefore, public use of a modified version, on
+a publicly accessible server, gives the public access to the source
+code of the modified version.
+
+ An older license, called the Affero General Public License and
+published by Affero, was designed to accomplish similar goals. This is
+a different license, not a version of the Affero GPL, but Affero has
+released a new version of the Affero GPL which permits relicensing under
+this license.
+
+ The precise terms and conditions for copying, distribution and
+modification follow.
+
+ TERMS AND CONDITIONS
+
+ 0. Definitions.
+
+ "This License" refers to version 3 of the GNU Affero General Public License.
+
+ "Copyright" also means copyright-like laws that apply to other kinds of
+works, such as semiconductor masks.
+
+ "The Program" refers to any copyrightable work licensed under this
+License. Each licensee is addressed as "you". "Licensees" and
+"recipients" may be individuals or organizations.
+
+ To "modify" a work means to copy from or adapt all or part of the work
+in a fashion requiring copyright permission, other than the making of an
+exact copy. The resulting work is called a "modified version" of the
+earlier work or a work "based on" the earlier work.
+
+ A "covered work" means either the unmodified Program or a work based
+on the Program.
+
+ To "propagate" a work means to do anything with it that, without
+permission, would make you directly or secondarily liable for
+infringement under applicable copyright law, except executing it on a
+computer or modifying a private copy. Propagation includes copying,
+distribution (with or without modification), making available to the
+public, and in some countries other activities as well.
+
+ To "convey" a work means any kind of propagation that enables other
+parties to make or receive copies. Mere interaction with a user through
+a computer network, with no transfer of a copy, is not conveying.
+
+ An interactive user interface displays "Appropriate Legal Notices"
+to the extent that it includes a convenient and prominently visible
+feature that (1) displays an appropriate copyright notice, and (2)
+tells the user that there is no warranty for the work (except to the
+extent that warranties are provided), that licensees may convey the
+work under this License, and how to view a copy of this License. If
+the interface presents a list of user commands or options, such as a
+menu, a prominent item in the list meets this criterion.
+
+ 1. Source Code.
+
+ The "source code" for a work means the preferred form of the work
+for making modifications to it. "Object code" means any non-source
+form of a work.
+
+ A "Standard Interface" means an interface that either is an official
+standard defined by a recognized standards body, or, in the case of
+interfaces specified for a particular programming language, one that
+is widely used among developers working in that language.
+
+ The "System Libraries" of an executable work include anything, other
+than the work as a whole, that (a) is included in the normal form of
+packaging a Major Component, but which is not part of that Major
+Component, and (b) serves only to enable use of the work with that
+Major Component, or to implement a Standard Interface for which an
+implementation is available to the public in source code form. A
+"Major Component", in this context, means a major essential component
+(kernel, window system, and so on) of the specific operating system
+(if any) on which the executable work runs, or a compiler used to
+produce the work, or an object code interpreter used to run it.
+
+ The "Corresponding Source" for a work in object code form means all
+the source code needed to generate, install, and (for an executable
+work) run the object code and to modify the work, including scripts to
+control those activities. However, it does not include the work's
+System Libraries, or general-purpose tools or generally available free
+programs which are used unmodified in performing those activities but
+which are not part of the work. For example, Corresponding Source
+includes interface definition files associated with source files for
+the work, and the source code for shared libraries and dynamically
+linked subprograms that the work is specifically designed to require,
+such as by intimate data communication or control flow between those
+subprograms and other parts of the work.
+
+ The Corresponding Source need not include anything that users
+can regenerate automatically from other parts of the Corresponding
+Source.
+
+ The Corresponding Source for a work in source code form is that
+same work.
+
+ 2. Basic Permissions.
+
+ All rights granted under this License are granted for the term of
+copyright on the Program, and are irrevocable provided the stated
+conditions are met. This License explicitly affirms your unlimited
+permission to run the unmodified Program. The output from running a
+covered work is covered by this License only if the output, given its
+content, constitutes a covered work. This License acknowledges your
+rights of fair use or other equivalent, as provided by copyright law.
+
+ You may make, run and propagate covered works that you do not
+convey, without conditions so long as your license otherwise remains
+in force. You may convey covered works to others for the sole purpose
+of having them make modifications exclusively for you, or provide you
+with facilities for running those works, provided that you comply with
+the terms of this License in conveying all material for which you do
+not control copyright. Those thus making or running the covered works
+for you must do so exclusively on your behalf, under your direction
+and control, on terms that prohibit them from making any copies of
+your copyrighted material outside their relationship with you.
+
+ Conveying under any other circumstances is permitted solely under
+the conditions stated below. Sublicensing is not allowed; section 10
+makes it unnecessary.
+
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
+
+ No covered work shall be deemed part of an effective technological
+measure under any applicable law fulfilling obligations under article
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
+similar laws prohibiting or restricting circumvention of such
+measures.
+
+ When you convey a covered work, you waive any legal power to forbid
+circumvention of technological measures to the extent such circumvention
+is effected by exercising rights under this License with respect to
+the covered work, and you disclaim any intention to limit operation or
+modification of the work as a means of enforcing, against the work's
+users, your or third parties' legal rights to forbid circumvention of
+technological measures.
+
+ 4. Conveying Verbatim Copies.
+
+ You may convey verbatim copies of the Program's source code as you
+receive it, in any medium, provided that you conspicuously and
+appropriately publish on each copy an appropriate copyright notice;
+keep intact all notices stating that this License and any
+non-permissive terms added in accord with section 7 apply to the code;
+keep intact all notices of the absence of any warranty; and give all
+recipients a copy of this License along with the Program.
+
+ You may charge any price or no price for each copy that you convey,
+and you may offer support or warranty protection for a fee.
+
+ 5. Conveying Modified Source Versions.
+
+ You may convey a work based on the Program, or the modifications to
+produce it from the Program, in the form of source code under the
+terms of section 4, provided that you also meet all of these conditions:
+
+ a) The work must carry prominent notices stating that you modified
+ it, and giving a relevant date.
+
+ b) The work must carry prominent notices stating that it is
+ released under this License and any conditions added under section
+ 7. This requirement modifies the requirement in section 4 to
+ "keep intact all notices".
+
+ c) You must license the entire work, as a whole, under this
+ License to anyone who comes into possession of a copy. This
+ License will therefore apply, along with any applicable section 7
+ additional terms, to the whole of the work, and all its parts,
+ regardless of how they are packaged. This License gives no
+ permission to license the work in any other way, but it does not
+ invalidate such permission if you have separately received it.
+
+ d) If the work has interactive user interfaces, each must display
+ Appropriate Legal Notices; however, if the Program has interactive
+ interfaces that do not display Appropriate Legal Notices, your
+ work need not make them do so.
+
+ A compilation of a covered work with other separate and independent
+works, which are not by their nature extensions of the covered work,
+and which are not combined with it such as to form a larger program,
+in or on a volume of a storage or distribution medium, is called an
+"aggregate" if the compilation and its resulting copyright are not
+used to limit the access or legal rights of the compilation's users
+beyond what the individual works permit. Inclusion of a covered work
+in an aggregate does not cause this License to apply to the other
+parts of the aggregate.
+
+ 6. Conveying Non-Source Forms.
+
+ You may convey a covered work in object code form under the terms
+of sections 4 and 5, provided that you also convey the
+machine-readable Corresponding Source under the terms of this License,
+in one of these ways:
+
+ a) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by the
+ Corresponding Source fixed on a durable physical medium
+ customarily used for software interchange.
+
+ b) Convey the object code in, or embodied in, a physical product
+ (including a physical distribution medium), accompanied by a
+ written offer, valid for at least three years and valid for as
+ long as you offer spare parts or customer support for that product
+ model, to give anyone who possesses the object code either (1) a
+ copy of the Corresponding Source for all the software in the
+ product that is covered by this License, on a durable physical
+ medium customarily used for software interchange, for a price no
+ more than your reasonable cost of physically performing this
+ conveying of source, or (2) access to copy the
+ Corresponding Source from a network server at no charge.
+
+ c) Convey individual copies of the object code with a copy of the
+ written offer to provide the Corresponding Source. This
+ alternative is allowed only occasionally and noncommercially, and
+ only if you received the object code with such an offer, in accord
+ with subsection 6b.
+
+ d) Convey the object code by offering access from a designated
+ place (gratis or for a charge), and offer equivalent access to the
+ Corresponding Source in the same way through the same place at no
+ further charge. You need not require recipients to copy the
+ Corresponding Source along with the object code. If the place to
+ copy the object code is a network server, the Corresponding Source
+ may be on a different server (operated by you or a third party)
+ that supports equivalent copying facilities, provided you maintain
+ clear directions next to the object code saying where to find the
+ Corresponding Source. Regardless of what server hosts the
+ Corresponding Source, you remain obligated to ensure that it is
+ available for as long as needed to satisfy these requirements.
+
+ e) Convey the object code using peer-to-peer transmission, provided
+ you inform other peers where the object code and Corresponding
+ Source of the work are being offered to the general public at no
+ charge under subsection 6d.
+
+ A separable portion of the object code, whose source code is excluded
+from the Corresponding Source as a System Library, need not be
+included in conveying the object code work.
+
+ A "User Product" is either (1) a "consumer product", which means any
+tangible personal property which is normally used for personal, family,
+or household purposes, or (2) anything designed or sold for incorporation
+into a dwelling. In determining whether a product is a consumer product,
+doubtful cases shall be resolved in favor of coverage. For a particular
+product received by a particular user, "normally used" refers to a
+typical or common use of that class of product, regardless of the status
+of the particular user or of the way in which the particular user
+actually uses, or expects or is expected to use, the product. A product
+is a consumer product regardless of whether the product has substantial
+commercial, industrial or non-consumer uses, unless such uses represent
+the only significant mode of use of the product.
+
+ "Installation Information" for a User Product means any methods,
+procedures, authorization keys, or other information required to install
+and execute modified versions of a covered work in that User Product from
+a modified version of its Corresponding Source. The information must
+suffice to ensure that the continued functioning of the modified object
+code is in no case prevented or interfered with solely because
+modification has been made.
+
+ If you convey an object code work under this section in, or with, or
+specifically for use in, a User Product, and the conveying occurs as
+part of a transaction in which the right of possession and use of the
+User Product is transferred to the recipient in perpetuity or for a
+fixed term (regardless of how the transaction is characterized), the
+Corresponding Source conveyed under this section must be accompanied
+by the Installation Information. But this requirement does not apply
+if neither you nor any third party retains the ability to install
+modified object code on the User Product (for example, the work has
+been installed in ROM).
+
+ The requirement to provide Installation Information does not include a
+requirement to continue to provide support service, warranty, or updates
+for a work that has been modified or installed by the recipient, or for
+the User Product in which it has been modified or installed. Access to a
+network may be denied when the modification itself materially and
+adversely affects the operation of the network or violates the rules and
+protocols for communication across the network.
+
+ Corresponding Source conveyed, and Installation Information provided,
+in accord with this section must be in a format that is publicly
+documented (and with an implementation available to the public in
+source code form), and must require no special password or key for
+unpacking, reading or copying.
+
+ 7. Additional Terms.
+
+ "Additional permissions" are terms that supplement the terms of this
+License by making exceptions from one or more of its conditions.
+Additional permissions that are applicable to the entire Program shall
+be treated as though they were included in this License, to the extent
+that they are valid under applicable law. If additional permissions
+apply only to part of the Program, that part may be used separately
+under those permissions, but the entire Program remains governed by
+this License without regard to the additional permissions.
+
+ When you convey a copy of a covered work, you may at your option
+remove any additional permissions from that copy, or from any part of
+it. (Additional permissions may be written to require their own
+removal in certain cases when you modify the work.) You may place
+additional permissions on material, added by you to a covered work,
+for which you have or can give appropriate copyright permission.
+
+ Notwithstanding any other provision of this License, for material you
+add to a covered work, you may (if authorized by the copyright holders of
+that material) supplement the terms of this License with terms:
+
+ a) Disclaiming warranty or limiting liability differently from the
+ terms of sections 15 and 16 of this License; or
+
+ b) Requiring preservation of specified reasonable legal notices or
+ author attributions in that material or in the Appropriate Legal
+ Notices displayed by works containing it; or
+
+ c) Prohibiting misrepresentation of the origin of that material, or
+ requiring that modified versions of such material be marked in
+ reasonable ways as different from the original version; or
+
+ d) Limiting the use for publicity purposes of names of licensors or
+ authors of the material; or
+
+ e) Declining to grant rights under trademark law for use of some
+ trade names, trademarks, or service marks; or
+
+ f) Requiring indemnification of licensors and authors of that
+ material by anyone who conveys the material (or modified versions of
+ it) with contractual assumptions of liability to the recipient, for
+ any liability that these contractual assumptions directly impose on
+ those licensors and authors.
+
+ All other non-permissive additional terms are considered "further
+restrictions" within the meaning of section 10. If the Program as you
+received it, or any part of it, contains a notice stating that it is
+governed by this License along with a term that is a further
+restriction, you may remove that term. If a license document contains
+a further restriction but permits relicensing or conveying under this
+License, you may add to a covered work material governed by the terms
+of that license document, provided that the further restriction does
+not survive such relicensing or conveying.
+
+ If you add terms to a covered work in accord with this section, you
+must place, in the relevant source files, a statement of the
+additional terms that apply to those files, or a notice indicating
+where to find the applicable terms.
+
+ Additional terms, permissive or non-permissive, may be stated in the
+form of a separately written license, or stated as exceptions;
+the above requirements apply either way.
+
+ 8. Termination.
+
+ You may not propagate or modify a covered work except as expressly
+provided under this License. Any attempt otherwise to propagate or
+modify it is void, and will automatically terminate your rights under
+this License (including any patent licenses granted under the third
+paragraph of section 11).
+
+ However, if you cease all violation of this License, then your
+license from a particular copyright holder is reinstated (a)
+provisionally, unless and until the copyright holder explicitly and
+finally terminates your license, and (b) permanently, if the copyright
+holder fails to notify you of the violation by some reasonable means
+prior to 60 days after the cessation.
+
+ Moreover, your license from a particular copyright holder is
+reinstated permanently if the copyright holder notifies you of the
+violation by some reasonable means, this is the first time you have
+received notice of violation of this License (for any work) from that
+copyright holder, and you cure the violation prior to 30 days after
+your receipt of the notice.
+
+ Termination of your rights under this section does not terminate the
+licenses of parties who have received copies or rights from you under
+this License. If your rights have been terminated and not permanently
+reinstated, you do not qualify to receive new licenses for the same
+material under section 10.
+
+ 9. Acceptance Not Required for Having Copies.
+
+ You are not required to accept this License in order to receive or
+run a copy of the Program. Ancillary propagation of a covered work
+occurring solely as a consequence of using peer-to-peer transmission
+to receive a copy likewise does not require acceptance. However,
+nothing other than this License grants you permission to propagate or
+modify any covered work. These actions infringe copyright if you do
+not accept this License. Therefore, by modifying or propagating a
+covered work, you indicate your acceptance of this License to do so.
+
+ 10. Automatic Licensing of Downstream Recipients.
+
+ Each time you convey a covered work, the recipient automatically
+receives a license from the original licensors, to run, modify and
+propagate that work, subject to this License. You are not responsible
+for enforcing compliance by third parties with this License.
+
+ An "entity transaction" is a transaction transferring control of an
+organization, or substantially all assets of one, or subdividing an
+organization, or merging organizations. If propagation of a covered
+work results from an entity transaction, each party to that
+transaction who receives a copy of the work also receives whatever
+licenses to the work the party's predecessor in interest had or could
+give under the previous paragraph, plus a right to possession of the
+Corresponding Source of the work from the predecessor in interest, if
+the predecessor has it or can get it with reasonable efforts.
+
+ You may not impose any further restrictions on the exercise of the
+rights granted or affirmed under this License. For example, you may
+not impose a license fee, royalty, or other charge for exercise of
+rights granted under this License, and you may not initiate litigation
+(including a cross-claim or counterclaim in a lawsuit) alleging that
+any patent claim is infringed by making, using, selling, offering for
+sale, or importing the Program or any portion of it.
+
+ 11. Patents.
+
+ A "contributor" is a copyright holder who authorizes use under this
+License of the Program or a work on which the Program is based. The
+work thus licensed is called the contributor's "contributor version".
+
+ A contributor's "essential patent claims" are all patent claims
+owned or controlled by the contributor, whether already acquired or
+hereafter acquired, that would be infringed by some manner, permitted
+by this License, of making, using, or selling its contributor version,
+but do not include claims that would be infringed only as a
+consequence of further modification of the contributor version. For
+purposes of this definition, "control" includes the right to grant
+patent sublicenses in a manner consistent with the requirements of
+this License.
+
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
+patent license under the contributor's essential patent claims, to
+make, use, sell, offer for sale, import and otherwise run, modify and
+propagate the contents of its contributor version.
+
+ In the following three paragraphs, a "patent license" is any express
+agreement or commitment, however denominated, not to enforce a patent
+(such as an express permission to practice a patent or covenant not to
+sue for patent infringement). To "grant" such a patent license to a
+party means to make such an agreement or commitment not to enforce a
+patent against the party.
+
+ If you convey a covered work, knowingly relying on a patent license,
+and the Corresponding Source of the work is not available for anyone
+to copy, free of charge and under the terms of this License, through a
+publicly available network server or other readily accessible means,
+then you must either (1) cause the Corresponding Source to be so
+available, or (2) arrange to deprive yourself of the benefit of the
+patent license for this particular work, or (3) arrange, in a manner
+consistent with the requirements of this License, to extend the patent
+license to downstream recipients. "Knowingly relying" means you have
+actual knowledge that, but for the patent license, your conveying the
+covered work in a country, or your recipient's use of the covered work
+in a country, would infringe one or more identifiable patents in that
+country that you have reason to believe are valid.
+
+ If, pursuant to or in connection with a single transaction or
+arrangement, you convey, or propagate by procuring conveyance of, a
+covered work, and grant a patent license to some of the parties
+receiving the covered work authorizing them to use, propagate, modify
+or convey a specific copy of the covered work, then the patent license
+you grant is automatically extended to all recipients of the covered
+work and works based on it.
+
+ A patent license is "discriminatory" if it does not include within
+the scope of its coverage, prohibits the exercise of, or is
+conditioned on the non-exercise of one or more of the rights that are
+specifically granted under this License. You may not convey a covered
+work if you are a party to an arrangement with a third party that is
+in the business of distributing software, under which you make payment
+to the third party based on the extent of your activity of conveying
+the work, and under which the third party grants, to any of the
+parties who would receive the covered work from you, a discriminatory
+patent license (a) in connection with copies of the covered work
+conveyed by you (or copies made from those copies), or (b) primarily
+for and in connection with specific products or compilations that
+contain the covered work, unless you entered into that arrangement,
+or that patent license was granted, prior to 28 March 2007.
+
+ Nothing in this License shall be construed as excluding or limiting
+any implied license or other defenses to infringement that may
+otherwise be available to you under applicable patent law.
+
+ 12. No Surrender of Others' Freedom.
+
+ If conditions are imposed on you (whether by court order, agreement or
+otherwise) that contradict the conditions of this License, they do not
+excuse you from the conditions of this License. If you cannot convey a
+covered work so as to satisfy simultaneously your obligations under this
+License and any other pertinent obligations, then as a consequence you may
+not convey it at all. For example, if you agree to terms that obligate you
+to collect a royalty for further conveying from those to whom you convey
+the Program, the only way you could satisfy both those terms and this
+License would be to refrain entirely from conveying the Program.
+
+ 13. Remote Network Interaction; Use with the GNU General Public License.
+
+ Notwithstanding any other provision of this License, if you modify the
+Program, your modified version must prominently offer all users
+interacting with it remotely through a computer network (if your version
+supports such interaction) an opportunity to receive the Corresponding
+Source of your version by providing access to the Corresponding Source
+from a network server at no charge, through some standard or customary
+means of facilitating copying of software. This Corresponding Source
+shall include the Corresponding Source for any work covered by version 3
+of the GNU General Public License that is incorporated pursuant to the
+following paragraph.
+
+ Notwithstanding any other provision of this License, you have
+permission to link or combine any covered work with a work licensed
+under version 3 of the GNU General Public License into a single
+combined work, and to convey the resulting work. The terms of this
+License will continue to apply to the part which is the covered work,
+but the work with which it is combined will remain governed by version
+3 of the GNU General Public License.
+
+ 14. Revised Versions of this License.
+
+ The Free Software Foundation may publish revised and/or new versions of
+the GNU Affero General Public License from time to time. Such new versions
+will be similar in spirit to the present version, but may differ in detail to
+address new problems or concerns.
+
+ Each version is given a distinguishing version number. If the
+Program specifies that a certain numbered version of the GNU Affero General
+Public License "or any later version" applies to it, you have the
+option of following the terms and conditions either of that numbered
+version or of any later version published by the Free Software
+Foundation. If the Program does not specify a version number of the
+GNU Affero General Public License, you may choose any version ever published
+by the Free Software Foundation.
+
+ If the Program specifies that a proxy can decide which future
+versions of the GNU Affero General Public License can be used, that proxy's
+public statement of acceptance of a version permanently authorizes you
+to choose that version for the Program.
+
+ Later license versions may give you additional or different
+permissions. However, no additional obligations are imposed on any
+author or copyright holder as a result of your choosing to follow a
+later version.
+
+ 15. Disclaimer of Warranty.
+
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
+
+ 16. Limitation of Liability.
+
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
+SUCH DAMAGES.
+
+ 17. Interpretation of Sections 15 and 16.
+
+ If the disclaimer of warranty and limitation of liability provided
+above cannot be given local legal effect according to their terms,
+reviewing courts shall apply local law that most closely approximates
+an absolute waiver of all civil liability in connection with the
+Program, unless a warranty or assumption of liability accompanies a
+copy of the Program in return for a fee.
+
+ END OF TERMS AND CONDITIONS
+
+ How to Apply These Terms to Your New Programs
+
+ If you develop a new program, and you want it to be of the greatest
+possible use to the public, the best way to achieve this is to make it
+free software which everyone can redistribute and change under these terms.
+
+ To do so, attach the following notices to the program. It is safest
+to attach them to the start of each source file to most effectively
+state the exclusion of warranty; and each file should have at least
+the "copyright" line and a pointer to where the full notice is found.
+
+
+ Copyright (C)
+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License as published
+ by the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with this program. If not, see .
+
+Also add information on how to contact you by electronic and paper mail.
+
+ If your software can interact with users remotely through a computer
+network, you should also make sure that it provides a way for users to
+get its source. For example, if your program is a web application, its
+interface could display a "Source" link that leads users to an archive
+of the code. There are many ways you could offer source, and different
+solutions will be better for different programs; see section 13 for the
+specific requirements.
+
+ You should also get your employer (if you work as a programmer) or school,
+if any, to sign a "copyright disclaimer" for the program, if necessary.
+For more information on this, and how to apply and follow the GNU AGPL, see
+.
diff --git a/packages/validmind-tracking-core/README.md b/packages/validmind-tracking-core/README.md
new file mode 100644
index 000000000..f90868284
--- /dev/null
+++ b/packages/validmind-tracking-core/README.md
@@ -0,0 +1,8 @@
+# ValidMind Tracking Core
+
+`validmind-tracking-core` contains the dependency-light authentication and tracking
+transport shared by ValidMind SDKs. It supports API-key authentication and OIDC
+device-flow authentication without importing the full `validmind` package.
+
+This package is an implementation dependency of ValidMind SDKs. Application code
+should normally use `validmind-metrics` or `validmind` directly.
diff --git a/packages/validmind-tracking-core/pyproject.toml b/packages/validmind-tracking-core/pyproject.toml
new file mode 100644
index 000000000..2c306bf48
--- /dev/null
+++ b/packages/validmind-tracking-core/pyproject.toml
@@ -0,0 +1,42 @@
+[build-system]
+requires = ["hatchling>=1.26.0"]
+build-backend = "hatchling.build"
+
+[project]
+name = "validmind-tracking-core"
+version = "0.1.0"
+description = "Dependency-light authentication and tracking transport for ValidMind SDKs"
+readme = "README.md"
+requires-python = ">=3.9,<3.15"
+license = { file = "LICENSE" }
+authors = [
+ { name = "Andres Rodriguez", email = "andres@validmind.ai" },
+ { name = "Juan Martinez", email = "juan@validmind.ai" },
+ { name = "Anil Sorathiya", email = "anil@validmind.ai" },
+ { name = "Luis Pallares", email = "luis@validmind.ai" },
+ { name = "John Walz", email = "john@validmind.ai" },
+]
+classifiers = [
+ "Programming Language :: Python :: 3",
+ "Operating System :: OS Independent",
+ "Typing :: Typed",
+]
+dependencies = [
+ "requests (>=2.28.0,<3.0.0)",
+]
+
+[project.urls]
+Homepage = "https://validmind.com"
+Documentation = "https://docs.validmind.ai"
+Repository = "https://github.com/validmind/validmind-library"
+
+[tool.hatch.build.targets.sdist]
+include = [
+ "/src",
+ "/README.md",
+ "/LICENSE",
+ "/pyproject.toml",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/validmind_tracking_core"]
diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/__init__.py b/packages/validmind-tracking-core/src/validmind_tracking_core/__init__.py
new file mode 100644
index 000000000..e1e6e42c1
--- /dev/null
+++ b/packages/validmind-tracking-core/src/validmind_tracking_core/__init__.py
@@ -0,0 +1,25 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Dependency-light authentication and tracking primitives for ValidMind SDKs."""
+
+from .errors import (
+ TrackingAPIError,
+ TrackingAuthError,
+ TrackingConfigurationError,
+ TrackingConnectionError,
+ TrackingError,
+)
+from .metrics import MetricsClient, post_metric, serialize_metric
+
+__all__ = [
+ "MetricsClient",
+ "TrackingAPIError",
+ "TrackingAuthError",
+ "TrackingConfigurationError",
+ "TrackingConnectionError",
+ "TrackingError",
+ "post_metric",
+ "serialize_metric",
+]
diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/credentials_store.py b/packages/validmind-tracking-core/src/validmind_tracking_core/credentials_store.py
new file mode 100644
index 000000000..195bd9bcb
--- /dev/null
+++ b/packages/validmind-tracking-core/src/validmind_tracking_core/credentials_store.py
@@ -0,0 +1,203 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Small, file-backed OIDC credential store used by the tracking SDKs."""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import tempfile
+from contextlib import contextmanager
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from typing import Any, Dict, Iterator, Optional
+
+try:
+ import fcntl
+except ImportError: # Windows
+ fcntl = None
+
+from .errors import TrackingAuthError
+
+_CREDENTIALS_VERSION = 1
+
+
+def normalize_issuer(issuer: str) -> str:
+ base = issuer.strip().rstrip("/")
+ while len(base) >= 2 and base[0] == base[-1] and base[0] in ('"', "'"):
+ base = base[1:-1].strip().rstrip("/")
+ return base
+
+
+def normalize_client_id(client_id: str) -> str:
+ base = client_id.strip()
+ while len(base) >= 2 and base[0] == base[-1] and base[0] in ('"', "'"):
+ base = base[1:-1].strip()
+ return base
+
+
+def normalize_audience(audience: Optional[str]) -> str:
+ if not audience:
+ return ""
+ base = audience.strip()
+ while len(base) >= 2 and base[0] == base[-1] and base[0] in ('"', "'"):
+ base = base[1:-1].strip()
+ return base
+
+
+def credential_key(issuer: str, client_id: str, audience: Optional[str] = None) -> str:
+ base = f"{normalize_issuer(issuer)}|{normalize_client_id(client_id)}"
+ aud = normalize_audience(audience)
+ return f"{base}|{aud}" if aud else base
+
+
+def credentials_path() -> Path:
+ return Path.home() / ".validmind" / "credentials.json"
+
+
+def _empty_store() -> Dict[str, Any]:
+ return {"version": _CREDENTIALS_VERSION, "credentials": {}}
+
+
+def load_credentials_file(path: Optional[Path] = None) -> Dict[str, Any]:
+ path = path or credentials_path()
+ if not path.is_file():
+ return _empty_store()
+ try:
+ with open(path, encoding="utf-8") as handle:
+ data = json.load(handle)
+ except (json.JSONDecodeError, OSError) as exc:
+ raise TrackingAuthError(
+ f"Could not read credentials file {path}: {exc}"
+ ) from exc
+ if not isinstance(data, dict):
+ raise TrackingAuthError(f"Invalid credentials file format at {path}")
+ data.setdefault("version", _CREDENTIALS_VERSION)
+ data.setdefault("credentials", {})
+ return data
+
+
+def _atomic_write(path: Path, payload: Dict[str, Any]) -> None:
+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+ fd, temp_name = tempfile.mkstemp(
+ dir=str(path.parent), prefix=".credentials-", suffix=".tmp", text=True
+ )
+ temp_path = Path(temp_name)
+ try:
+ with os.fdopen(fd, "w", encoding="utf-8") as handle:
+ json.dump(payload, handle, indent=2)
+ os.chmod(temp_path, 0o600)
+ os.replace(temp_path, path)
+ except Exception:
+ try:
+ temp_path.unlink()
+ except OSError:
+ pass
+ raise
+
+
+def save_credentials_file(data: Dict[str, Any], path: Optional[Path] = None) -> None:
+ path = path or credentials_path()
+ normalized = dict(data)
+ normalized["version"] = _CREDENTIALS_VERSION
+ if not isinstance(normalized.get("credentials"), dict):
+ normalized["credentials"] = {}
+ _atomic_write(path, normalized)
+
+
+@contextmanager
+def _locked(path: Path) -> Iterator[None]:
+ """Hold an exclusive lock across a read-modify-write of the credentials file.
+
+ A sidecar lock file is used because the credentials file itself is replaced
+ atomically on every write.
+ """
+ # ponytail: no cross-process lock on Windows (no fcntl); add msvcrt.locking if
+ # multi-worker Windows services need it.
+ if fcntl is None:
+ yield
+ return
+ path.parent.mkdir(mode=0o700, parents=True, exist_ok=True)
+ with open(path.with_name(path.name + ".lock"), "a") as handle:
+ fcntl.flock(handle, fcntl.LOCK_EX)
+ try:
+ yield
+ finally:
+ fcntl.flock(handle, fcntl.LOCK_UN)
+
+
+def get_cached_entry(
+ issuer: str,
+ client_id: str,
+ path: Optional[Path] = None,
+ audience: Optional[str] = None,
+) -> Optional[Dict[str, Any]]:
+ key = credential_key(issuer, client_id, audience)
+ entry = load_credentials_file(path).get("credentials", {}).get(key)
+ return dict(entry) if entry else None
+
+
+def upsert_cached_entry(
+ issuer: str,
+ client_id: str,
+ entry: Dict[str, Any],
+ path: Optional[Path] = None,
+ audience: Optional[str] = None,
+) -> None:
+ path = path or credentials_path()
+ key = credential_key(issuer, client_id, audience)
+ row = {"issuer": normalize_issuer(issuer), "client_id": client_id, **entry}
+ normalized_audience = normalize_audience(audience)
+ if normalized_audience:
+ row["audience"] = normalized_audience
+ with _locked(path):
+ data = load_credentials_file(path)
+ credentials = dict(data.get("credentials", {}))
+ credentials[key] = row
+ data["credentials"] = credentials
+ save_credentials_file(data, path)
+
+
+def delete_cached_entry(
+ issuer: str,
+ client_id: str,
+ path: Optional[Path] = None,
+ audience: Optional[str] = None,
+) -> None:
+ path = path or credentials_path()
+ with _locked(path):
+ data = load_credentials_file(path)
+ credentials = dict(data.get("credentials", {}))
+ credentials.pop(credential_key(issuer, client_id, audience), None)
+ data["credentials"] = credentials
+ save_credentials_file(data, path)
+
+
+def _parse_timestamp(raw: str) -> datetime:
+ # datetime.fromisoformat before Python 3.11 accepts only 3 or 6 fractional
+ # digits and "+HH:MM" offsets, so normalize "Z", nanoseconds and "+HHMM".
+ raw = raw.strip().replace("Z", "+00:00")
+ raw = re.sub(r"\.(\d+)", lambda m: "." + m.group(1)[:6].ljust(6, "0"), raw)
+ raw = re.sub(r"([+-]\d{2})(\d{2})$", r"\1:\2", raw)
+ return datetime.fromisoformat(raw)
+
+
+def is_expired(entry: Dict[str, Any], skew_seconds: int = 120) -> bool:
+ raw = entry.get("expires_at")
+ if not raw:
+ return True
+ try:
+ expires = _parse_timestamp(raw)
+ except (TypeError, ValueError, AttributeError):
+ return True
+ if expires.tzinfo is None:
+ expires = expires.replace(tzinfo=timezone.utc)
+ return datetime.now(timezone.utc) >= expires - timedelta(seconds=skew_seconds)
+
+
+def expires_at_from_secs(expires_in: Optional[int]) -> str:
+ seconds = int(expires_in) if expires_in is not None else 3600
+ return (datetime.now(timezone.utc) + timedelta(seconds=seconds)).isoformat()
diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/errors.py b/packages/validmind-tracking-core/src/validmind_tracking_core/errors.py
new file mode 100644
index 000000000..3176a9b91
--- /dev/null
+++ b/packages/validmind-tracking-core/src/validmind_tracking_core/errors.py
@@ -0,0 +1,30 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Errors raised by the dependency-light tracking core."""
+
+
+class TrackingError(Exception):
+ """Base class for tracking-core failures."""
+
+
+class TrackingConfigurationError(TrackingError):
+ """The tracking client configuration is invalid or incomplete."""
+
+
+class TrackingAuthError(TrackingError):
+ """API-key or OIDC authentication failed."""
+
+
+class TrackingConnectionError(TrackingError):
+ """The tracking API could not be reached (connection failure or timeout)."""
+
+
+class TrackingAPIError(TrackingError):
+ """The tracking API rejected a request or returned an invalid response."""
+
+ def __init__(self, status_code: int, message: str, response_text: str = ""):
+ super().__init__(message)
+ self.status_code = status_code
+ self.response_text = response_text
diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/metrics.py b/packages/validmind-tracking-core/src/validmind_tracking_core/metrics.py
new file mode 100644
index 000000000..d596650db
--- /dev/null
+++ b/packages/validmind-tracking-core/src/validmind_tracking_core/metrics.py
@@ -0,0 +1,277 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Dependency-light synchronous and async-compatible metric transport."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import os
+from typing import Any, Dict, List, Optional, Type
+from urllib.parse import urljoin
+
+import requests
+
+from .errors import (
+ TrackingAPIError,
+ TrackingConfigurationError,
+ TrackingConnectionError,
+)
+from .oidc import OIDCAuthenticator
+
+_DEFAULT_TIMEOUT = 30.0
+
+
+class ScalarEncoder(json.JSONEncoder):
+ """Serialize numpy-style scalars and arrays without importing numpy."""
+
+ def default(self, o: Any) -> Any:
+ if hasattr(o, "tolist"):
+ return o.tolist()
+ return super().default(o)
+
+
+def _validate_metric(
+ key: str,
+ value: Any,
+ thresholds: Optional[Dict[str, Any]],
+) -> Any:
+ if not key or not isinstance(key, str):
+ raise ValueError("`key` must be a non-empty string")
+ if value is None:
+ raise ValueError("Must provide a value for the metric")
+ if not isinstance(value, (int, float)) and hasattr(value, "item"):
+ try:
+ value = value.item() # numpy scalars such as np.int64 / np.float32
+ except (TypeError, ValueError):
+ pass
+ if isinstance(value, bool):
+ raise ValueError(
+ "Booleans are not metric values; pass a pass/fail result as `passed`."
+ )
+ if not isinstance(value, (int, float)):
+ raise ValueError(
+ "Only scalar values (int or float) are allowed for logging metrics."
+ )
+ if thresholds is not None and not isinstance(thresholds, dict):
+ raise ValueError("`thresholds` must be a dictionary or None")
+ return value
+
+
+def timeout_from_env() -> float:
+ """Read ``VM_API_TIMEOUT``, treating a blank value as unset."""
+ raw = os.getenv("VM_API_TIMEOUT", "").strip()
+ if not raw:
+ return _DEFAULT_TIMEOUT
+ try:
+ return float(raw)
+ except ValueError:
+ raise TrackingConfigurationError(
+ f"VM_API_TIMEOUT must be a number of seconds, got {raw!r}"
+ ) from None
+
+
+def serialize_metric(
+ key: str,
+ value: Any,
+ inputs: Optional[List[str]] = None,
+ params: Optional[Dict[str, Any]] = None,
+ recorded_at: Optional[str] = None,
+ thresholds: Optional[Dict[str, Any]] = None,
+ passed: Optional[bool] = None,
+ *,
+ encoder: Optional[Type[json.JSONEncoder]] = None,
+) -> str:
+ """Validate and serialize a metric using the tracking API schema."""
+ value = _validate_metric(key, value, thresholds)
+ payload = {
+ "key": key,
+ "value": value,
+ "inputs": inputs or [],
+ "params": params or {},
+ "recorded_at": recorded_at,
+ "thresholds": thresholds or {},
+ "passed": passed if passed is not None else None,
+ }
+ return json.dumps(payload, cls=encoder or ScalarEncoder, allow_nan=False)
+
+
+def post_metric(
+ url: str,
+ body: str,
+ headers: Dict[str, str],
+ *,
+ timeout: float = _DEFAULT_TIMEOUT,
+) -> Dict[str, Any]:
+ """POST a serialized metric and return the JSON response."""
+ try:
+ response = requests.post(url, data=body, headers=headers, timeout=timeout)
+ except requests.RequestException as exc:
+ raise TrackingConnectionError(
+ f"Could not reach ValidMind at {url!r}: {exc}"
+ ) from exc
+ if response.status_code != 200:
+ raise TrackingAPIError(response.status_code, response.text[:500], response.text)
+ try:
+ result = response.json()
+ except ValueError as exc:
+ raise TrackingAPIError(
+ response.status_code,
+ "ValidMind returned a non-JSON metric response",
+ response.text,
+ ) from exc
+ if not isinstance(result, dict):
+ raise TrackingAPIError(
+ response.status_code,
+ "ValidMind returned an invalid metric response",
+ response.text,
+ )
+ return result
+
+
+class MetricsClient:
+ """Client for the ValidMind ``log_unit_metric`` endpoint."""
+
+ def __init__(
+ self,
+ *,
+ api_host: Optional[str] = None,
+ api_url: Optional[str] = None,
+ model: Optional[str] = None,
+ api_key: Optional[str] = None,
+ api_secret: Optional[str] = None,
+ monitoring: bool = False,
+ document: Optional[str] = None,
+ client_version: str = "validmind-tracking-core/0.1.0",
+ timeout: Optional[float] = None,
+ issuer: Optional[str] = None,
+ client_id: Optional[str] = None,
+ scope: Optional[str] = None,
+ audience: Optional[str] = None,
+ credentials_path=None,
+ status_callback=None,
+ interactive: bool = False,
+ ):
+ self.api_host = (
+ api_url or api_host or os.getenv("VM_API_URL") or os.getenv("VM_API_HOST")
+ )
+ self.model = model or os.getenv("VM_API_MODEL")
+ self.monitoring = monitoring
+ self.document = document
+ self.client_version = client_version
+ self.timeout = float(timeout) if timeout is not None else timeout_from_env()
+ self._api_key = api_key if api_key is not None else os.getenv("VM_API_KEY")
+ self._api_secret = (
+ api_secret if api_secret is not None else os.getenv("VM_API_SECRET")
+ )
+ issuer = issuer if issuer is not None else os.getenv("VM_OIDC_ISSUER")
+ client_id = (
+ client_id if client_id is not None else os.getenv("VM_OIDC_CLIENT_ID")
+ )
+ scope = scope if scope is not None else os.getenv("VM_OIDC_SCOPE")
+ audience = audience if audience is not None else os.getenv("VM_OIDC_AUDIENCE")
+
+ has_api_creds = bool(self._api_key and self._api_secret)
+ has_oidc = bool(issuer and client_id)
+ if not self.api_host:
+ raise TrackingConfigurationError("API host must be provided")
+ if not self.model:
+ raise TrackingConfigurationError("Model ID must be provided")
+ if has_api_creds and has_oidc:
+ raise TrackingConfigurationError(
+ "Provide either API credentials or OIDC credentials, not both"
+ )
+ if bool(issuer) != bool(client_id):
+ raise TrackingConfigurationError(
+ "issuer and client_id must be provided together"
+ )
+ if not has_api_creds and not has_oidc:
+ raise TrackingConfigurationError(
+ "Provide API credentials or issuer and client_id for OIDC"
+ )
+
+ self._oidc = None
+ if has_oidc:
+ self._oidc = OIDCAuthenticator(
+ issuer,
+ client_id,
+ scope=scope,
+ audience=audience,
+ timeout=self.timeout,
+ credentials_path=credentials_path,
+ status_callback=status_callback,
+ interactive=interactive,
+ )
+ self._oidc.initialize()
+
+ def log_metric(
+ self,
+ key: str,
+ value: Any,
+ inputs: Optional[List[str]] = None,
+ params: Optional[Dict[str, Any]] = None,
+ recorded_at: Optional[str] = None,
+ thresholds: Optional[Dict[str, Any]] = None,
+ passed: Optional[bool] = None,
+ ) -> Dict[str, Any]:
+ body = serialize_metric(
+ key,
+ value,
+ inputs,
+ params,
+ recorded_at,
+ thresholds,
+ passed,
+ )
+ return post_metric(
+ self._url("log_unit_metric"),
+ body,
+ self._headers(),
+ timeout=self.timeout,
+ )
+
+ async def alog_metric(
+ self,
+ key: str,
+ value: Any,
+ inputs: Optional[List[str]] = None,
+ params: Optional[Dict[str, Any]] = None,
+ recorded_at: Optional[str] = None,
+ thresholds: Optional[Dict[str, Any]] = None,
+ passed: Optional[bool] = None,
+ ) -> Dict[str, Any]:
+ """Run ``log_metric`` in the default executor so the event loop isn't blocked.
+
+ Each call occupies one executor thread for a full round trip; there is no
+ background queue.
+ """
+ return await asyncio.to_thread(
+ self.log_metric,
+ key,
+ value,
+ inputs,
+ params,
+ recorded_at,
+ thresholds,
+ passed,
+ )
+
+ def _headers(self) -> Dict[str, str]:
+ headers = {
+ "X-MODEL-CUID": self.model,
+ "X-MONITORING": str(self.monitoring),
+ "X-LIBRARY-VERSION": self.client_version,
+ }
+ if self.document:
+ headers["X-DOCUMENT-TYPE"] = self.document
+ if self._oidc:
+ headers["Authorization"] = f"Bearer {self._oidc.token()}"
+ else:
+ headers["X-API-KEY"] = self._api_key
+ headers["X-API-SECRET"] = self._api_secret
+ return headers
+
+ def _url(self, endpoint: str) -> str:
+ return urljoin(f"{self.api_host.rstrip('/')}/", endpoint)
diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/oidc.py b/packages/validmind-tracking-core/src/validmind_tracking_core/oidc.py
new file mode 100644
index 000000000..fa3e9c914
--- /dev/null
+++ b/packages/validmind-tracking-core/src/validmind_tracking_core/oidc.py
@@ -0,0 +1,392 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+"""Synchronous OIDC device-flow authentication for tracking clients."""
+
+from __future__ import annotations
+
+import threading
+import time
+from typing import Any, Callable, Dict, Optional
+from urllib.parse import urlparse
+
+import requests
+
+from .credentials_store import (
+ delete_cached_entry,
+ expires_at_from_secs,
+ get_cached_entry,
+ is_expired,
+ normalize_audience,
+ normalize_client_id,
+ normalize_issuer,
+ upsert_cached_entry,
+)
+from .errors import TrackingAuthError, TrackingConfigurationError
+
+_OPENID_CONFIG_SUFFIX = "/.well-known/openid-configuration"
+_DEFAULT_TIMEOUT = 30.0
+_DEFAULT_SCOPE = "openid profile email offline_access"
+_LOOPBACK_HOSTS = {"localhost", "127.0.0.1", "::1"}
+
+
+def _require_https(issuer: str) -> None:
+ parsed = urlparse(issuer)
+ if parsed.scheme == "https":
+ return
+ if parsed.scheme == "http" and (parsed.hostname or "").lower() in _LOOPBACK_HOSTS:
+ return
+ raise TrackingConfigurationError(
+ f"OIDC issuer must be an https:// URL (http is allowed only for "
+ f"localhost), got {issuer!r}"
+ )
+
+
+def _token_entry(payload: Dict[str, Any]) -> Dict[str, Any]:
+ entry = dict(payload)
+ if not entry.get("expires_at"):
+ entry["expires_at"] = expires_at_from_secs(entry.get("expires_in"))
+ return entry
+
+
+def _bearer_token(entry: Dict[str, Any]) -> str:
+ issuer = entry.get("issuer", "")
+ try:
+ issuer_host = (urlparse(issuer).hostname or "").lower()
+ except ValueError:
+ issuer_host = ""
+ if issuer_host == "login.microsoftonline.com" and entry.get("id_token"):
+ return entry["id_token"]
+ token = entry.get("access_token")
+ if not token:
+ raise TrackingAuthError("OIDC response did not contain an access token")
+ return token
+
+
+def _response_json(response: requests.Response) -> Dict[str, Any]:
+ try:
+ body = response.json()
+ except ValueError:
+ body = {}
+ return body if isinstance(body, dict) else {}
+
+
+def fetch_openid_configuration(
+ issuer: str, timeout: float = _DEFAULT_TIMEOUT
+) -> Dict[str, Any]:
+ base = normalize_issuer(issuer)
+ url = f"{base}{_OPENID_CONFIG_SUFFIX}"
+ try:
+ response = requests.get(url, timeout=timeout)
+ except requests.RequestException as exc:
+ raise TrackingAuthError(
+ f"Could not reach OIDC discovery URL {url!r}: {exc}"
+ ) from exc
+ if response.status_code != 200:
+ raise TrackingAuthError(
+ f"OIDC discovery failed for {url!r}: HTTP {response.status_code} "
+ f"{response.text[:500]}"
+ )
+ body = _response_json(response)
+ for key in ("device_authorization_endpoint", "token_endpoint"):
+ if key not in body:
+ raise TrackingAuthError(
+ f"OIDC discovery document from {url!r} is missing {key!r}"
+ )
+ return body
+
+
+def request_device_authorization(
+ endpoint: str,
+ client_id: str,
+ scope: str,
+ timeout: float = _DEFAULT_TIMEOUT,
+ audience: Optional[str] = None,
+) -> Dict[str, Any]:
+ payload: Dict[str, str] = {"client_id": client_id, "scope": scope}
+ normalized_audience = normalize_audience(audience)
+ if normalized_audience:
+ payload["audience"] = normalized_audience
+ try:
+ response = requests.post(
+ endpoint,
+ data=payload,
+ headers={"Accept": "application/json"},
+ timeout=timeout,
+ )
+ except requests.RequestException as exc:
+ raise TrackingAuthError(f"Device authorization request failed: {exc}") from exc
+ body = _response_json(response)
+ if response.status_code != 200:
+ raise TrackingAuthError(
+ "Device authorization endpoint rejected the request: "
+ f"HTTP {response.status_code} {body or response.text[:500]}"
+ )
+ for key in ("device_code", "user_code", "verification_uri"):
+ if key not in body:
+ raise TrackingAuthError(f"Device authorization response missing {key!r}")
+ return body
+
+
+def poll_device_token(
+ endpoint: str,
+ client_id: str,
+ device_code: str,
+ *,
+ interval: float = 5.0,
+ expires_in: float = 900.0,
+ timeout: float = _DEFAULT_TIMEOUT,
+ audience: Optional[str] = None,
+) -> Dict[str, Any]:
+ deadline = time.monotonic() + float(expires_in)
+ current_interval = float(interval)
+ while time.monotonic() < deadline:
+ payload: Dict[str, str] = {
+ "grant_type": "urn:ietf:params:oauth:grant-type:device_code",
+ "device_code": device_code,
+ "client_id": client_id,
+ }
+ normalized_audience = normalize_audience(audience)
+ if normalized_audience:
+ payload["audience"] = normalized_audience
+ try:
+ response = requests.post(
+ endpoint,
+ data=payload,
+ headers={"Accept": "application/json"},
+ timeout=timeout,
+ )
+ except requests.RequestException as exc:
+ raise TrackingAuthError(f"Token poll request failed: {exc}") from exc
+ body = _response_json(response)
+ if response.status_code == 200 and body.get("access_token"):
+ return _token_entry(body)
+ error = body.get("error")
+ if error == "authorization_pending":
+ time.sleep(current_interval)
+ continue
+ if error == "slow_down":
+ current_interval += 5
+ time.sleep(current_interval)
+ continue
+ if error == "expired_token":
+ raise TrackingAuthError("Device login expired before completion")
+ if error == "access_denied":
+ raise TrackingAuthError("Device authorization was denied")
+ raise TrackingAuthError(
+ f"Token poll failed: HTTP {response.status_code} "
+ f"error={error!r} {body or response.text[:500]}"
+ )
+ raise TrackingAuthError("Device login timed out waiting for authorization")
+
+
+def refresh_access_token(
+ endpoint: str,
+ client_id: str,
+ refresh_token: str,
+ scope: Optional[str] = None,
+ timeout: float = _DEFAULT_TIMEOUT,
+ audience: Optional[str] = None,
+) -> Dict[str, Any]:
+ payload: Dict[str, str] = {
+ "grant_type": "refresh_token",
+ "refresh_token": refresh_token,
+ "client_id": client_id,
+ }
+ if scope:
+ payload["scope"] = scope
+ normalized_audience = normalize_audience(audience)
+ if normalized_audience:
+ payload["audience"] = normalized_audience
+ try:
+ response = requests.post(
+ endpoint,
+ data=payload,
+ headers={"Accept": "application/json"},
+ timeout=timeout,
+ )
+ except requests.RequestException as exc:
+ raise TrackingAuthError(f"Token refresh request failed: {exc}") from exc
+ body = _response_json(response)
+ if response.status_code != 200 or not body.get("access_token"):
+ raise TrackingAuthError(
+ f"Token refresh failed: HTTP {response.status_code} "
+ f"{body or response.text[:500]}"
+ )
+ return _token_entry(body)
+
+
+def run_device_flow(
+ issuer: str,
+ client_id: str,
+ scope: str,
+ *,
+ audience: Optional[str] = None,
+ timeout: float = _DEFAULT_TIMEOUT,
+ status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
+ configuration: Optional[Dict[str, Any]] = None,
+) -> Dict[str, Any]:
+ configuration = configuration or fetch_openid_configuration(issuer, timeout=timeout)
+ device = request_device_authorization(
+ configuration["device_authorization_endpoint"],
+ client_id,
+ scope,
+ timeout=timeout,
+ audience=audience,
+ )
+ status = {
+ "verification_uri": device["verification_uri"],
+ "user_code": device["user_code"],
+ "verification_uri_complete": device.get("verification_uri_complete"),
+ }
+ if status_callback:
+ status_callback(status)
+ else:
+ complete = status.get("verification_uri_complete") or status["verification_uri"]
+ print(
+ f"Visit: {complete}\nCode: {status['user_code']}\nWaiting for authorization..."
+ )
+ return poll_device_token(
+ configuration["token_endpoint"],
+ client_id,
+ device["device_code"],
+ interval=float(device.get("interval", 5)),
+ expires_in=float(device.get("expires_in", 900)),
+ timeout=timeout,
+ audience=audience,
+ )
+
+
+class OIDCAuthenticator:
+ """Load, refresh, or (with ``interactive=True``) device-login for a bearer token.
+
+ Non-interactive by default so a service never blocks waiting for a human:
+ with no usable cached credential it raises ``TrackingAuthError`` instead of
+ starting the device flow.
+ """
+
+ def __init__(
+ self,
+ issuer: str,
+ client_id: str,
+ *,
+ scope: Optional[str] = None,
+ audience: Optional[str] = None,
+ timeout: float = _DEFAULT_TIMEOUT,
+ credentials_path=None,
+ status_callback: Optional[Callable[[Dict[str, Any]], None]] = None,
+ interactive: bool = False,
+ ):
+ self.issuer = normalize_issuer(issuer)
+ _require_https(self.issuer)
+ self.client_id = normalize_client_id(client_id)
+ self.scope = scope or _DEFAULT_SCOPE
+ self.audience = normalize_audience(audience) or None
+ self.timeout = timeout
+ self.credentials_path = credentials_path
+ self.status_callback = status_callback
+ self.interactive = interactive
+ self._entry: Optional[Dict[str, Any]] = None
+ self._configuration: Optional[Dict[str, Any]] = None
+ self._refresh_lock = threading.Lock()
+
+ def initialize(self) -> None:
+ cached = get_cached_entry(
+ self.issuer,
+ self.client_id,
+ path=self.credentials_path,
+ audience=self.audience,
+ )
+ if cached and not is_expired(cached):
+ self._entry = cached
+ return
+ if cached and cached.get("refresh_token"):
+ try:
+ refreshed = refresh_access_token(
+ self._token_endpoint(),
+ self.client_id,
+ cached["refresh_token"],
+ scope=self.scope,
+ timeout=self.timeout,
+ audience=self.audience,
+ )
+ except TrackingAuthError:
+ delete_cached_entry(
+ self.issuer,
+ self.client_id,
+ path=self.credentials_path,
+ audience=self.audience,
+ )
+ else:
+ refreshed.setdefault("refresh_token", cached["refresh_token"])
+ self._save(refreshed)
+ return
+ if not self.interactive:
+ raise TrackingAuthError(
+ "No usable cached OIDC credentials for "
+ f"{self.issuer!r}. Log in once with interactive=True (or "
+ "`validmind_metrics.init(interactive=True)`), or use API-key "
+ "credentials for unattended services."
+ )
+ entry = run_device_flow(
+ self.issuer,
+ self.client_id,
+ self.scope,
+ audience=self.audience,
+ timeout=self.timeout,
+ status_callback=self.status_callback,
+ configuration=self._discovery(),
+ )
+ self._save(entry)
+
+ def token(self) -> str:
+ if self._entry and not is_expired(self._entry):
+ return _bearer_token(self._entry)
+ with self._refresh_lock:
+ if self._entry and not is_expired(self._entry):
+ return _bearer_token(self._entry)
+ cached = get_cached_entry(
+ self.issuer,
+ self.client_id,
+ path=self.credentials_path,
+ audience=self.audience,
+ )
+ if not cached or not cached.get("refresh_token"):
+ raise TrackingAuthError(
+ "OIDC access token is missing or expired; initialize the client again"
+ )
+ refreshed = refresh_access_token(
+ self._token_endpoint(),
+ self.client_id,
+ cached["refresh_token"],
+ scope=self.scope,
+ timeout=self.timeout,
+ audience=self.audience,
+ )
+ refreshed.setdefault("refresh_token", cached["refresh_token"])
+ self._save(refreshed)
+ return _bearer_token(self._entry)
+
+ def _save(self, entry: Dict[str, Any]) -> None:
+ saved = _token_entry(entry)
+ saved["issuer"] = self.issuer
+ saved["client_id"] = self.client_id
+ self._entry = saved
+ upsert_cached_entry(
+ self.issuer,
+ self.client_id,
+ saved,
+ path=self.credentials_path,
+ audience=self.audience,
+ )
+
+ def _discovery(self) -> Dict[str, Any]:
+ if self._configuration is None:
+ self._configuration = fetch_openid_configuration(
+ self.issuer, timeout=self.timeout
+ )
+ return self._configuration
+
+ def _token_endpoint(self) -> str:
+ return self._discovery()["token_endpoint"]
diff --git a/packages/validmind-tracking-core/src/validmind_tracking_core/py.typed b/packages/validmind-tracking-core/src/validmind_tracking_core/py.typed
new file mode 100644
index 000000000..e69de29bb
diff --git a/packages/validmind-tracking-core/tests/test_oidc.py b/packages/validmind-tracking-core/tests/test_oidc.py
new file mode 100644
index 000000000..5049e4b5a
--- /dev/null
+++ b/packages/validmind-tracking-core/tests/test_oidc.py
@@ -0,0 +1,91 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import unittest
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest.mock import Mock, patch
+
+from validmind_tracking_core.credentials_store import (
+ get_cached_entry,
+ upsert_cached_entry,
+)
+from validmind_tracking_core.oidc import OIDCAuthenticator, _bearer_token
+
+
+class TestOIDCAuthenticator(unittest.TestCase):
+ def test_bearer_token_matches_entra_hostname(self):
+ self.assertEqual(
+ _bearer_token(
+ {
+ "issuer": "https://login.microsoftonline.com/tenant/v2.0",
+ "access_token": "access-token",
+ "id_token": "id-token",
+ }
+ ),
+ "id-token",
+ )
+ self.assertEqual(
+ _bearer_token(
+ {
+ "issuer": "https://login.microsoftonline.com.evil.example/tenant",
+ "access_token": "access-token",
+ "id_token": "id-token",
+ }
+ ),
+ "access-token",
+ )
+
+ @patch("validmind_tracking_core.oidc.requests.post")
+ @patch("validmind_tracking_core.oidc.requests.get")
+ def test_refreshes_expired_cached_token(self, mock_get, mock_post):
+ discovery = Mock(status_code=200)
+ discovery.json.return_value = {
+ "device_authorization_endpoint": "https://issuer.example/device",
+ "token_endpoint": "https://issuer.example/token",
+ }
+ mock_get.return_value = discovery
+ refreshed = Mock(status_code=200)
+ refreshed.json.return_value = {
+ "access_token": "refreshed-token",
+ "expires_in": 3600,
+ }
+ mock_post.return_value = refreshed
+
+ with TemporaryDirectory() as temp_dir:
+ credentials_path = Path(temp_dir) / "credentials.json"
+ upsert_cached_entry(
+ "https://issuer.example",
+ "client-1",
+ {
+ "access_token": "expired-token",
+ "refresh_token": "refresh-token",
+ "expires_at": (
+ datetime.now(timezone.utc) - timedelta(hours=1)
+ ).isoformat(),
+ },
+ path=credentials_path,
+ )
+ auth = OIDCAuthenticator(
+ "https://issuer.example",
+ "client-1",
+ credentials_path=credentials_path,
+ )
+ auth.initialize()
+
+ self.assertEqual(auth.token(), "refreshed-token")
+ self.assertEqual(
+ mock_post.call_args.args[0], "https://issuer.example/token"
+ )
+ self.assertEqual(
+ get_cached_entry(
+ "https://issuer.example", "client-1", path=credentials_path
+ )["refresh_token"],
+ "refresh-token",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/packages/validmind-tracking-core/tests/test_review_fixes.py b/packages/validmind-tracking-core/tests/test_review_fixes.py
new file mode 100644
index 000000000..73bf3f75e
--- /dev/null
+++ b/packages/validmind-tracking-core/tests/test_review_fixes.py
@@ -0,0 +1,139 @@
+# Copyright © 2023-2026 ValidMind Inc. All rights reserved.
+# Refer to the LICENSE file in the root of this repository for details.
+# SPDX-License-Identifier: AGPL-3.0 AND ValidMind Commercial
+
+import json
+import os
+import threading
+import unittest
+from pathlib import Path
+from tempfile import TemporaryDirectory
+from unittest.mock import Mock, patch
+
+import requests
+from validmind_tracking_core import (
+ MetricsClient,
+ TrackingAuthError,
+ TrackingConfigurationError,
+ TrackingConnectionError,
+ TrackingError,
+ post_metric,
+ serialize_metric,
+)
+from validmind_tracking_core.credentials_store import (
+ is_expired,
+ load_credentials_file,
+ upsert_cached_entry,
+)
+from validmind_tracking_core.oidc import OIDCAuthenticator
+
+
+class FakeNumpyScalar:
+ def __init__(self, value):
+ self.value = value
+
+ def item(self):
+ return self.value
+
+ def tolist(self):
+ return self.value
+
+
+class TestCredentialsStore(unittest.TestCase):
+ def test_concurrent_upserts_keep_every_entry(self):
+ with TemporaryDirectory() as temp_dir:
+ path = Path(temp_dir) / "credentials.json"
+ threads = [
+ threading.Thread(
+ target=upsert_cached_entry,
+ args=("https://issuer.example", f"client-{i}", {"a": i}),
+ kwargs={"path": path},
+ )
+ for i in range(25)
+ ]
+ for thread in threads:
+ thread.start()
+ for thread in threads:
+ thread.join()
+ self.assertEqual(len(load_credentials_file(path)["credentials"]), 25)
+
+ def test_is_expired_parses_formats_older_pythons_reject(self):
+ for raw in (
+ "2999-01-01T00:00:00.123456789+00:00",
+ "2999-01-01T00:00:00+0000",
+ "2999-01-01T00:00:00.12345Z",
+ ):
+ self.assertFalse(is_expired({"expires_at": raw}), raw)
+ self.assertTrue(is_expired({"expires_at": "2000-01-01T00:00:00+0000"}))
+
+
+class TestMetricValidation(unittest.TestCase):
+ def test_bool_value_rejected(self):
+ with self.assertRaisesRegex(ValueError, "passed"):
+ serialize_metric("passed", True)
+
+ def test_numpy_style_values_serialize(self):
+ body = json.loads(
+ serialize_metric("n", FakeNumpyScalar(5), params={"n": FakeNumpyScalar(3)})
+ )
+ self.assertEqual((body["value"], body["params"]["n"]), (5, 3))
+
+ @patch("validmind_tracking_core.metrics.requests.post")
+ def test_network_errors_are_tracking_errors(self, mock_post):
+ mock_post.side_effect = requests.ConnectionError("down")
+ with self.assertRaises(TrackingConnectionError) as ctx:
+ post_metric("https://x.example/log_unit_metric", "{}", {})
+ self.assertIsInstance(ctx.exception, TrackingError)
+
+
+class TestClientConfiguration(unittest.TestCase):
+ kwargs = dict(api_host="https://x.example", model="m", api_key="k", api_secret="s")
+
+ def test_blank_timeout_env_is_default(self):
+ with patch.dict(os.environ, {"VM_API_TIMEOUT": ""}):
+ self.assertEqual(MetricsClient(**self.kwargs).timeout, 30.0)
+
+ def test_invalid_timeout_env_names_variable(self):
+ with patch.dict(os.environ, {"VM_API_TIMEOUT": "soon"}):
+ with self.assertRaisesRegex(TrackingConfigurationError, "VM_API_TIMEOUT"):
+ MetricsClient(**self.kwargs)
+
+ def test_zero_timeout_is_respected(self):
+ self.assertEqual(MetricsClient(timeout=0, **self.kwargs).timeout, 0.0)
+
+
+class TestOIDCSafety(unittest.TestCase):
+ def test_http_issuer_rejected_except_loopback(self):
+ with self.assertRaises(TrackingConfigurationError):
+ OIDCAuthenticator("http://idp.internal", "c")
+ with self.assertRaises(TrackingConfigurationError):
+ OIDCAuthenticator("idp.internal", "c")
+ OIDCAuthenticator("http://localhost:8080", "c")
+
+ @patch("validmind_tracking_core.oidc.run_device_flow")
+ def test_non_interactive_by_default(self, mock_flow):
+ with TemporaryDirectory() as temp_dir:
+ auth = OIDCAuthenticator(
+ "https://issuer.example",
+ "c",
+ credentials_path=Path(temp_dir) / "credentials.json",
+ )
+ with self.assertRaises(TrackingAuthError):
+ auth.initialize()
+ mock_flow.assert_not_called()
+
+ @patch("validmind_tracking_core.oidc.requests.get")
+ def test_discovery_fetched_once(self, mock_get):
+ mock_get.return_value = Mock(status_code=200)
+ mock_get.return_value.json.return_value = {
+ "device_authorization_endpoint": "https://issuer.example/device",
+ "token_endpoint": "https://issuer.example/token",
+ }
+ auth = OIDCAuthenticator("https://issuer.example", "c")
+ auth._token_endpoint()
+ auth._token_endpoint()
+ self.assertEqual(mock_get.call_count, 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/pyproject.toml b/pyproject.toml
index 37b9d360c..a5868c652 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -14,6 +14,7 @@ authors = [
]
dependencies = [
"aiohttp[speedups] (<3.13.1)",
+ "validmind-tracking-core (>=0.1.0,<0.2.0)",
"requests (>=2.28.0,<3.0.0)",
"ipywidgets",
"kaleido (>=1.2.0,<2.0.0)",
@@ -24,7 +25,9 @@ dependencies = [
"numpy (>=2.3,<3.0.0) ; python_version >= '3.14'",
"openai (>=1)",
"pandas (>=2.0.3,<3.0.0)",
- "plotly (>=6.0.0)",
+ # Plotly 7 is a new major that the figure code and kaleido export haven't been
+ # validated against yet; lift the cap once they have.
+ "plotly (>=6.0.0,<7.0.0)",
"polars",
"python-dotenv",
"scikit-learn",
@@ -195,3 +198,12 @@ profile = "black"
[tool.uv]
exclude-newer = "7 days"
+
+[tool.uv.sources]
+validmind-tracking-core = { workspace = true }
+
+[tool.uv.workspace]
+members = [
+ "packages/validmind-tracking-core",
+ "packages/validmind-metrics",
+]
diff --git a/scripts/verify_copyright.py b/scripts/verify_copyright.py
index f0b9e24da..87ca3d4d0 100644
--- a/scripts/verify_copyright.py
+++ b/scripts/verify_copyright.py
@@ -19,8 +19,8 @@
with open(copyright_path) as f:
copyright = f.read()
-# Scan the Python package directory
-directory = os.path.join(os.getcwd(), "validmind")
+# Scan the library and the workspace packages
+directories = [os.path.join(os.getcwd(), d) for d in ("validmind", "packages")]
# List of file extensions to process
extensions = [".py"]
@@ -29,7 +29,7 @@
errors = []
# Loop through all files in the directory and its subdirectories
-for root, dirs, files in os.walk(directory):
+for root, dirs, files in (w for d in directories for w in os.walk(d)):
for file in files:
# Check if the file has a valid extension
if file.endswith(tuple(extensions)) and file != "__version__.py":
diff --git a/tests/test_api_client.py b/tests/test_api_client.py
index 57e506f92..09461d230 100644
--- a/tests/test_api_client.py
+++ b/tests/test_api_client.py
@@ -148,6 +148,24 @@ def test_get_api_model(self):
model = api_client.get_api_model()
self.assertEqual(model, "your_model")
+ @patch("requests.post")
+ def test_log_metric_is_safe_inside_running_event_loop(self, mock_post):
+ mock_post.return_value = MockResponse(200, json={"ok": True})
+
+ async def handler():
+ first = api_client.log_metric("accuracy", 0.95)
+ second = api_client.log_metric("accuracy", 0.96)
+ return first, second
+
+ self.assertEqual(asyncio.run(handler()), ({"ok": True}, {"ok": True}))
+ self.assertEqual(mock_post.call_count, 2)
+ self.assertTrue(
+ all(
+ call.args[0].endswith("/log_unit_metric")
+ for call in mock_post.call_args_list
+ )
+ )
+
@patch("requests.get")
def test_init_missing_model_id(self, mock_requests_get):
mock_requests_get.return_value = Mock()
diff --git a/uv.lock b/uv.lock
index 882938164..128a60f41 100644
--- a/uv.lock
+++ b/uv.lock
@@ -18,6 +18,13 @@ resolution-markers = [
exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values.
exclude-newer-span = "P7D"
+[manifest]
+members = [
+ "validmind",
+ "validmind-metrics",
+ "validmind-tracking-core",
+]
+
[[package]]
name = "aiodns"
version = "3.6.1"
@@ -11298,6 +11305,7 @@ dependencies = [
{ name = "tabulate" },
{ name = "tiktoken" },
{ name = "tqdm" },
+ { name = "validmind-tracking-core" },
]
[package.optional-dependencies]
@@ -11494,7 +11502,7 @@ requires-dist = [
{ name = "numpy", marker = "python_full_version >= '3.14'", specifier = ">=2.3,<3.0.0" },
{ name = "openai", specifier = ">=1" },
{ name = "pandas", specifier = ">=2.0.3,<3.0.0" },
- { name = "plotly", specifier = ">=6.0.0" },
+ { name = "plotly", specifier = ">=6.0.0,<7.0.0" },
{ name = "polars" },
{ name = "presidio-analyzer", marker = "python_full_version >= '3.14' and extra == 'pii-detection'", specifier = "<2.2.360" },
{ name = "presidio-analyzer", marker = "python_full_version < '3.14' and extra == 'pii-detection'" },
@@ -11549,6 +11557,7 @@ requires-dist = [
{ name = "transformers", marker = "extra == 'huggingface'", specifier = ">=4.32.0,<5.0.0" },
{ name = "transformers", marker = "extra == 'llm'", specifier = ">=4.32.0,<5.0.0" },
{ name = "transformers", marker = "extra == 'nlp'", specifier = ">=4.32.0,<5.0.0" },
+ { name = "validmind-tracking-core", editable = "packages/validmind-tracking-core" },
{ name = "xgboost", marker = "extra == 'all'", specifier = ">=1.5.2,<3.1" },
{ name = "xgboost", marker = "extra == 'xgboost'", specifier = ">=1.5.2,<3.1" },
]
@@ -11575,6 +11584,28 @@ dev = [
{ name = "twine", specifier = ">=4.0.2,<5" },
]
+[[package]]
+name = "validmind-metrics"
+version = "0.1.0"
+source = { editable = "packages/validmind-metrics" }
+dependencies = [
+ { name = "validmind-tracking-core" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "validmind-tracking-core", editable = "packages/validmind-tracking-core" }]
+
+[[package]]
+name = "validmind-tracking-core"
+version = "0.1.0"
+source = { editable = "packages/validmind-tracking-core" }
+dependencies = [
+ { name = "requests" },
+]
+
+[package.metadata]
+requires-dist = [{ name = "requests", specifier = ">=2.28.0,<3.0.0" }]
+
[[package]]
name = "virtualenv"
version = "21.3.1"
diff --git a/validmind/api_client.py b/validmind/api_client.py
index 69d6e43a8..6307732c6 100644
--- a/validmind/api_client.py
+++ b/validmind/api_client.py
@@ -19,6 +19,12 @@
import aiohttp
import requests
from aiohttp import FormData
+from validmind_tracking_core.errors import TrackingAPIError
+from validmind_tracking_core.metrics import (
+ post_metric,
+ serialize_metric,
+ timeout_from_env,
+)
from .__version__ import __version__
from .client_config import client_config
@@ -995,6 +1001,23 @@ def log_text(
return _render_logged_text(logged_text)
+def _send_metric_sync(body: str):
+ """Send one serialized metric without creating or depending on an event loop."""
+ try:
+ _ensure_fresh_oidc_token()
+ return post_metric(
+ _get_url("log_unit_metric"),
+ body,
+ _get_api_headers(),
+ timeout=timeout_from_env(),
+ )
+ except Exception as e:
+ logger.error("Error logging metric to ValidMind API")
+ if isinstance(e, TrackingAPIError):
+ _raise_for_api_error(e.status_code, e.response_text)
+ raise
+
+
async def alog_metric(
key: str,
value: Union[int, float],
@@ -1004,42 +1027,18 @@ async def alog_metric(
thresholds: Optional[Dict[str, Any]] = None,
passed: Optional[bool] = None,
):
- """See log_metric for details."""
- if not key or not isinstance(key, str):
- raise ValueError("`key` must be a non-empty string")
-
- if value is None:
- raise ValueError("Must provide a value for the metric")
-
- # Validate that value is a scalar (int or float)
- if not isinstance(value, (int, float)):
- raise ValueError(
- "Only scalar values (int or float) are allowed for logging metrics."
- )
-
- if thresholds is not None and not isinstance(thresholds, dict):
- raise ValueError("`thresholds` must be a dictionary or None")
-
- try:
- return await _post(
- "log_unit_metric",
- data=json.dumps(
- {
- "key": key,
- "value": value,
- "inputs": inputs or [],
- "params": params or {},
- "recorded_at": recorded_at,
- "thresholds": thresholds or {},
- "passed": passed if passed is not None else None,
- },
- cls=NumpyEncoder,
- allow_nan=False,
- ),
- )
- except Exception as e:
- logger.error("Error logging metric to ValidMind API")
- raise e
+ """See log_metric for details, without blocking the current event loop."""
+ body = serialize_metric(
+ key,
+ value,
+ inputs,
+ params,
+ recorded_at,
+ thresholds,
+ passed,
+ encoder=NumpyEncoder,
+ )
+ return await asyncio.to_thread(_send_metric_sync, body)
def log_metric(
@@ -1068,16 +1067,17 @@ def log_metric(
thresholds (Dict[str, Any], optional): Thresholds for the metric
passed (bool, optional): Whether the metric passed validation thresholds
"""
- return run_async(
- alog_metric,
- key=key,
- value=value,
- inputs=inputs,
- params=params,
- recorded_at=recorded_at,
- thresholds=thresholds,
- passed=passed,
+ body = serialize_metric(
+ key,
+ value,
+ inputs,
+ params,
+ recorded_at,
+ thresholds,
+ passed,
+ encoder=NumpyEncoder,
)
+ return _send_metric_sync(body)
def generate_test_result_description(test_result_data: Dict[str, Any]) -> str: