From ef5e942249ac8bcab5272534f48ea4690e19c117 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 23 Sep 2026 13:56:41 -0300 Subject: [PATCH 1/7] =?UTF-8?q?Atualiza=20a=20depend=C3=AAncia=20da=20bibl?= =?UTF-8?q?ioteca=20de=20contagem?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/base.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/base.txt b/requirements/base.txt index 48a6cac..2cf8ecc 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -66,7 +66,7 @@ git+https://github.com/scieloorg/scielo_log_validator@2.0.4#egg=scielo_log_valid git+https://github.com/scieloorg/scielo_scholarly_data@v0.1.4#egg=scielo_scholarly_data # SciELO Usage COUNTER -git+https://github.com/scieloorg/scielo_usage_counter@2.2.3#egg=scielo_usage_counter +git+https://github.com/scieloorg/scielo_usage_counter@2.2.4#egg=scielo_usage_counter # Incremental JSON ijson==3.5.1 From 8eddda443b281186fac261530351c5824d0bbd0a Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 23 Sep 2026 13:56:37 -0300 Subject: [PATCH 2/7] =?UTF-8?q?Testa=20a=20unidade=20do=20artigo=20entre?= =?UTF-8?q?=20os=20sites=20cl=C3=A1ssico=20e=20novo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/tests/integration/test_pipelines.py | 49 +++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/metrics/tests/integration/test_pipelines.py b/metrics/tests/integration/test_pipelines.py index 48c0537..7f0ace5 100644 --- a/metrics/tests/integration/test_pipelines.py +++ b/metrics/tests/integration/test_pipelines.py @@ -1,6 +1,7 @@ import unittest from datetime import datetime +from scielo_usage_counter.url_translator import URLTranslationManager from scielo_usage_counter.values import ( CONTENT_TYPE_ABSTRACT, CONTENT_TYPE_FULL_TEXT, @@ -9,6 +10,7 @@ from metrics.counter.access import accumulation, extraction from metrics.counter.access.daily_accumulator import DailyAccessAccumulator +from metrics.opensearch.keys import document_key from metrics.tests.helpers import convert_accumulator @@ -124,3 +126,50 @@ def test_opac_article_produces_journal_article_metrics(self): self.assertTrue(doc["source_key"].startswith("k1_")) self.assertTrue(doc["document_key"].startswith("k1_")) self.assertEqual(doc["total_requests"], 1) + + def test_classic_and_opac_urls_share_article_and_session(self): + pid_v2 = "S0103-63512021000100001" + pid_v3 = "dqLRqnpmnncSmnzMCB8bzPG" + translator = URLTranslationManager( + sources_metadata=[{"acronym": "neco", "scielo_issn": "0103-6351"}], + documents_metadata=[ + { + "pid_v2": pid_v2, + "pid_v3": pid_v3, + "default_lang": "pt", + "text_langs": ["pt"], + "scielo_issn": "0103-6351", + "publication_year": "2021", + } + ], + ) + results = DailyAccessAccumulator() + urls = ( + f"/scielo.php?script=sci_arttext&pid={pid_v2}", + f"/j/neco/a/{pid_v2}/?format=html", + f"/j/neco/a/{pid_v3}/?format=html", + ) + + for minute, url in enumerate(urls): + line = { + "url": url, + "client_name": "Chrome", + "client_version": "120.0", + "ip_address": "189.10.20.30", + "country_code": "BR", + "local_datetime": datetime(2026, 7, 15, 8, 15 + minute, 0), + } + access = extraction.extract("scl", translator.translate(url)) + accumulation.accumulate(results, access, line) + + metrics = convert_accumulator(results) + self.assertEqual(len(metrics["counter"]), 1) + self.assertEqual(len(metrics["analytics"]), 1) + + article = next(iter(metrics["counter"].values())) + self.assertEqual( + article["document_key"], + document_key("scl", "article", pid_v3), + ) + self.assertEqual(article["total_requests"], 3) + self.assertEqual(article["unique_requests"], 1) From 38ac2b175678ed4ec8df3a77d95248ae20ec6c1a Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 23 Sep 2026 14:42:48 -0300 Subject: [PATCH 3/7] =?UTF-8?q?Controla=20a=20importa=C3=A7=C3=A3o=20de=20?= =?UTF-8?q?meses=20hist=C3=B3ricos=20com=20lacunas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/legacy_matomo/validation.py | 20 ++++++++++++++++++- .../commands/import_legacy_matomo.py | 2 ++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/metrics/legacy_matomo/validation.py b/metrics/legacy_matomo/validation.py index b7e31a6..f0e7976 100644 --- a/metrics/legacy_matomo/validation.py +++ b/metrics/legacy_matomo/validation.py @@ -230,6 +230,7 @@ def validate_manifest( progress_callback=None, progress_interval=DEFAULT_PROGRESS_INTERVAL, stop_controller=None, + allow_partial=False, ): if progress_interval <= 0: raise ValueError("Progress interval must be greater than zero.") @@ -238,6 +239,9 @@ def validate_manifest( if manifest.get("migration") != MIGRATION_NAME: raise ValueError("Unsupported migration manifest.") validate_migration_scope(manifest) + missing_days = manifest.get("missing_days", []) + if missing_days and not allow_partial: + raise ValueError("Partial migration manifest has missing source days.") collection = manifest["target"]["collection"] month = manifest["month"] @@ -262,6 +266,19 @@ def validate_manifest( if value[:7] != month: raise ValueError("Empty day outside migration month: %s." % value) + if missing_days != sorted(set(missing_days)): + raise ValueError("Migration missing days must be unique and sorted.") + if set(missing_days) & (set(source_days) | set(empty_days)): + raise ValueError("Migration missing days overlap available days.") + for value in missing_days: + date.fromisoformat(value) + if value[:7] != month: + raise ValueError("Missing day outside migration month: %s." % value) + + missing_day_totals = manifest.get("missing_day_totals", {}) + if set(missing_day_totals) != set(missing_days): + raise ValueError("Migration missing day totals differ from missing days.") + if manifest["counter"].get("month") != month: raise ValueError("Counter summary month differs from manifest.") if manifest["analytics"].get("source_month") != month: @@ -291,10 +308,11 @@ def validate_manifest( raise ValueError("Counter and analytics totals do not reconcile.") return { - "status": "valid", + "status": "partial" if missing_days else "valid", "collection": collection, "month": month, "source_days": len(source_days), + "missing_days": missing_days, "counter": counter, "analytics": analytics, } diff --git a/metrics/management/commands/import_legacy_matomo.py b/metrics/management/commands/import_legacy_matomo.py index ed68fd9..42b582c 100644 --- a/metrics/management/commands/import_legacy_matomo.py +++ b/metrics/management/commands/import_legacy_matomo.py @@ -29,6 +29,7 @@ def add_arguments(self, parser): parser.add_argument("--report") parser.add_argument("--temporary-directory") parser.add_argument("--progress-every", type=int, default=100000) + parser.add_argument("--allow-partial", action="store_true") def _progress(self, dataset, stage, documents): self.stdout.write("[%s] %s: %d documents" % (stage, dataset, documents)) @@ -72,6 +73,7 @@ def handle(self, *args, **options): progress_callback=self._progress, progress_interval=options["progress_every"], stop_controller=stop_controller, + allow_partial=options["allow_partial"], ) report["validation"] = validation From f10f5ebdebac5240a26509fa186caa5d41366ea0 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 23 Sep 2026 14:42:52 -0300 Subject: [PATCH 4/7] =?UTF-8?q?Testa=20a=20importa=C3=A7=C3=A3o=20expl?= =?UTF-8?q?=C3=ADcita=20de=20pacotes=20hist=C3=B3ricos=20parciais?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/test_legacy_matomo_validation.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/metrics/tests/services/test_legacy_matomo_validation.py b/metrics/tests/services/test_legacy_matomo_validation.py index 5911c1b..5b3c54b 100644 --- a/metrics/tests/services/test_legacy_matomo_validation.py +++ b/metrics/tests/services/test_legacy_matomo_validation.py @@ -60,6 +60,28 @@ def test_migration_identity_includes_payload_hash(self): self.assertIn(self.manifest["counter"]["uncompressed_sha256"], migration_id) + def test_rejects_partial_manifest_with_missing_days(self): + self.manifest["missing_days"] = ["2025-08-02"] + + with self.assertRaisesMessage(ValueError, "missing source days"): + validation.validate_manifest(self.manifest) + + def test_accepts_partial_manifest_when_explicit(self): + self.manifest["missing_days"] = ["2025-08-02"] + self.manifest["missing_day_totals"] = {"2025-08-02": {"nbr": {}}} + + result = validation.validate_manifest(self.manifest, allow_partial=True) + + self.assertEqual(result["status"], "partial") + self.assertEqual(result["missing_days"], ["2025-08-02"]) + + def test_rejects_missing_day_that_overlaps_source_day(self): + self.manifest["missing_days"] = ["2025-08-01"] + self.manifest["missing_day_totals"] = {"2025-08-01": {"nbr": {}}} + + with self.assertRaisesMessage(ValueError, "overlap available days"): + validation.validate_manifest(self.manifest, allow_partial=True) + def test_rejects_manifest_without_source_days(self): self.manifest["source_days"] = [] From 89cd8afa5e890fc7aaa871ffd0e2897e4ca0aa81 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 23 Sep 2026 13:56:45 -0300 Subject: [PATCH 5/7] =?UTF-8?q?Atualiza=20a=20vers=C3=A3o=20da=20aplica?= =?UTF-8?q?=C3=A7=C3=A3o=20para=202.6.1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index e70b452..6a6a3d8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.6.0 +2.6.1 From c2eae18b3dcc05adc2a5c4e5c6dd5360f9374cc9 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 23 Sep 2026 16:14:51 -0300 Subject: [PATCH 6/7] =?UTF-8?q?Aceita=20relat=C3=B3rios=20parciais=20na=20?= =?UTF-8?q?recomposi=C3=A7=C3=A3o=20da=20cobertura=20mensal?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/management/commands/backfill_counter_month_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metrics/management/commands/backfill_counter_month_status.py b/metrics/management/commands/backfill_counter_month_status.py index 3666596..de8b82f 100644 --- a/metrics/management/commands/backfill_counter_month_status.py +++ b/metrics/management/commands/backfill_counter_month_status.py @@ -96,7 +96,7 @@ def _from_reports(self, paths, client): if report.get("status") != "completed": raise CommandError(f"migration report is not completed: {path}") - if report.get("validation", {}).get("status") != "valid": + if report.get("validation", {}).get("status") not in {"valid", "partial"}: raise CommandError(f"migration report is not valid: {path}") for dataset in ("counter", "analytics"): From 844b485633443afe8a883a846a0c54b228702dc6 Mon Sep 17 00:00:00 2001 From: Rafael JPD Date: Wed, 23 Sep 2026 16:14:51 -0300 Subject: [PATCH 7/7] =?UTF-8?q?Testa=20a=20cobertura=20de=20meses=20hist?= =?UTF-8?q?=C3=B3ricos=20com=20lacunas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../services/test_month_status_command.py | 55 ++++++++++++++++++- 1 file changed, 54 insertions(+), 1 deletion(-) diff --git a/metrics/tests/services/test_month_status_command.py b/metrics/tests/services/test_month_status_command.py index 9d721c3..0304b9e 100644 --- a/metrics/tests/services/test_month_status_command.py +++ b/metrics/tests/services/test_month_status_command.py @@ -6,7 +6,7 @@ from unittest.mock import patch from django.core.management import call_command -from django.test import TestCase +from django.test import SimpleTestCase, TestCase from collection.models import Collection from metrics.models import DailyMetricJob @@ -99,3 +99,56 @@ def test_uses_completed_migration_reports(self, client_class, replace_status): date(2025, 2, 1), (1 << 0) | (1 << 1) | (1 << 27), ) + + +class PartialMigrationMonthStatusTests(SimpleTestCase): + @patch( + "metrics.management.commands.backfill_counter_month_status." + "replace_month_status" + ) + @patch( + "metrics.management.commands.backfill_counter_month_status." + "OpenSearchUsageClient" + ) + def test_replaces_partial_month_coverage_from_import_report( + self, client_class, replace_status + ): + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + manifest_path = root / "manifest.json" + report_path = root / "report.json" + manifest = { + "month": "2025-02", + "source_days": ["2025-02-01"], + "empty_days": ["2025-02-02"], + "missing_days": ["2025-02-03"], + "target": {"collection": "scl"}, + } + report = { + "status": "completed", + "manifest": str(manifest_path), + "collection": "scl", + "month": "2025-02", + "validation": { + "status": "partial", + "counter": {"documents": 10}, + "analytics": {"documents": 20}, + }, + "imported": {"counter": 10, "analytics": 20}, + } + manifest_path.write_text(json.dumps(manifest), encoding="utf-8") + report_path.write_text(json.dumps(report), encoding="utf-8") + + call_command( + "backfill_counter_month_status", + report=[str(report_path)], + apply=True, + stdout=StringIO(), + ) + + replace_status.assert_called_once_with( + client_class.return_value, + "scl", + date(2025, 2, 1), + (1 << 0) | (1 << 1), + )