From 7167391366f0deb1c16db09bbd58ae8fba8666a5 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:08 -0300 Subject: [PATCH 01/23] =?UTF-8?q?Compacta=20a=20acumula=C3=A7=C3=A3o=20di?= =?UTF-8?q?=C3=A1ria=20de=20acessos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/counter/access/accumulation.py | 29 ++------ metrics/counter/access/daily_accumulator.py | 74 +++---------------- .../tests/counter/access/test_accumulation.py | 46 ++++++------ .../counter/access/test_daily_accumulator.py | 12 +-- 4 files changed, 42 insertions(+), 119 deletions(-) diff --git a/metrics/counter/access/accumulation.py b/metrics/counter/access/accumulation.py index 3f6a616..a00ef54 100644 --- a/metrics/counter/access/accumulation.py +++ b/metrics/counter/access/accumulation.py @@ -4,7 +4,7 @@ from core.utils.date_utils import coerce_datetime -def accumulate(results, counter_access, line): +def accumulate(results, counter_access, line, reporting_date=None): access_url = counter_access.get("access_url") or _normalized_access_path( line.get("url") ) @@ -18,6 +18,7 @@ def accumulate(results, counter_access, line): if local_datetime is None: raise ValueError("Invalid local_datetime in parsed log line.") + reporting_date = reporting_date or local_datetime.date() access_datetime = local_datetime.replace(minute=0, second=0, microsecond=0) second_of_hour = local_datetime.minute * 60 + local_datetime.second @@ -25,13 +26,13 @@ def accumulate(results, counter_access, line): client_name, client_version, ip_address, - access_datetime.date().toordinal(), + reporting_date.toordinal(), access_datetime.hour, ) raw_record = _build_record( counter_access=counter_access, line=line, - access_datetime=access_datetime, + reporting_date=reporting_date, ) access_url_key = access_url or "|".join( [ @@ -52,7 +53,7 @@ def accumulate(results, counter_access, line): def _build_record( counter_access, line, - access_datetime, + reporting_date, ): collection = counter_access.get("collection") source_key = _source_key(counter_access, collection) @@ -63,7 +64,7 @@ def _build_record( content_language = counter_access.get("media_language") content_type = counter_access.get("content_type") access_country_code = line.get("country_code") - access_date = access_datetime.strftime("%Y-%m-%d") + access_date = reporting_date.strftime("%Y-%m-%d") return { "collection": collection, @@ -72,15 +73,12 @@ def _build_record( "pid_v2": pid_v2, "pid_v3": pid_v3, "pid_generic": pid_generic, - "document": _document_metadata(counter_access), "title_pid_generic": counter_access.get("title_pid_generic") or pid_generic, "media_format": media_format, "content_language": content_language, "content_type": content_type, "access_country_code": access_country_code, "access_date": access_date, - "access_year": access_date[:4], - "access_month": access_date[:7].replace("-", ""), "publication_year": counter_access.get("publication_year"), "counter_access_type": counter_access.get("counter_access_type") or "Open", "access_method": counter_access.get("access_method") or "Regular", @@ -102,25 +100,10 @@ def _normalized_access_path(url): return path or None -def _document_metadata(counter_access): - document_title = counter_access.get("document_title") - return {"title": document_title} if document_title else {} - - def _source_metadata(counter_access): return { "source_type": counter_access.get("source_type"), "source_id": counter_access.get("source_id"), - "scielo_issn": counter_access.get("scielo_issn"), - "main_title": counter_access.get("source_main_title"), - "identifiers": counter_access.get("source_identifiers"), - "access_type": counter_access.get("source_access_type"), - "city": counter_access.get("source_city"), - "country": counter_access.get("source_country"), - "subject_area_capes": counter_access.get("source_subject_area_capes"), - "subject_area_wos": counter_access.get("source_subject_area_wos"), - "acronym": counter_access.get("source_acronym"), - "publisher_name": counter_access.get("source_publisher_name"), } diff --git a/metrics/counter/access/daily_accumulator.py b/metrics/counter/access/daily_accumulator.py index 56e1793..b728163 100644 --- a/metrics/counter/access/daily_accumulator.py +++ b/metrics/counter/access/daily_accumulator.py @@ -2,22 +2,17 @@ _ACCESS_KEY = Struct("!10I") _SESSION_KEY = Struct("!5I") -_NONE_METADATA_ID = 0 -_EMPTY_DOCUMENT_ID = 1 class _CompactAccessRecord: __slots__ = ( "access_method", "access_date", - "access_month", - "access_year", "collection", "content_language", "content_type", "counter_access_type", "country_code", - "document", "document_type", "first_second", "first_url", @@ -28,7 +23,8 @@ class _CompactAccessRecord: "pid_v3", "publication_year", "session", - "source", + "source_id", + "source_type", "source_key", "title_pid_generic", ) @@ -46,16 +42,12 @@ def __init__(self, accumulator, data, session, url, second): self.content_type = accumulator._intern(data.get("content_type")) self.country_code = accumulator._intern(data.get("access_country_code")) self.access_date = accumulator._intern(data.get("access_date")) - self.access_year = accumulator._intern(data.get("access_year")) - self.access_month = accumulator._intern(data.get("access_month")) self.publication_year = accumulator._intern(data.get("publication_year")) self.counter_access_type = accumulator._intern(data.get("counter_access_type")) self.access_method = accumulator._intern(data.get("access_method")) - self.source = accumulator._intern_source( - data.get("source_key"), - data.get("source"), - ) - self.document = accumulator._intern_document(data, share_empty=True) + source = data.get("source") or {} + self.source_id = accumulator._intern(source.get("source_id")) + self.source_type = accumulator._intern(source.get("source_type")) self.session = session self.first_url = accumulator._intern(url) self.first_second = second @@ -84,7 +76,6 @@ def as_dict(self, accumulator): "pid_v2": accumulator._resolve(self.pid_v2), "pid_v3": accumulator._resolve(self.pid_v3), "pid_generic": accumulator._resolve(self.pid_generic), - "document": accumulator._documents[self.document], "title_pid_generic": accumulator._resolve(self.title_pid_generic), "user_session_id": self.session, "click_timestamps_by_url": self._timestamps_as_dict(accumulator), @@ -93,12 +84,13 @@ def as_dict(self, accumulator): "content_type": accumulator._resolve(self.content_type), "access_country_code": accumulator._resolve(self.country_code), "access_date": accumulator._resolve(self.access_date), - "access_year": accumulator._resolve(self.access_year), - "access_month": accumulator._resolve(self.access_month), "publication_year": accumulator._resolve(self.publication_year), "counter_access_type": accumulator._resolve(self.counter_access_type), "access_method": accumulator._resolve(self.access_method), - "source": accumulator._sources[self.source], + "source": { + "source_id": accumulator._resolve(self.source_id), + "source_type": accumulator._resolve(self.source_type), + }, } def _timestamps_as_dict(self, accumulator): @@ -122,10 +114,6 @@ class DailyAccessAccumulator: def __init__(self): self._records = {} - self._documents = [None, {}] - self._document_ids = {} - self._sources = [None] - self._source_ids = {} self._sessions = {} self._strings = [None] self._string_ids = {} @@ -181,10 +169,6 @@ def iter_materialized_record_keys(self, record_keys, consume=False): def clear(self): self._records.clear() - self._documents.clear() - self._document_ids.clear() - self._sources.clear() - self._source_ids.clear() self._sessions.clear() self._strings.clear() self._string_ids.clear() @@ -215,43 +199,3 @@ def _intern_session(self, session_key): session_id = len(self._sessions) + 1 self._sessions[compact_key] = session_id return session_id - - def _intern_source(self, source_key, source): - if source is None: - return _NONE_METADATA_ID - if not source_key: - self._sources.append(source) - return len(self._sources) - 1 - source_id = self._source_ids.get(source_key) - if source_id is None: - source_id = len(self._sources) - self._source_ids[source_key] = source_id - self._sources.append(source) - return source_id - - def _intern_document(self, data, share_empty=False): - document_key = self._document_key(data) - document = data.get("document") - if document is None: - return _NONE_METADATA_ID - if share_empty and not document: - return _EMPTY_DOCUMENT_ID - if not any(document_key[1:]): - self._documents.append(document) - return len(self._documents) - 1 - document_id = self._document_ids.get(document_key) - if document_id is None: - document_id = len(self._documents) - self._document_ids[document_key] = document_id - self._documents.append(document) - return document_id - - @staticmethod - def _document_key(data): - return ( - data.get("document_type"), - data.get("pid_v2"), - data.get("pid_v3"), - data.get("pid_generic"), - data.get("title_pid_generic"), - ) diff --git a/metrics/tests/counter/access/test_accumulation.py b/metrics/tests/counter/access/test_accumulation.py index 6c4c45e..6c81f0b 100644 --- a/metrics/tests/counter/access/test_accumulation.py +++ b/metrics/tests/counter/access/test_accumulation.py @@ -1,5 +1,5 @@ import unittest -from datetime import datetime +from datetime import date, datetime from scielo_usage_counter.values import ( CONTENT_TYPE_FULL_TEXT, @@ -57,16 +57,28 @@ def test_stores_source_and_periods(self): result = next(results.iter_materialized_values()) self.assertEqual(result["source"]["source_type"], "book") self.assertEqual(result["source"]["source_id"], "q7gtd") - self.assertEqual(result["source"]["main_title"], "Book Title") self.assertEqual(result["access_date"], "2024-01-15") - self.assertEqual(result["access_month"], "202401") - self.assertEqual(result["access_year"], "2024") + self.assertNotIn("access_month", result) + self.assertNotIn("access_year", result) self.assertEqual(result["access_country_code"], "BR") self.assertEqual(result["content_language"], "en") self.assertEqual(result["title_pid_generic"], "BOOK:Q7GTD") - self.assertEqual(result["document"], {"title": "Book Title"}) + self.assertNotIn("document", result) self.assertIn("user_session_id", result) + def test_reporting_date_overrides_timestamp_date(self): + results = DailyAccessAccumulator() + accumulation.accumulate( + results, + self._book_counter_access(), + self._line(local_datetime=datetime(2025, 12, 31, 23, 59, 59)), + reporting_date=date(2026, 1, 1), + ) + + result = next(results.iter_materialized_values()) + + self.assertEqual(result["access_date"], "2026-01-01") + def test_rejects_invalid_local_datetime(self): results = DailyAccessAccumulator() with self.assertRaises(ValueError): @@ -223,10 +235,10 @@ def test_compact_accumulator_preserves_metrics_after_repeated_events(self): accumulation.accumulate(compact, self._book_counter_access(), event) values = list(compact.iter_materialized_values()) - month = dict(index_docs.iter_partitioned_values(values, "month")) - year = dict(index_docs.iter_partitioned_values(values, "year")) - self.assertEqual(len(month), 2) - self.assertEqual(len(year), 2) + counter = dict(index_docs.iter_partitioned_values(values, "counter")) + analytics = dict(index_docs.iter_partitioned_values(values, "analytics")) + self.assertEqual(len(counter), 2) + self.assertEqual(len(analytics), 2) def test_compact_accumulator_promotes_only_repeated_timestamps(self): compact = DailyAccessAccumulator() @@ -257,7 +269,7 @@ def test_compact_accumulator_promotes_only_repeated_timestamps(self): {"/id/q7gtd/full-text": {5: 1, 20: 1}}, ) - def test_compact_accumulator_reuses_missing_metadata_slots(self): + def test_compact_accumulator_does_not_retain_display_metadata(self): compact = DailyAccessAccumulator() counter_access = self._book_counter_access( source_type=None, @@ -279,24 +291,12 @@ def test_compact_accumulator_reuses_missing_metadata_slots(self): ) self.assertEqual(len(compact), 2) - self.assertEqual(compact._documents, [None, {}]) - self.assertEqual(len(compact._sources), 2) for record in compact.iter_materialized_values(): - self.assertEqual(record["document"], {}) + self.assertNotIn("document", record) self.assertEqual( record["source"], { "source_type": None, "source_id": None, - "scielo_issn": None, - "main_title": None, - "identifiers": None, - "access_type": None, - "city": None, - "country": None, - "subject_area_capes": [], - "subject_area_wos": [], - "acronym": None, - "publisher_name": ["SciELO Books"], }, ) diff --git a/metrics/tests/counter/access/test_daily_accumulator.py b/metrics/tests/counter/access/test_daily_accumulator.py index 819dcbf..99b4d68 100644 --- a/metrics/tests/counter/access/test_daily_accumulator.py +++ b/metrics/tests/counter/access/test_daily_accumulator.py @@ -43,10 +43,6 @@ def test_consuming_materialization_releases_all_internal_structures(): assert [value["pid_v3"] for value in values] == ["first", "second"] assert len(accumulator) == 0 - assert accumulator._documents == [] - assert accumulator._document_ids == {} - assert accumulator._sources == [] - assert accumulator._source_ids == {} assert accumulator._sessions == {} assert accumulator._strings == [] assert accumulator._string_ids == {} @@ -61,10 +57,10 @@ def test_consuming_materialization_releases_structures_after_consumer_error(): values.close() assert len(accumulator) == 0 - assert accumulator._documents == [] + assert accumulator._strings == [] -def test_partitioned_year_conversion_releases_accumulator_after_consumer_error(): +def test_partitioned_analytics_conversion_releases_accumulator_after_consumer_error(): from metrics.counter.indexing import converter accumulator = DailyAccessAccumulator() @@ -72,7 +68,7 @@ def test_partitioned_year_conversion_releases_accumulator_after_consumer_error() _accumulate(accumulator, "second", "127.0.0.2") documents = converter.iter_partitioned_documents( accumulator, - "year", + "analytics", partition_count=2, ) @@ -80,4 +76,4 @@ def test_partitioned_year_conversion_releases_accumulator_after_consumer_error() documents.close() assert len(accumulator) == 0 - assert accumulator._documents == [] + assert accumulator._strings == [] From 55a66f9099e079fedec19ef090fecdd03cb333f5 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:19 -0300 Subject: [PATCH 02/23] =?UTF-8?q?Converte=20acessos=20em=20fatos=20COUNTER?= =?UTF-8?q?=20e=20anal=C3=ADticos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/counter/indexing/converter.py | 57 +- metrics/counter/indexing/engines/base.py | 321 ++++------ metrics/counter/indexing/engines/book.py | 140 ++--- metrics/opensearch/keys.py | 68 +++ .../tests/counter/indexing/test_converter.py | 570 +++--------------- metrics/tests/helpers.py | 4 +- .../integration/test_books_log_to_metrics.py | 29 +- .../test_bunnynet_log_to_metrics.py | 14 +- .../test_classic_log_to_metrics.py | 15 +- metrics/tests/integration/test_pipelines.py | 36 +- 10 files changed, 412 insertions(+), 842 deletions(-) create mode 100644 metrics/opensearch/keys.py diff --git a/metrics/counter/indexing/converter.py b/metrics/counter/indexing/converter.py index c4c73a6..153d536 100644 --- a/metrics/counter/indexing/converter.py +++ b/metrics/counter/indexing/converter.py @@ -15,29 +15,46 @@ } _DEFAULT = DocumentPipeline() _DEFAULT_PARTITION_COUNT = 64 +ANALYTICS_PROJECTION = "country_language" def iter_partitioned_documents( accumulator, - granularity, + dataset, partition_count=_DEFAULT_PARTITION_COUNT, ): + if dataset not in {"counter", "analytics"}: + raise ValueError("Dataset must be 'counter' or 'analytics'.") if partition_count <= 0: raise ValueError("Partition count must be greater than zero.") - consume = granularity == "year" + consume = dataset == "analytics" partitions = [[] for _ in range(partition_count)] for record_key, value in accumulator.iter_materialized_record_items(): - partition = _partition_for_value(value, granularity, partition_count) + partition = _partition_for_value( + value, + partition_count, + projection=ANALYTICS_PROJECTION if dataset == "analytics" else None, + ) partitions[partition].append(record_key) try: for record_keys in partitions: - values = accumulator.iter_materialized_record_keys( - record_keys, - consume=consume, - ) - yield from _convert_partition(values, granularity) + if dataset == "counter": + values = accumulator.iter_materialized_record_keys(record_keys) + yield from _convert_partition(values) + else: + materialized = list( + accumulator.iter_materialized_record_keys( + record_keys, + consume=True, + ) + ) + yield from _convert_partition( + materialized, + projection=ANALYTICS_PROJECTION, + ) + materialized.clear() record_keys.clear() finally: for record_keys in partitions: @@ -49,7 +66,7 @@ def iter_partitioned_documents( def iter_partitioned_values( values, - granularity, + dataset, partition_count=_DEFAULT_PARTITION_COUNT, ): if partition_count <= 0: @@ -57,12 +74,22 @@ def iter_partitioned_values( partitions = [[] for _ in range(partition_count)] for value in values: - partition = _partition_for_value(value, granularity, partition_count) + partition = _partition_for_value( + value, + partition_count, + projection=ANALYTICS_PROJECTION if dataset == "analytics" else None, + ) partitions[partition].append(value) try: for partition_values in partitions: - yield from _convert_partition(partition_values, granularity) + if dataset == "counter": + yield from _convert_partition(partition_values) + else: + yield from _convert_partition( + partition_values, + projection=ANALYTICS_PROJECTION, + ) partition_values.clear() finally: for partition_values in partitions: @@ -70,9 +97,9 @@ def iter_partitioned_values( partitions.clear() -def _partition_for_value(value, granularity, partition_count): +def _partition_for_value(value, partition_count, projection=None): pipeline = _get_pipeline(value) - partition_key = pipeline.partition_key(value, granularity) + partition_key = pipeline.partition_key(value, projection=projection) digest = hashlib.blake2b( partition_key.encode("utf-8"), digest_size=8, @@ -80,7 +107,7 @@ def _partition_for_value(value, granularity, partition_count): return int.from_bytes(digest, "big") % partition_count -def _convert_partition(values, granularity): +def _convert_partition(values, projection=None): converted_data = {} unique_state = _initialize_unique_state() @@ -91,7 +118,7 @@ def _convert_partition(values, granularity): data=converted_data, unique_state=unique_state, value=value, - granularity=granularity, + projection=projection, ) for document_id in sorted(converted_data): diff --git a/metrics/counter/indexing/engines/base.py b/metrics/counter/indexing/engines/base.py index e0ca076..2803979 100644 --- a/metrics/counter/indexing/engines/base.py +++ b/metrics/counter/indexing/engines/base.py @@ -1,17 +1,19 @@ from scielo_usage_counter.counter import get_valid_clicks, is_request +from metrics.opensearch.keys import UNKNOWN_VALUE, document_key, metric_key, source_key + class DocumentPipeline: data_type = "Other" - def accumulate(self, data, unique_state, value, granularity): + def accumulate(self, data, unique_state, value, projection=None): if not isinstance(value, dict): return - document_id = self._generate_document_id(value, granularity) + document_id = self._generate_document_id(value, projection=projection) document = data.setdefault( document_id, - self._build_document(value=value, granularity=granularity), + self._build_document(value=value, projection=projection), ) self._apply_totals( @@ -29,173 +31,134 @@ def accumulate(self, data, unique_state, value, granularity): is_request_event=is_request(value.get("content_type")), ) - def partition_key(self, value, granularity): - return self._generate_document_id(value, granularity) + def partition_key(self, value, projection=None): + return self._generate_document_id(value, projection=projection) def _generate_document_id( - self, value, granularity, metric_scope=None, pid_generic=None + self, value, projection=None, metric_scope=None, pid_generic=None ): pid_generic = pid_generic or value.get("pid_generic") - publication_year = str(value.get("publication_year") or "0001") - if granularity == "month": - access_month = ( - value.get("access_date", "")[:7] if value.get("access_date") else "" - ) - return _generate_month_document_id( - collection=value.get("collection"), - source_key=value.get("source_key"), - pid_v2=value.get("pid_v2"), - pid_v3=value.get("pid_v3"), - pid_generic=pid_generic, - access_month=access_month, - counter_access_type=value.get("counter_access_type") or "Open", - access_method=value.get("access_method") or "Regular", - publication_year=publication_year, - metric_scope="title" if metric_scope == "title" else None, - ) - - return _generate_year_document_id( - collection=value.get("collection"), - source_key=value.get("source_key"), - pid_v2=value.get("pid_v2"), - pid_v3=value.get("pid_v3"), + counter = self._build_counter_section(value) + if metric_scope == "title": + counter["metric_scope"] = "title" + counter["data_type"] = "Book" + counter.pop("parent_data_type", None) + source_identifier = self._source_key(value) + document_identifier = self._document_key( + value, pid_generic=pid_generic, - content_language=value.get("content_language"), - access_country_code=value.get("access_country_code"), - access_year=value.get("access_year"), - counter_access_type=value.get("counter_access_type") or "Open", - access_method=value.get("access_method") or "Regular", - publication_year=publication_year, - metric_scope="title" if metric_scope == "title" else None, + metric_scope=metric_scope, ) + country_code, content_language = self._projection_values(value, projection) + return metric_key( + collection=value.get("collection"), + source_identifier=source_identifier, + document_identifier=document_identifier, + period=( + self._access_year(value) if projection else self._access_month(value) + ), + metric_scope=counter.get("metric_scope"), + data_type=counter.get("data_type"), + parent_data_type=counter.get("parent_data_type"), + article_version=counter.get("article_version"), + access_type=counter.get("access_type"), + access_method=counter.get("access_method"), + projection=projection, + country_code=country_code, + content_language=content_language, + ) + + def _build_document(self, value, projection=None, **kwargs): + counter = self._build_counter_section(value) + source_identifier = self._source_key(value) + document_identifier = self._document_key(value) - def _build_document(self, value, granularity, **kwargs): - document_type = value.get("document_type") - document_id = self._resolve_document_id(value) + if projection: + country_code, content_language = self._projection_values( + value, + projection, + ) + analytics_document = { + "year": self._access_year(value), + "source_key": source_identifier, + "document_key": document_identifier, + **counter, + "country_code": country_code, + "content_language": content_language, + "total_requests": 0, + "total_investigations": 0, + "unique_requests": 0, + "unique_investigations": 0, + } + return _strip_empty_values(analytics_document) base_document = { "collection": value.get("collection"), - "source": self._build_source(value.get("source")), - "document": self._build_document_section( - value=value, - document_id=document_id, - document_type=document_type, - ), - "counter": self._build_counter_section(value), + "source_key": source_identifier, + "document_key": document_identifier, + "month": self._access_month(value), + **counter, "total_requests": 0, "total_investigations": 0, "unique_requests": 0, "unique_investigations": 0, } - - base_document["access"] = self._build_access(value, granularity) - if granularity == "month": - base_document["daily_metrics"] = self._build_daily_metrics(value) - return base_document + base_document["daily_metrics"] = self._build_daily_metrics(value) + return _strip_empty_values(base_document) def _resolve_document_id(self, value): return value.get("pid_v3") or value.get("pid_v2") or value.get("pid_generic") - def _resolve_parent_data_type(self, value): - return None - - def _build_counter_section(self, value): - return _strip_empty_values( - { - "metric_scope": "item", - "data_type": self.data_type, - "parent_data_type": self._resolve_parent_data_type(value), - "access_type": value.get("counter_access_type") or "Open", - "access_method": value.get("access_method") or "Regular", - } + def _source_key(self, value): + source = value.get("source") or {} + raw_source_id = source.get("source_id") or value.get("source_key") + return source_key( + value.get("collection"), + source.get("source_type"), + raw_source_id, ) - def _build_document_section( - self, - value, - document_id, - document_type, - parent_id=None, - source_identifiers=None, - metric_scope="item", - ): - document = value.get("document") or {} - title = document.get("title") - if metric_scope == "title": - title = (value.get("source") or {}).get("main_title") or title - - identifiers = self._document_identifiers( - value=value, - document_id=document_id, - source_identifiers=source_identifiers, - metric_scope=metric_scope, + def _document_key(self, value, pid_generic=None, metric_scope=None): + raw_document_id = pid_generic or self._resolve_document_id(value) + document_type = ( + "book" if metric_scope == "title" else value.get("document_type") ) - - return _strip_empty_values( - { - "id": document_id, - "type": document_type, - "title": title, - "parent_id": parent_id, - "publication_year": value.get("publication_year"), - "identifiers": identifiers, - } + return document_key( + value.get("collection"), + document_type, + raw_document_id, ) - def _document_identifiers( - self, value, document_id, source_identifiers=None, metric_scope="item" - ): - document_identifiers = (value.get("document") or {}).get("identifiers") or {} - identifiers = { - "pid_v2": value.get("pid_v2"), - "pid_v3": value.get("pid_v3"), - "pid_generic": value.get("pid_generic"), - } - identifiers.update(document_identifiers) - return _strip_empty_identifiers(identifiers, canonical_id=document_id) - @staticmethod - def _build_source(source): - source = source or {} - source_id = source.get("source_id") - source_type = source.get("source_type") - identifiers = _strip_empty_identifiers( - source.get("identifiers") or {}, canonical_id=source_id - ) + def _access_month(value): + access_date = value.get("access_date") or "" + return access_date[:7] - return _strip_empty_values( - { - "id": source_id, - "type": source_type, - "title": source.get("main_title"), - "scielo_issn": None - if source_type == "book" - else source.get("scielo_issn"), - "acronym": source.get("acronym"), - "publisher_name": source.get("publisher_name"), - "subject_area_capes": source.get("subject_area_capes"), - "subject_area_wos": source.get("subject_area_wos"), - "access_type": source.get("access_type"), - "city": source.get("city"), - "country": source.get("country"), - "identifiers": identifiers, - } - ) + @staticmethod + def _access_year(value): + access_date = value.get("access_date") or "" + return access_date[:4] @staticmethod - def _build_access(value, granularity): - if granularity == "month": - return { - "month": value.get("access_date", "")[:7] - if value.get("access_date") - else "" - } + def _projection_values(value, projection): + if projection == "country_language": + return ( + value.get("access_country_code") or UNKNOWN_VALUE, + value.get("content_language") or UNKNOWN_VALUE, + ) + return None, None + + def _resolve_parent_data_type(self, value): + return None + def _build_counter_section(self, value): return _strip_empty_values( { - "year": value.get("access_year"), - "country_code": value.get("access_country_code"), - "content_language": value.get("content_language"), + "metric_scope": "item", + "data_type": self.data_type, + "parent_data_type": self._resolve_parent_data_type(value), + "access_type": value.get("counter_access_type") or "Open", + "access_method": value.get("access_method") or "Regular", } ) @@ -288,87 +251,7 @@ def _count_valid_clicks(click_timestamps, click_timestamps_by_url=None): return get_valid_clicks(click_timestamps or {}) -def _strip_empty_identifiers(identifiers, canonical_id=None): - compact = {} - canonical_value = str(canonical_id or "").strip().upper() - for key, value in (identifiers or {}).items(): - if value in (None, "", [], {}, ()): - continue - if canonical_value and str(value).strip().upper() == canonical_value: - continue - compact[key] = value - return compact - - def _strip_empty_values(data): return { key: value for key, value in data.items() if value not in (None, "", [], {}, ()) } - - -def _generate_month_document_id( - collection, - source_key, - pid_v2, - pid_v3, - pid_generic, - access_month, - counter_access_type, - access_method, - publication_year, - metric_scope=None, -): - parts = [] - if metric_scope: - parts.append(metric_scope) - - parts.extend( - [ - str(collection or ""), - str(source_key or ""), - pid_v2 or "", - pid_v3 or "", - pid_generic or "", - str(access_month or ""), - str(counter_access_type or ""), - str(access_method or ""), - str(publication_year or ""), - ] - ) - return "|".join(parts) - - -def _generate_year_document_id( - collection, - source_key, - pid_v2, - pid_v3, - pid_generic, - content_language, - access_country_code, - access_year, - counter_access_type, - access_method, - publication_year, - metric_scope=None, -): - parts = [] - if metric_scope: - parts.append(metric_scope) - - parts.extend( - [ - str(collection or ""), - str(source_key or ""), - pid_v2 or "", - pid_v3 or "", - pid_generic or "", - content_language or "", - access_country_code or "", - str(access_year or ""), - str(counter_access_type or ""), - str(access_method or ""), - str(publication_year or ""), - ] - ) - return "|".join(parts) diff --git a/metrics/counter/indexing/engines/book.py b/metrics/counter/indexing/engines/book.py index e52b570..a94ed91 100644 --- a/metrics/counter/indexing/engines/book.py +++ b/metrics/counter/indexing/engines/book.py @@ -1,50 +1,44 @@ from scielo_usage_counter.counter import is_request -from metrics.counter.indexing.engines.base import ( - DocumentPipeline, - _strip_empty_identifiers, - _strip_empty_values, -) +from metrics.counter.indexing.engines.base import DocumentPipeline, _strip_empty_values class BookPipeline(DocumentPipeline): - def partition_key(self, value, granularity): + def partition_key(self, value, projection=None): title_pid_generic = _extract_title_pid_generic(value) if title_pid_generic: return self._generate_document_id( value, - granularity, + projection=projection, metric_scope="title", pid_generic=title_pid_generic, ) - return self._generate_document_id(value, granularity) + return self._generate_document_id(value, projection=projection) - def accumulate(self, data, unique_state, value, granularity): + def accumulate(self, data, unique_state, value, projection=None): if not isinstance(value, dict): return if _should_create_item_document(value): - self._accumulate_item(data, unique_state, value, granularity) + self._accumulate_item(data, unique_state, value, projection) title_pid_generic = _extract_title_pid_generic(value) if not title_pid_generic: return - self._accumulate_title( - data, unique_state, value, granularity, title_pid_generic - ) + self._accumulate_title(data, unique_state, value, projection, title_pid_generic) - def _accumulate_item(self, data, unique_state, value, granularity): + def _accumulate_item(self, data, unique_state, value, projection): item_document_id = self._generate_document_id( value, - granularity, + projection=projection, metric_scope="item", ) item_document = data.setdefault( item_document_id, self._build_document( value=value, - granularity=granularity, + projection=projection, metric_scope="item", ), ) @@ -64,11 +58,11 @@ def _accumulate_item(self, data, unique_state, value, granularity): ) def _accumulate_title( - self, data, unique_state, value, granularity, title_pid_generic + self, data, unique_state, value, projection, title_pid_generic ): title_document_id = self._generate_document_id( value, - granularity, + projection=projection, metric_scope="title", pid_generic=title_pid_generic, ) @@ -76,10 +70,9 @@ def _accumulate_title( title_document_id, self._build_document( value=value, - granularity=granularity, + projection=projection, metric_scope="title", pid_generic=title_pid_generic, - document_type="book", ), ) self._apply_totals( @@ -97,67 +90,61 @@ def _accumulate_title( is_request_event=is_request(value.get("content_type")), ) - def _build_document(self, value, granularity, **kwargs): + def _build_document(self, value, projection=None, **kwargs): metric_scope = kwargs.get("metric_scope") or "item" pid_generic = kwargs.get("pid_generic") - document_type = kwargs.get("document_type") - document_id = pid_generic or value.get("pid_generic") - parent_id = _extract_title_pid_generic(value, fallback=document_id) - if parent_id == document_id or metric_scope == "title": - parent_id = None - raw_source = value.get("source") or {} - source = self._build_source(raw_source) + counter = _strip_empty_values( + { + "metric_scope": metric_scope, + "data_type": "Book" if metric_scope == "title" else "Book_Segment", + "parent_data_type": "Book" if metric_scope != "title" else None, + "access_type": value.get("counter_access_type") or "Open", + "access_method": value.get("access_method") or "Regular", + } + ) + + if projection: + country_code, content_language = self._projection_values( + value, + projection, + ) + analytics_document = { + "year": self._access_year(value), + "source_key": self._source_key(value), + "document_key": self._document_key( + value, + pid_generic=document_id, + metric_scope=metric_scope, + ), + **counter, + "country_code": country_code, + "content_language": content_language, + "total_requests": 0, + "total_investigations": 0, + "unique_requests": 0, + "unique_investigations": 0, + } + return _strip_empty_values(analytics_document) base_document = { "collection": value.get("collection"), - "source": source, - "document": self._build_document_section( - value=value, - document_id=document_id, - document_type=document_type or value.get("document_type"), - parent_id=parent_id, - source_identifiers=raw_source.get("identifiers"), + "source_key": self._source_key(value), + "document_key": self._document_key( + value, + pid_generic=document_id, metric_scope=metric_scope, ), - "counter": _strip_empty_values( - { - "metric_scope": metric_scope, - "data_type": "Book" if metric_scope == "title" else "Book_Segment", - "parent_data_type": "Book" if metric_scope != "title" else None, - "access_type": value.get("counter_access_type") or "Open", - "access_method": value.get("access_method") or "Regular", - } - ), + "month": self._access_month(value), + **counter, "total_requests": 0, "total_investigations": 0, "unique_requests": 0, "unique_investigations": 0, } - base_document["access"] = self._build_access(value, granularity) - if granularity == "month": - base_document["daily_metrics"] = self._build_daily_metrics(value) - return base_document - - def _document_identifiers( - self, value, document_id, source_identifiers=None, metric_scope="item" - ): - if metric_scope == "title": - identifiers = _book_identifiers_from_pid(document_id) - identifiers.update(source_identifiers or {}) - return _strip_empty_identifiers(identifiers, canonical_id=document_id) - - document_identifiers = (value.get("document") or {}).get("identifiers") or {} - identifiers = { - "pid_v2": value.get("pid_v2"), - "pid_v3": value.get("pid_v3"), - "pid_generic": value.get("pid_generic"), - } - identifiers.update(document_identifiers) - identifiers.update(_book_identifiers_from_pid(value.get("pid_generic"))) - identifiers.update(source_identifiers or {}) - return _strip_empty_identifiers(identifiers, canonical_id=document_id) + base_document["daily_metrics"] = self._build_daily_metrics(value) + return _strip_empty_values(base_document) def _should_create_item_document(value): @@ -185,22 +172,3 @@ def _extract_title_pid_generic(value, fallback=None): return f"BOOK:{str(source_id).upper()}" return fallback - - -def _book_identifiers_from_pid(pid_generic): - value = str(pid_generic or "") - if not value.upper().startswith("BOOK:"): - return {} - - identifiers = {} - parts = value.split("/", 1) - book_id = parts[0].split(":", 1)[1] if ":" in parts[0] else "" - if book_id: - identifiers["book_id"] = book_id - - if len(parts) > 1 and parts[1].upper().startswith("CHAPTER:"): - chapter_id = parts[1].split(":", 1)[1] if ":" in parts[1] else "" - if chapter_id: - identifiers["chapter_id"] = chapter_id - - return identifiers diff --git a/metrics/opensearch/keys.py b/metrics/opensearch/keys.py new file mode 100644 index 0000000..61cfe82 --- /dev/null +++ b/metrics/opensearch/keys.py @@ -0,0 +1,68 @@ +import base64 +import hashlib +import json + +KEY_PREFIX = "k1_" +UNKNOWN_VALUE = "_unknown" + + +def stable_key(*parts): + payload = json.dumps( + list(parts), + ensure_ascii=True, + separators=(",", ":"), + ).encode("utf-8") + digest = hashlib.blake2b(payload, digest_size=16).digest() + token = base64.urlsafe_b64encode(digest).decode("ascii").rstrip("=") + return f"{KEY_PREFIX}{token}" + + +def source_key(collection, source_type, source_id): + return stable_key( + "source", + collection or UNKNOWN_VALUE, + source_type or UNKNOWN_VALUE, + source_id or UNKNOWN_VALUE, + ) + + +def document_key(collection, document_type, document_id): + return stable_key( + "document", + collection or UNKNOWN_VALUE, + document_type or UNKNOWN_VALUE, + document_id or UNKNOWN_VALUE, + ) + + +def metric_key( + collection, + source_identifier, + document_identifier, + period, + metric_scope, + data_type, + parent_data_type, + article_version, + access_type, + access_method, + projection=None, + country_code=None, + content_language=None, +): + return stable_key( + "metric", + collection or UNKNOWN_VALUE, + source_identifier or UNKNOWN_VALUE, + document_identifier or UNKNOWN_VALUE, + period or UNKNOWN_VALUE, + metric_scope or UNKNOWN_VALUE, + data_type or UNKNOWN_VALUE, + parent_data_type or UNKNOWN_VALUE, + article_version or UNKNOWN_VALUE, + access_type or UNKNOWN_VALUE, + access_method or UNKNOWN_VALUE, + projection or "counter", + country_code or UNKNOWN_VALUE, + content_language or UNKNOWN_VALUE, + ) diff --git a/metrics/tests/counter/indexing/test_converter.py b/metrics/tests/counter/indexing/test_converter.py index 45cd93e..30e9999 100644 --- a/metrics/tests/counter/indexing/test_converter.py +++ b/metrics/tests/counter/indexing/test_converter.py @@ -1,484 +1,112 @@ -import unittest +from scielo_usage_counter.values import CONTENT_TYPE_FULL_TEXT -from scielo_usage_counter.values import ( - CONTENT_TYPE_ABSTRACT, - CONTENT_TYPE_FULL_TEXT, - DEFAULT_SCIELO_ISSN, - MEDIA_FORMAT_HTML, -) +from metrics.counter.indexing import converter -from metrics.counter.indexing import converter as index_docs - -def _convert(data): - values = list(data.values()) +def _article(country="BR", language="pt", session="session-1"): return { - "month": dict(index_docs.iter_partitioned_values(values, "month")), - "year": dict(index_docs.iter_partitioned_values(values, "year")), + "collection": "scl", + "source_key": "0101-0101", + "document_type": "article", + "pid_v3": "abc123", + "user_session_id": session, + "click_timestamps": {"00:05": 1}, + "access_country_code": country, + "content_language": language, + "content_type": CONTENT_TYPE_FULL_TEXT, + "access_date": "2026-08-20", + "publication_year": "2025", + "source": { + "source_type": "journal", + "source_id": "0101-0101", + "main_title": "Journal title that must not be copied", + }, + "document": {"title": "Article title that must not be copied"}, } -class TestConverter(unittest.TestCase): - def test_creates_month_and_year_views_for_book_chapter(self): - data = { - "books|q7gtd|||BOOK:Q7GTD/CHAPTER:03|browser|1.0|127.0.0.1|BR|en|html|full_text": { - "collection": "books", - "source_key": "q7gtd", - "document_type": "chapter", - "pid_v2": None, - "pid_v3": None, - "pid_generic": "BOOK:Q7GTD/CHAPTER:03", - "document": {"title": "Chapter Title"}, - "title_pid_generic": "BOOK:Q7GTD", - "user_session_id": "browser|1.0|127.0.0.1|2024-01-15|10", - "click_timestamps": {"00:05": 1}, - "access_country_code": "BR", - "content_language": "en", - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_month": "202401", - "access_year": "2024", - "source": { - "source_type": "book", - "source_id": "q7gtd", - "scielo_issn": DEFAULT_SCIELO_ISSN, - "main_title": "Book Title", - "identifiers": {"book_id": "q7gtd", "isbn": "9788578791889"}, - "city": "Sao Paulo", - "country": "BR", - "subject_area_capes": [], - "subject_area_wos": [], - "acronym": None, - "publisher_name": ["SciELO Books"], - }, - "publication_year": "2023", - } - } - - metrics_data = _convert(data) - - self.assertEqual(set(metrics_data.keys()), {"month", "year"}) - self.assertEqual(len(metrics_data["month"]), 2) - self.assertEqual(len(metrics_data["year"]), 2) - - month_item = metrics_data["month"][ - "books|q7gtd|||BOOK:Q7GTD/CHAPTER:03|2024-01|Open|Regular|2023" - ] - self.assertEqual(month_item["access"], {"month": "2024-01"}) - self.assertIn("daily_metrics", month_item) - self.assertNotIn("access_country_code", month_item) - self.assertNotIn("content_language", month_item) - self.assertEqual(month_item["document"]["id"], "BOOK:Q7GTD/CHAPTER:03") - self.assertEqual(month_item["document"]["type"], "chapter") - self.assertEqual(month_item["document"]["title"], "Chapter Title") - self.assertEqual(month_item["document"]["parent_id"], "BOOK:Q7GTD") - self.assertEqual(month_item["document"]["publication_year"], "2023") - self.assertEqual(month_item["document"]["identifiers"]["book_id"], "q7gtd") - self.assertEqual(month_item["document"]["identifiers"]["chapter_id"], "03") - self.assertEqual(month_item["document"]["identifiers"]["isbn"], "9788578791889") - self.assertNotIn("pid_generic", month_item["document"]["identifiers"]) - self.assertEqual(month_item["counter"]["metric_scope"], "item") - self.assertEqual(month_item["counter"]["data_type"], "Book_Segment") - self.assertEqual(month_item["total_requests"], 1) - self.assertEqual(month_item["unique_requests"], 1) - self.assertNotIn("scielo_issn", month_item["source"]) - self.assertNotIn("book_id", month_item["source"].get("identifiers", {})) - self.assertEqual(month_item["source"]["publisher_name"], ["SciELO Books"]) - - month_title = metrics_data["month"][ - "title|books|q7gtd|||BOOK:Q7GTD|2024-01|Open|Regular|2023" - ] - self.assertEqual(month_title["document"]["id"], "BOOK:Q7GTD") - self.assertEqual(month_title["document"]["type"], "book") - self.assertEqual(month_title["document"]["title"], "Book Title") - self.assertNotIn("parent_id", month_title["document"]) - self.assertEqual(month_title["counter"]["metric_scope"], "title") - self.assertEqual(month_title["counter"]["data_type"], "Book") - self.assertEqual(month_title["total_requests"], 1) - self.assertEqual(month_title["total_investigations"], 1) - self.assertEqual(month_title["unique_requests"], 1) - self.assertEqual(month_title["unique_investigations"], 1) - - year_item = metrics_data["year"][ - "books|q7gtd|||BOOK:Q7GTD/CHAPTER:03|en|BR|2024|Open|Regular|2023" - ] - self.assertEqual( - year_item["access"], - {"year": "2024", "country_code": "BR", "content_language": "en"}, - ) - self.assertNotIn("daily_metrics", year_item) - self.assertEqual(year_item["document"]["title"], "Chapter Title") - self.assertEqual(year_item["counter"]["metric_scope"], "item") - self.assertEqual(year_item["total_requests"], 1) - - year_title = metrics_data["year"][ - "title|books|q7gtd|||BOOK:Q7GTD|en|BR|2024|Open|Regular|2023" - ] - self.assertEqual(year_title["counter"]["metric_scope"], "title") - self.assertEqual(year_title["document"]["title"], "Book Title") - self.assertNotIn("daily_metrics", year_title) - self.assertEqual(year_title["total_requests"], 1) - self.assertEqual(year_title["total_investigations"], 1) - self.assertEqual(year_title["unique_requests"], 1) - self.assertEqual(year_title["unique_investigations"], 1) - - def test_maps_counter_data_types_for_preprint_and_dataset(self): - data = { - "preprints|scielo-preprints|||10.1590/SCIELOPREPRINTS.1234|sess|BR|un|html|full_text": { - "collection": "preprints", - "source_key": "scielo-preprints", - "document_type": "preprint", - "pid_generic": "10.1590/SCIELOPREPRINTS.1234", - "user_session_id": "browser|1.0|127.0.0.1|2024-01-15|10", - "click_timestamps": {"00:05": 1}, - "access_country_code": "BR", - "content_language": "un", - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_year": "2024", - "source": { - "source_type": "preprint_server", - "source_id": "scielo-preprints", - "main_title": "SciELO Preprints", - }, - "publication_year": "2024", - }, - "data|scielo-data|||10.48331/SCIELODATA.ABC123|sess|BR|un|html|abstract": { - "collection": "data", - "source_key": "scielo-data", - "document_type": "dataset", - "pid_generic": "10.48331/SCIELODATA.ABC123", - "user_session_id": "browser|1.0|127.0.0.1|2024-01-15|10", - "click_timestamps": {"00:05": 1}, - "access_country_code": "BR", - "content_language": "un", - "content_type": CONTENT_TYPE_ABSTRACT, - "access_date": "2024-01-15", - "access_year": "2024", - "source": { - "source_type": "data_repository", - "source_id": "scielo-data", - "main_title": "SciELO Data", - }, - "publication_year": "2024", - }, - } - - metrics_data = _convert(data) - preprint_doc = metrics_data["month"][ - "preprints|scielo-preprints|||10.1590/SCIELOPREPRINTS.1234|2024-01|Open|Regular|2024" - ] - dataset_doc = metrics_data["month"][ - "data|scielo-data|||10.48331/SCIELODATA.ABC123|2024-01|Open|Regular|2024" - ] - - self.assertEqual(preprint_doc["counter"]["data_type"], "Article") - self.assertEqual(preprint_doc["document"]["type"], "preprint") - self.assertEqual(preprint_doc["document"]["id"], "10.1590/SCIELOPREPRINTS.1234") - self.assertEqual(preprint_doc["counter"]["article_version"], "Preprint") - self.assertEqual(dataset_doc["counter"]["data_type"], "Dataset") - self.assertNotIn("article_version", dataset_doc["counter"]) - - def test_dedupes_book_unique_item_across_formats(self): - data = { - "books|c2248|||BOOK:C2248/CHAPTER:03|sess|BR|pt|html|full_text": { - "collection": "books", - "source_key": "c2248", - "document_type": "chapter", - "pid_v2": None, - "pid_v3": None, - "pid_generic": "BOOK:C2248/CHAPTER:03", - "title_pid_generic": "BOOK:C2248", - "user_session_id": "browser|1.0|127.0.0.1|2024-01-15|10", - "click_timestamps": {"00:05": 1}, - "access_country_code": "BR", - "content_language": "pt", - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_month": "202401", - "access_year": "2024", - "source": { - "source_type": "book", - "source_id": "c2248", - "main_title": "C2248 Book", - "identifiers": {"book_id": "c2248", "isbn": "9788599662830"}, - "publisher_name": ["SciELO Books"], - }, - "publication_year": "2018", - }, - "books|c2248|||BOOK:C2248/CHAPTER:03|sess|BR|pt|pdf|full_text": { - "collection": "books", - "source_key": "c2248", - "document_type": "chapter", - "pid_v2": None, - "pid_v3": None, - "pid_generic": "BOOK:C2248/CHAPTER:03", - "title_pid_generic": "BOOK:C2248", - "user_session_id": "browser|1.0|127.0.0.1|2024-01-15|10", - "click_timestamps": {"00:45": 1}, - "access_country_code": "BR", - "content_language": "pt", - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_month": "202401", - "access_year": "2024", - "source": { - "source_type": "book", - "source_id": "c2248", - "main_title": "C2248 Book", - "identifiers": {"book_id": "c2248", "isbn": "9788599662830"}, - "publisher_name": ["SciELO Books"], - }, - "publication_year": "2018", - }, - } - - metrics_data = _convert(data) - month_item = metrics_data["month"][ - "books|c2248|||BOOK:C2248/CHAPTER:03|2024-01|Open|Regular|2018" - ] - month_title = metrics_data["month"][ - "title|books|c2248|||BOOK:C2248|2024-01|Open|Regular|2018" - ] - - self.assertEqual(month_item["total_requests"], 2) - self.assertEqual(month_item["total_investigations"], 2) - self.assertEqual(month_item["unique_requests"], 1) - self.assertEqual(month_item["unique_investigations"], 1) - self.assertEqual(month_title["unique_requests"], 1) - self.assertEqual(month_title["unique_investigations"], 1) - - def test_skips_book_landing_page_from_item_scope(self): - data = { - "books|c2248|||BOOK:C2248|sess|BR|pt|html|abstract": { - "collection": "books", - "source_key": "c2248", - "document_type": "book", - "pid_v2": None, - "pid_v3": None, - "pid_generic": "BOOK:C2248", - "document": {"title": "C2248 Book"}, - "title_pid_generic": "BOOK:C2248", - "user_session_id": "browser|1.0|127.0.0.1|2024-01-15|10", - "click_timestamps": {"00:05": 1}, - "access_country_code": "BR", - "content_language": "pt", - "content_type": CONTENT_TYPE_ABSTRACT, - "access_date": "2024-01-15", - "access_month": "202401", - "access_year": "2024", - "source": { - "source_type": "book", - "source_id": "c2248", - "main_title": "C2248 Book", - "identifiers": {"book_id": "c2248", "isbn": "9788599662830"}, - "publisher_name": ["SciELO Books"], - }, - "publication_year": "2018", - }, - } - - metrics_data = _convert(data) - self.assertEqual( - set(metrics_data["month"].keys()), - {"title|books|c2248|||BOOK:C2248|2024-01|Open|Regular|2018"}, - ) - self.assertEqual( - set(metrics_data["year"].keys()), - {"title|books|c2248|||BOOK:C2248|pt|BR|2024|Open|Regular|2018"}, - ) - - def test_whole_book_without_segments_counts_as_book_segment(self): - data = { - "books|c2248|||BOOK:C2248|sess|BR|pt|pdf|full_text": { - "collection": "books", - "source_key": "c2248", - "document_type": "book", - "pid_v2": None, - "pid_v3": None, - "pid_generic": "BOOK:C2248", - "document": {"title": "C2248 Book"}, - "title_pid_generic": "BOOK:C2248", - "user_session_id": "browser|1.0|127.0.0.1|2024-01-15|10", - "click_timestamps": {"00:05": 1}, - "access_country_code": "BR", - "content_language": "pt", - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_month": "202401", - "access_year": "2024", - "source": { - "source_type": "book", - "source_id": "c2248", - "main_title": "C2248 Book", - "identifiers": {"book_id": "c2248"}, - "publisher_name": ["SciELO Books"], - }, - "publication_year": "2018", - }, - } - - metrics_data = _convert(data) - month_item = metrics_data["month"][ - "books|c2248|||BOOK:C2248|2024-01|Open|Regular|2018" - ] - month_title = metrics_data["month"][ - "title|books|c2248|||BOOK:C2248|2024-01|Open|Regular|2018" - ] - - self.assertEqual(month_item["counter"]["data_type"], "Book_Segment") - self.assertEqual(month_item["counter"]["metric_scope"], "item") - self.assertEqual(month_item["document"]["id"], "BOOK:C2248") - self.assertNotIn("parent_id", month_item["document"]) - self.assertEqual(month_title["counter"]["data_type"], "Book") - self.assertEqual(month_title["counter"]["metric_scope"], "title") - - def test_aggregates_multiple_chapters_at_title_level(self): - data = { - "books|q7gtd|||BOOK:Q7GTD/CHAPTER:01|session1|BR|en|html|full_text": { - "collection": "books", - "source_key": "q7gtd", - "document_type": "chapter", - "pid_generic": "BOOK:Q7GTD/CHAPTER:01", - "title_pid_generic": "BOOK:Q7GTD", - "user_session_id": "session1", - "click_timestamps": {"00:05": 1}, - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_year": "2024", - "source": { - "source_type": "book", - "source_id": "q7gtd", - "scielo_issn": DEFAULT_SCIELO_ISSN, - "main_title": "Book Title", - "identifiers": {"book_id": "q7gtd"}, - "publisher_name": ["SciELO Books"], - }, - "publication_year": "2023", - }, - "books|q7gtd|||BOOK:Q7GTD/CHAPTER:02|session1|BR|en|html|full_text": { - "collection": "books", - "source_key": "q7gtd", - "document_type": "chapter", - "pid_generic": "BOOK:Q7GTD/CHAPTER:02", - "title_pid_generic": "BOOK:Q7GTD", - "user_session_id": "session1", - "click_timestamps": {"00:10": 1}, - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_year": "2024", - "source": { - "source_type": "book", - "source_id": "q7gtd", - "scielo_issn": DEFAULT_SCIELO_ISSN, - "main_title": "Book Title", - "identifiers": {"book_id": "q7gtd"}, - "publisher_name": ["SciELO Books"], - }, - "publication_year": "2023", - }, - } - - metrics_data = _convert(data) - self.assertEqual(len(metrics_data["month"]), 3) - self.assertEqual(len(metrics_data["year"]), 3) - - month_title = metrics_data["month"][ - "title|books|q7gtd|||BOOK:Q7GTD|2024-01|Open|Regular|2023" - ] - self.assertEqual(month_title["total_requests"], 2) - self.assertEqual(month_title["total_investigations"], 2) - self.assertEqual(month_title["unique_requests"], 1) - self.assertEqual(month_title["unique_investigations"], 1) - - def test_double_click_collapses_same_url_within_30_seconds(self): - from datetime import datetime - - from metrics.counter.access import accumulation - from metrics.counter.access.daily_accumulator import DailyAccessAccumulator - - results = DailyAccessAccumulator() - counter_access = { - "collection": "books", - "source_type": "book", - "source_id": "c2248", - "scielo_issn": DEFAULT_SCIELO_ISSN, - "pid_v2": None, - "pid_v3": None, - "pid_generic": "BOOK:C2248/CHAPTER:03", - "title_pid_generic": "BOOK:C2248", - "media_language": "pt", - "media_format": MEDIA_FORMAT_HTML, - "content_type": CONTENT_TYPE_FULL_TEXT, - "publication_year": "2018", - "source_main_title": "C2248 Book", - } - base_line = { - "client_name": "browser", - "client_version": "1.0", - "ip_address": "127.0.0.1", - "country_code": "BR", - "url": "/id/c2248/03?from=search", - } +def test_counter_facts_are_monthly_and_reference_metadata_keys(): + documents = dict(converter.iter_partitioned_values([_article()], "counter")) + + assert len(documents) == 1 + document_id, document = next(iter(documents.items())) + assert document_id.startswith("k1_") + assert document["month"] == "2026-08" + assert document["data_type"] == "Article" + assert document["parent_data_type"] == "Journal" + assert document["total_requests"] == 1 + assert document["unique_requests"] == 1 + assert document["source_key"].startswith("k1_") + assert document["document_key"].startswith("k1_") + assert "source" not in document + assert "document" not in document + assert "country_code" not in document + assert "content_language" not in document + assert "year" not in document + + +def test_analytics_builds_annual_country_and_language_matrix(): + values = [ + _article(country="BR", language="pt", session="same-session"), + _article(country="US", language="pt", session="same-session"), + ] + documents = list(converter.iter_partitioned_values(values, "analytics")) + + assert len(documents) == 2 + assert {item["year"] for _key, item in documents} == {"2026"} + assert {item["country_code"] for _key, item in documents} == {"BR", "US"} + assert {item["content_language"] for _key, item in documents} == {"pt"} + assert all("collection" not in item for _key, item in documents) + assert all(item["source_key"].startswith("k1_") for _key, item in documents) + assert all(item["document_key"].startswith("k1_") for _key, item in documents) + assert all("month" not in item for _key, item in documents) + assert all("counter_key" not in item for _key, item in documents) + assert all("projection" not in item for _key, item in documents) + assert all("daily_metrics" not in item for _key, item in documents) + + +def test_analytics_combines_the_same_dimensions_across_months(): + january = _article() + february = {**_article(), "access_date": "2026-02-20"} + + documents = dict( + converter.iter_partitioned_values([january, february], "analytics") + ) + + assert len(documents) == 1 + document = next(iter(documents.values())) + assert document["year"] == "2026" + assert document["total_requests"] == 2 + + +def test_book_and_chapter_keep_item_and_title_scopes(): + chapter = { + **_article(), + "collection": "books", + "source_key": "book-1", + "document_type": "chapter", + "pid_v3": None, + "pid_generic": "BOOK:BOOK-1/CHAPTER:01", + "title_pid_generic": "BOOK:BOOK-1", + "source": {"source_type": "book", "source_id": "book-1"}, + } - accumulation.accumulate( - results, - counter_access, - {**base_line, "local_datetime": datetime(2024, 1, 15, 10, 0, 5)}, - ) - accumulation.accumulate( - results, - counter_access, - {**base_line, "local_datetime": datetime(2024, 1, 15, 10, 0, 20)}, - ) + documents = dict(converter.iter_partitioned_values([chapter], "counter")) - values = list(results.iter_materialized_values()) - metrics_data = { - "month": dict(index_docs.iter_partitioned_values(values, "month")), - "year": dict(index_docs.iter_partitioned_values(values, "year")), - } - month_item = metrics_data["month"][ - "books|c2248|||BOOK:C2248/CHAPTER:03|2024-01|Open|Regular|2018" - ] - self.assertEqual(month_item["total_requests"], 1) - self.assertEqual(month_item["unique_requests"], 1) + assert len(documents) == 2 + assert {item["metric_scope"] for item in documents.values()} == {"item", "title"} + assert {item["data_type"] for item in documents.values()} == { + "Book", + "Book_Segment", + } + assert all(item["total_requests"] == 1 for item in documents.values()) - def test_article_pipeline_sets_journal_parent(self): - data = { - "scl|1234-5678||abc123||sess|BR|en|pdf|full_text": { - "collection": "scl", - "source_key": "1234-5678", - "document_type": "article", - "pid_v2": None, - "pid_v3": "abc123", - "pid_generic": None, - "document": {"title": "Article Title"}, - "user_session_id": "sess", - "click_timestamps": {"00:05": 1}, - "access_country_code": "BR", - "content_language": "en", - "content_type": CONTENT_TYPE_FULL_TEXT, - "access_date": "2024-01-15", - "access_year": "2024", - "source": { - "source_type": "journal", - "source_id": "1234-5678", - "scielo_issn": "1234-5678", - "main_title": "Test Journal", - }, - "publication_year": "2024", - } - } - metrics_data = _convert(data) - month_doc = list(metrics_data["month"].values())[0] +def test_compact_keys_and_output_are_deterministic(): + first = list(converter.iter_partitioned_values([_article()], "counter")) + second = list(converter.iter_partitioned_values([_article()], "counter")) + assert first == second - self.assertEqual(month_doc["counter"]["data_type"], "Article") - self.assertEqual(month_doc["counter"]["parent_data_type"], "Journal") - self.assertEqual(month_doc["counter"]["metric_scope"], "item") - self.assertEqual(month_doc["document"]["type"], "article") - self.assertEqual(month_doc["total_requests"], 1) - self.assertEqual(month_doc["total_investigations"], 1) - def test_empty_iterable_returns_empty(self): - self.assertEqual(dict(index_docs.iter_partitioned_values((), "month")), {}) +def test_empty_iterable_returns_empty(): + assert list(converter.iter_partitioned_values((), "counter")) == [] diff --git a/metrics/tests/helpers.py b/metrics/tests/helpers.py index c4d0de6..23d7186 100644 --- a/metrics/tests/helpers.py +++ b/metrics/tests/helpers.py @@ -4,6 +4,6 @@ def convert_accumulator(accumulator): values = list(accumulator.iter_materialized_values()) return { - "month": dict(converter.iter_partitioned_values(values, "month")), - "year": dict(converter.iter_partitioned_values(values, "year")), + "counter": dict(converter.iter_partitioned_values(values, "counter")), + "analytics": dict(converter.iter_partitioned_values(values, "analytics")), } diff --git a/metrics/tests/integration/test_books_log_to_metrics.py b/metrics/tests/integration/test_books_log_to_metrics.py index 6edf141..b156eac 100644 --- a/metrics/tests/integration/test_books_log_to_metrics.py +++ b/metrics/tests/integration/test_books_log_to_metrics.py @@ -2,13 +2,13 @@ from datetime import datetime from pathlib import Path -from scielo_usage_counter import log_handler from scielo_usage_counter.translator.books import URLTranslatorBooksSite from scielo_usage_counter.url_translator import URLTranslationManager from metrics.counter.access import accumulation, extraction, validation from metrics.counter.access.daily_accumulator import DailyAccessAccumulator from metrics.tests.helpers import convert_accumulator +from scielo_usage_counter import log_handler FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" @@ -119,19 +119,19 @@ def test_full_pipeline_with_synthetic_metadata(self): ) metrics = convert_accumulator(results) - self.assertGreater(len(metrics["month"]), 0) - self.assertGreater(len(metrics["year"]), 0) + self.assertGreater(len(metrics["counter"]), 0) + self.assertGreater(len(metrics["analytics"]), 0) has_item = False has_title = False - for doc in metrics["month"].values(): - scope = doc["counter"]["metric_scope"] + for doc in metrics["counter"].values(): + scope = doc["metric_scope"] if scope == "item": has_item = True - self.assertEqual(doc["counter"]["data_type"], "Book_Segment") + self.assertEqual(doc["data_type"], "Book_Segment") elif scope == "title": has_title = True - self.assertEqual(doc["counter"]["data_type"], "Book") + self.assertEqual(doc["data_type"], "Book") self.assertTrue(has_item) self.assertTrue(has_title) @@ -168,19 +168,18 @@ def test_all_metric_fields_present_in_converted_document(self): ) metrics = convert_accumulator(results) - for doc in metrics["month"].values(): + for doc in metrics["counter"].values(): self.assertIn("total_requests", doc) self.assertIn("total_investigations", doc) self.assertIn("unique_requests", doc) self.assertIn("unique_investigations", doc) self.assertIn("collection", doc) - self.assertIn("source", doc) - self.assertIn("document", doc) - self.assertIn("counter", doc) - self.assertIn("access", doc) + self.assertIn("source_key", doc) + self.assertIn("document_key", doc) + self.assertIn("data_type", doc) + self.assertIn("month", doc) self.assertIn("daily_metrics", doc) - for doc in metrics["year"].values(): - access = doc.get("access", {}) - self.assertIn("year", access) + for doc in metrics["analytics"].values(): + self.assertEqual(doc["year"], "2012") self.assertNotIn("daily_metrics", doc) diff --git a/metrics/tests/integration/test_bunnynet_log_to_metrics.py b/metrics/tests/integration/test_bunnynet_log_to_metrics.py index 2865bc7..576b7ef 100644 --- a/metrics/tests/integration/test_bunnynet_log_to_metrics.py +++ b/metrics/tests/integration/test_bunnynet_log_to_metrics.py @@ -1,12 +1,12 @@ import unittest from pathlib import Path -from scielo_usage_counter import log_handler from scielo_usage_counter.values import CONTENT_TYPE_FULL_TEXT, MEDIA_FORMAT_HTML from metrics.counter.access import accumulation, extraction from metrics.counter.access.daily_accumulator import DailyAccessAccumulator from metrics.tests.helpers import convert_accumulator +from scielo_usage_counter import log_handler FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" @@ -80,7 +80,11 @@ def test_uses_client_country_and_millisecond_timestamp_in_year_metrics(self): self.assertEqual(line["country_code"], "US") self.assertEqual(line["local_datetime"], "2026-08-04 23:59:59") - year_document = next(iter(metrics["year"].values())) - self.assertEqual(year_document["access"]["country_code"], "US") - month_document = next(iter(metrics["month"].values())) - self.assertNotIn("country_code", month_document["access"]) + country_document = next( + document + for document in metrics["analytics"].values() + if document["country_code"] == "US" + ) + self.assertEqual(country_document["country_code"], "US") + counter_document = next(iter(metrics["counter"].values())) + self.assertNotIn("country_code", counter_document) diff --git a/metrics/tests/integration/test_classic_log_to_metrics.py b/metrics/tests/integration/test_classic_log_to_metrics.py index e75b79e..7333963 100644 --- a/metrics/tests/integration/test_classic_log_to_metrics.py +++ b/metrics/tests/integration/test_classic_log_to_metrics.py @@ -1,13 +1,13 @@ import unittest from pathlib import Path -from scielo_usage_counter import log_handler from scielo_usage_counter.translator.classic import URLTranslatorClassicSite from scielo_usage_counter.url_translator import URLTranslationManager from metrics.counter.access import accumulation, extraction, validation from metrics.counter.access.daily_accumulator import DailyAccessAccumulator from metrics.tests.helpers import convert_accumulator +from scielo_usage_counter import log_handler FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" @@ -78,10 +78,9 @@ def test_produces_article_type_metrics(self): metrics = convert_accumulator(results) - for doc in metrics["month"].values(): - self.assertEqual(doc["counter"]["data_type"], "Article") - self.assertEqual(doc["counter"]["metric_scope"], "item") - self.assertEqual(doc["document"]["type"], "article") + for doc in metrics["counter"].values(): + self.assertEqual(doc["data_type"], "Article") + self.assertEqual(doc["metric_scope"], "item") def test_sets_journal_parent_data_type(self): results, _, _, _ = self._full_pipeline() @@ -90,10 +89,8 @@ def test_sets_journal_parent_data_type(self): return metrics = convert_accumulator(results) - for doc in metrics["month"].values(): - source_type = doc.get("source", {}).get("type") - if source_type == "journal": - self.assertEqual(doc["counter"]["parent_data_type"], "Journal") + for doc in metrics["counter"].values(): + self.assertEqual(doc["parent_data_type"], "Journal") def test_handles_truncated_user_agent(self): lines, _ = self._parse_log() diff --git a/metrics/tests/integration/test_pipelines.py b/metrics/tests/integration/test_pipelines.py index 292bab2..48c0537 100644 --- a/metrics/tests/integration/test_pipelines.py +++ b/metrics/tests/integration/test_pipelines.py @@ -42,14 +42,12 @@ def test_full_pipeline_produces_preprint_article_version(self): accumulation.accumulate(results, counter_access, line) metrics = convert_accumulator(results) - month_docs = list(metrics["month"].values()) - self.assertEqual(len(month_docs), 1) - doc = month_docs[0] - self.assertEqual(doc["counter"]["data_type"], "Article") - self.assertEqual(doc["counter"]["article_version"], "Preprint") - self.assertEqual(doc["counter"]["metric_scope"], "item") - self.assertEqual(doc["document"]["type"], "preprint") - self.assertEqual(doc["document"]["id"], "10.1590/SCIELOPREPRINTS.1234") + counter_docs = list(metrics["counter"].values()) + self.assertEqual(len(counter_docs), 1) + doc = counter_docs[0] + self.assertEqual(doc["data_type"], "Article") + self.assertEqual(doc["article_version"], "Preprint") + self.assertEqual(doc["metric_scope"], "item") self.assertEqual(doc["total_requests"], 1) self.assertEqual(doc["unique_requests"], 1) @@ -83,12 +81,11 @@ def test_full_pipeline_produces_dataset_metrics(self): accumulation.accumulate(results, counter_access, line) metrics = convert_accumulator(results) - month_docs = list(metrics["month"].values()) - self.assertEqual(len(month_docs), 1) - doc = month_docs[0] - self.assertEqual(doc["counter"]["data_type"], "Dataset") - self.assertNotIn("article_version", doc["counter"]) - self.assertEqual(doc["document"]["type"], "dataset") + counter_docs = list(metrics["counter"].values()) + self.assertEqual(len(counter_docs), 1) + doc = counter_docs[0] + self.assertEqual(doc["data_type"], "Dataset") + self.assertNotIn("article_version", doc) self.assertEqual(doc["total_investigations"], 1) self.assertEqual(doc["total_requests"], 0) @@ -121,10 +118,9 @@ def test_opac_article_produces_journal_article_metrics(self): accumulation.accumulate(results, counter_access, line) metrics = convert_accumulator(results) - doc = list(metrics["month"].values())[0] - self.assertEqual(doc["counter"]["data_type"], "Article") - self.assertEqual(doc["counter"]["parent_data_type"], "Journal") - self.assertEqual(doc["document"]["type"], "article") - self.assertEqual(doc["source"]["type"], "journal") - self.assertEqual(doc["source"]["id"], "1234-5678") + doc = list(metrics["counter"].values())[0] + self.assertEqual(doc["data_type"], "Article") + self.assertEqual(doc["parent_data_type"], "Journal") + self.assertTrue(doc["source_key"].startswith("k1_")) + self.assertTrue(doc["document_key"].startswith("k1_")) self.assertEqual(doc["total_requests"], 1) From 743211e23fe21fc588f2bceec02d98bee857e249 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:33 -0300 Subject: [PATCH 03/23] Define mappings normalizados de fatos e metadados --- metrics/opensearch/mappings.py | 191 +++++++++++++++++++-------------- 1 file changed, 111 insertions(+), 80 deletions(-) diff --git a/metrics/opensearch/mappings.py b/metrics/opensearch/mappings.py index de4dbae..dbfddf2 100644 --- a/metrics/opensearch/mappings.py +++ b/metrics/opensearch/mappings.py @@ -1,103 +1,134 @@ -DISPLAY_TEXT_MAPPING = { - "type": "text", - "index": False, +METRIC_PROPERTIES = { + "total_requests": {"type": "long"}, + "total_investigations": {"type": "long"}, + "unique_requests": {"type": "long"}, + "unique_investigations": {"type": "long"}, } -IDENTIFIERS_MAPPING = {"type": "object", "dynamic": True} - -DOCUMENT_MAPPINGS = { - "properties": { - "id": {"type": "keyword"}, - "type": {"type": "keyword"}, - "title": DISPLAY_TEXT_MAPPING, - "parent_id": {"type": "keyword"}, - "publication_year": {"type": "integer"}, - "identifiers": IDENTIFIERS_MAPPING, - } +COUNTER_DIMENSIONS = { + "metric_scope": {"type": "keyword"}, + "data_type": {"type": "keyword"}, + "parent_data_type": {"type": "keyword"}, + "article_version": {"type": "keyword"}, + "access_type": {"type": "keyword"}, + "access_method": {"type": "keyword"}, } -SOURCE_MAPPINGS = { - "properties": { - "id": {"type": "keyword"}, - "type": {"type": "keyword"}, - "title": DISPLAY_TEXT_MAPPING, - "scielo_issn": {"type": "keyword"}, - "acronym": {"type": "keyword"}, - "publisher_name": DISPLAY_TEXT_MAPPING, - "access_type": {"type": "keyword"}, - "city": {"type": "keyword"}, - "country": {"type": "keyword"}, - "subject_area_capes": {"type": "keyword"}, - "subject_area_wos": {"type": "keyword"}, - "identifiers": IDENTIFIERS_MAPPING, - } -} -COUNTER_MAPPINGS = { - "properties": { - "metric_scope": {"type": "keyword"}, - "data_type": {"type": "keyword"}, - "parent_data_type": {"type": "keyword"}, - "article_version": {"type": "keyword"}, - "access_type": {"type": "keyword"}, - "access_method": {"type": "keyword"}, +def _metric_properties(): + return { + "collection": {"type": "keyword"}, + "source_key": {"type": "keyword"}, + "document_key": {"type": "keyword"}, + "month": {"type": "date", "format": "yyyy-MM"}, + "applied_days": { + "type": "keyword", + "index": False, + "doc_values": False, + }, + **COUNTER_DIMENSIONS, + **METRIC_PROPERTIES, } -} -MONTH_ACCESS_MAPPINGS = { + +MONTH_INDEX_MAPPINGS = { + "dynamic": False, "properties": { - "month": {"type": "date", "format": "yyyy-MM"}, - } + **_metric_properties(), + "daily_metrics": {"type": "object", "dynamic": False}, + }, } -YEAR_ACCESS_MAPPINGS = { +ANALYTICS_INDEX_MAPPINGS = { + "dynamic": False, + "_source": { + "excludes": [ + "year", + "source_key", + "document_key", + *COUNTER_DIMENSIONS, + "country_code", + "content_language", + ] + }, "properties": { - "year": {"type": "date", "format": "yyyy"}, + "year": {"type": "keyword"}, + "source_key": {"type": "keyword"}, + "document_key": {"type": "keyword"}, + **COUNTER_DIMENSIONS, + "applied_day_masks": { + "type": "long", + "index": False, + "doc_values": False, + }, "country_code": {"type": "keyword"}, "content_language": {"type": "keyword"}, - } + **METRIC_PROPERTIES, + }, } -METRIC_PROPERTIES = { - "total_requests": {"type": "integer"}, - "total_investigations": {"type": "integer"}, - "unique_requests": {"type": "integer"}, - "unique_investigations": {"type": "integer"}, +SOURCE_INDEX_MAPPINGS = { + "dynamic": False, + "properties": { + "key": {"type": "keyword"}, + "collection": {"type": "keyword"}, + "source_id": {"type": "keyword"}, + "source_type": {"type": "keyword"}, + "title": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, + "scielo_issn": {"type": "keyword"}, + "acronym": {"type": "keyword"}, + "publisher_name": {"type": "keyword"}, + "access_type": {"type": "keyword"}, + "country": {"type": "keyword"}, + "subject_areas": {"type": "keyword"}, + "wos_subject_areas": {"type": "keyword"}, + "identifiers": {"type": "flat_object"}, + "active": {"type": "boolean"}, + "updated": {"type": "date"}, + }, } - -def _build_index_mappings(granularity): - properties = { +DOCUMENT_INDEX_MAPPINGS = { + "dynamic": False, + "properties": { + "key": {"type": "keyword"}, "collection": {"type": "keyword"}, - "source": SOURCE_MAPPINGS, - "document": DOCUMENT_MAPPINGS, - "access": MONTH_ACCESS_MAPPINGS - if granularity == "month" - else YEAR_ACCESS_MAPPINGS, - "counter": COUNTER_MAPPINGS, - "applied_jobs": {"type": "keyword", "index": False}, - **METRIC_PROPERTIES, - } - if granularity == "month": - properties["daily_metrics"] = {"type": "object", "dynamic": True} - return {"properties": properties} - + "document_id": {"type": "keyword"}, + "document_type": {"type": "keyword"}, + "source_key": {"type": "keyword"}, + "parent_document_key": {"type": "keyword"}, + "title": {"type": "text", "fields": {"keyword": {"type": "keyword"}}}, + "publication_year": {"type": "integer"}, + "default_lang": {"type": "keyword"}, + "text_langs": {"type": "keyword"}, + "identifiers": {"type": "flat_object"}, + "active": {"type": "boolean"}, + "updated": {"type": "date"}, + }, +} -YEAR_INDEX_MAPPINGS = _build_index_mappings("year") -MONTH_INDEX_MAPPINGS = _build_index_mappings("month") -BOOKS_YEAR_INDEX_MAPPINGS = _build_index_mappings("year") -BOOKS_MONTH_INDEX_MAPPINGS = _build_index_mappings("month") +def get_index_mappings(dataset): + mappings = { + "counter": MONTH_INDEX_MAPPINGS, + "analytics": ANALYTICS_INDEX_MAPPINGS, + "sources": SOURCE_INDEX_MAPPINGS, + "documents": DOCUMENT_INDEX_MAPPINGS, + } + try: + return mappings[dataset] + except KeyError as exc: + raise ValueError(f"Unknown OpenSearch dataset: {dataset}.") from exc -def get_index_mappings(collection, granularity): - if granularity not in {"month", "year"}: - raise ValueError("Granularity must be 'month' or 'year'.") - if collection == "books": - return ( - BOOKS_MONTH_INDEX_MAPPINGS - if granularity == "month" - else BOOKS_YEAR_INDEX_MAPPINGS - ) +def get_index_settings(primary_shards=1): + if primary_shards <= 0: + raise ValueError("OpenSearch primary shards must be greater than zero.") - return MONTH_INDEX_MAPPINGS if granularity == "month" else YEAR_INDEX_MAPPINGS + return { + "index": { + "number_of_shards": primary_shards, + "number_of_replicas": 0, + "codec": "best_compression", + } + } From f58b3193d07c654fd838aeba8e22ed6182179fc6 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:33 -0300 Subject: [PATCH 04/23] =?UTF-8?q?Nomeia=20=C3=ADndices=20anuais=20por=20co?= =?UTF-8?q?le=C3=A7=C3=A3o=20e=20conjunto=20de=20dados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/opensearch/names.py | 30 ++++++------- metrics/tests/opensearch/test_names.py | 62 ++++++++------------------ 2 files changed, 32 insertions(+), 60 deletions(-) diff --git a/metrics/opensearch/names.py b/metrics/opensearch/names.py index 7ad0e87..f90e074 100644 --- a/metrics/opensearch/names.py +++ b/metrics/opensearch/names.py @@ -1,6 +1,3 @@ -from django.conf import settings - - def _validate_index_inputs(index_prefix, collection, date): if not date or not isinstance(date, str): raise ValueError("Date must be a non-empty string in 'YYYY-MM-DD' format.") @@ -15,22 +12,21 @@ def extract_access_year(date): return date.split("-")[0] -def _uses_year_partition(collection): - configured_collections = { - value.strip().lower() for value in settings.YEAR_PARTITIONED_COLLECTIONS - } - return collection.lower() in configured_collections - - def generate_month_index_name(index_prefix, collection, date): _validate_index_inputs(index_prefix, collection, date) - if _uses_year_partition(collection): - return f"{index_prefix}_monthly_{collection}_{extract_access_year(date)}" - return f"{index_prefix}_monthly_{collection}" + return f"{index_prefix}_monthly_{collection}_{extract_access_year(date)}" -def generate_year_index_name(index_prefix, collection, date): +def generate_analytics_index_name(index_prefix, collection, date): _validate_index_inputs(index_prefix, collection, date) - if _uses_year_partition(collection): - return f"{index_prefix}_yearly_{collection}_{extract_access_year(date)}" - return f"{index_prefix}_yearly_{collection}" + return f"{index_prefix}_yearly_analytics_{collection}_{extract_access_year(date)}" + + +def generate_physical_index_name(alias_name): + return f"{alias_name}_000001" + + +def generate_metadata_alias(entity): + if entity not in {"documents", "sources"}: + raise ValueError("Metadata entity must be 'documents' or 'sources'.") + return f"usage_{entity}" diff --git a/metrics/tests/opensearch/test_names.py b/metrics/tests/opensearch/test_names.py index 58ad18a..8898204 100644 --- a/metrics/tests/opensearch/test_names.py +++ b/metrics/tests/opensearch/test_names.py @@ -1,47 +1,23 @@ -import unittest +from metrics.opensearch.names import ( + generate_analytics_index_name, + generate_metadata_alias, + generate_month_index_name, + generate_physical_index_name, +) -from django.conf import settings -from django.test import override_settings -from metrics.opensearch.names import generate_month_index_name, generate_year_index_name +def test_metric_aliases_are_partitioned_by_collection_and_year(): + assert generate_month_index_name("usage", "scl", "2026-08-20") == ( + "usage_monthly_scl_2026" + ) + assert generate_analytics_index_name("usage", "scl", "2026-08-20") == ( + "usage_yearly_analytics_scl_2026" + ) -class TestIndexNames(unittest.TestCase): - def test_year_partition_policy_is_explicit(self): - self.assertEqual( - set(settings.YEAR_PARTITIONED_COLLECTIONS), - {"chl", "col", "mex", "scl"}, - ) - - def test_generate_index_names_for_year_and_month(self): - self.assertEqual( - generate_year_index_name("usage", "scl", "2024-01-15"), - "usage_yearly_scl_2024", - ) - self.assertEqual( - generate_month_index_name("usage", "scl", "2024-01-15"), - "usage_monthly_scl_2024", - ) - self.assertEqual( - generate_year_index_name("usage", "books", "2024-01-15"), - "usage_yearly_books", - ) - self.assertEqual( - generate_month_index_name("usage", "books", "2024-01-15"), - "usage_monthly_books", - ) - - @override_settings(YEAR_PARTITIONED_COLLECTIONS=[" books ", "SCL"]) - def test_generate_index_names_uses_configured_collections(self): - self.assertEqual( - generate_year_index_name("usage", "books", "2024-01-15"), - "usage_yearly_books_2024", - ) - self.assertEqual( - generate_month_index_name("usage", "scl", "2024-01-15"), - "usage_monthly_scl_2024", - ) - self.assertEqual( - generate_year_index_name("usage", "chl", "2024-01-15"), - "usage_yearly_chl", - ) +def test_physical_index_and_metadata_alias_names_are_explicit(): + assert generate_physical_index_name("usage_monthly_scl_2026") == ( + "usage_monthly_scl_2026_000001" + ) + assert generate_metadata_alias("documents") == "usage_documents" + assert generate_metadata_alias("sources") == "usage_sources" From 9a11ebb6eb9507e9465deda7bbadb6d7ae69d520 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:33 -0300 Subject: [PATCH 05/23] =?UTF-8?q?Atualiza=20m=C3=A9tricas=20di=C3=A1rias?= =?UTF-8?q?=20com=20scripts=20idempotentes?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/opensearch/painless.py | 85 ++++++++++++++++++++++++++++++---- 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/metrics/opensearch/painless.py b/metrics/opensearch/painless.py index de3de81..e950f1b 100644 --- a/metrics/opensearch/painless.py +++ b/metrics/opensearch/painless.py @@ -1,3 +1,7 @@ +from datetime import date + +ANNUAL_MASK_BITS = 63 +ANNUAL_MASK_BUCKETS = 6 METRIC_FIELDS = ( "total_requests", "total_investigations", @@ -5,17 +9,17 @@ "unique_investigations", ) -IDEMPOTENT_JOB_INCREMENT_SCRIPT = """ -if (ctx._source.applied_jobs == null) { - ctx._source.applied_jobs = []; +IDEMPOTENT_DAY_INCREMENT_SCRIPT = """ +if (ctx._source.applied_days == null) { + ctx._source.applied_days = []; } -if (ctx._source.applied_jobs.contains(params.job_id)) { +if (ctx._source.applied_days.contains(params.access_day)) { ctx.op = 'none'; return; } for (entry in params.document.entrySet()) { if (!params.metric_fields.contains(entry.getKey()) - && !'applied_jobs'.equals(entry.getKey()) + && !'applied_days'.equals(entry.getKey()) && !'daily_metrics'.equals(entry.getKey())) { if (!ctx._source.containsKey(entry.getKey()) || ctx._source[entry.getKey()] != entry.getValue()) { ctx._source[entry.getKey()] = entry.getValue(); @@ -45,11 +49,70 @@ def increment = dayMetrics.containsKey(metric) ? dayMetrics[metric] : 0; } } } -ctx._source.applied_jobs.add(params.job_id); +ctx._source.applied_days.add(params.access_day); +""" + +IDEMPOTENT_ANNUAL_DAY_INCREMENT_SCRIPT = """ +if (ctx._source.applied_day_masks == null) { + ctx._source.applied_day_masks = params.empty_day_masks; +} +def currentMask = ctx._source.applied_day_masks[params.mask_index]; +if ((currentMask & params.day_mask) != 0) { + ctx.op = 'none'; + return; +} +for (entry in params.document.entrySet()) { + if (!params.metric_fields.contains(entry.getKey()) + && !'applied_day_masks'.equals(entry.getKey())) { + if (!ctx._source.containsKey(entry.getKey()) || ctx._source[entry.getKey()] != entry.getValue()) { + ctx._source[entry.getKey()] = entry.getValue(); + } + } +} +for (field in params.metric_fields) { + def currentValue = ctx._source.containsKey(field) ? ctx._source[field] : 0; + def increment = params.document.containsKey(field) ? params.document[field] : 0; + ctx._source[field] = currentValue + increment; +} +ctx._source.applied_day_masks[params.mask_index] = currentMask | params.day_mask; """ -def build_idempotent_job_increment_action(index_name, doc_id, document, job_id): +def build_idempotent_day_increment_action( + index_name, + doc_id, + document, + access_day, +): + return { + "_op_type": "update", + "_index": index_name, + "_id": doc_id, + "retry_on_conflict": 5, + "scripted_upsert": True, + "script": { + "lang": "painless", + "source": IDEMPOTENT_DAY_INCREMENT_SCRIPT, + "params": { + "document": document, + "access_day": access_day, + "metric_fields": list(METRIC_FIELDS), + }, + }, + "upsert": { + "applied_days": [], + }, + } + + +def build_idempotent_annual_day_increment_action( + index_name, + doc_id, + document, + access_day, +): + access_date = date.fromisoformat(access_day) + day_offset = access_date.timetuple().tm_yday - 1 return { "_op_type": "update", "_index": index_name, @@ -58,15 +121,17 @@ def build_idempotent_job_increment_action(index_name, doc_id, document, job_id): "scripted_upsert": True, "script": { "lang": "painless", - "source": IDEMPOTENT_JOB_INCREMENT_SCRIPT, + "source": IDEMPOTENT_ANNUAL_DAY_INCREMENT_SCRIPT, "params": { "document": document, - "job_id": job_id, + "mask_index": day_offset // ANNUAL_MASK_BITS, + "day_mask": 1 << (day_offset % ANNUAL_MASK_BITS), + "empty_day_masks": [0] * ANNUAL_MASK_BUCKETS, "metric_fields": list(METRIC_FIELDS), }, }, "upsert": { - "applied_jobs": [], + "applied_day_masks": [0] * ANNUAL_MASK_BUCKETS, }, } From 9f89749981c273bc80671c0d687dc71bbd8be1a3 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:42 -0300 Subject: [PATCH 06/23] Gerencia aliases e lotes de documentos no OpenSearch --- metrics/opensearch/client.py | 104 ++++++++-- metrics/tests/opensearch/test_client.py | 242 +++++++++++------------- 2 files changed, 200 insertions(+), 146 deletions(-) diff --git a/metrics/opensearch/client.py b/metrics/opensearch/client.py index ffb34d9..5547c7b 100644 --- a/metrics/opensearch/client.py +++ b/metrics/opensearch/client.py @@ -3,10 +3,15 @@ from django.conf import settings from opensearchpy import NotFoundError, OpenSearch, helpers -from metrics.opensearch.mappings import get_index_mappings -from metrics.opensearch.names import generate_month_index_name, generate_year_index_name +from metrics.opensearch.mappings import get_index_mappings, get_index_settings +from metrics.opensearch.names import ( + generate_analytics_index_name, + generate_month_index_name, + generate_physical_index_name, +) from metrics.opensearch.painless import ( - build_idempotent_job_increment_action, + build_idempotent_annual_day_increment_action, + build_idempotent_day_increment_action, merge_metric_document, ) @@ -71,20 +76,32 @@ def ping(self): logging.error("Error pinging OpenSearch client: %s", exc) return False - def create_index(self, index_name, mappings, ping_client=False): + def create_index( + self, + index_name, + mappings, + ping_client=False, + primary_shards=1, + ): if ping_client and not self.ping(): return response = self.client.indices.create( index=index_name, body={ - "settings": {"index": {"number_of_replicas": 0}}, + "settings": get_index_settings(primary_shards), "mappings": mappings, }, ) logging.info("Index %s created: %s", index_name, response) - def create_index_if_not_exists(self, index_name, mappings, ping_client=False): + def create_index_if_not_exists( + self, + index_name, + mappings, + ping_client=False, + primary_shards=1, + ): if ping_client and not self.ping(): return @@ -92,28 +109,59 @@ def create_index_if_not_exists(self, index_name, mappings, ping_client=False): self.create_index( index_name=index_name, mappings=mappings, + primary_shards=primary_shards, ping_client=False, ) + def create_alias_if_not_exists( + self, + alias_name, + mappings, + primary_shards=1, + ping_client=False, + ): + if ping_client and not self.ping(): + return + if self.client.indices.exists_alias(name=alias_name): + return + + physical_name = generate_physical_index_name(alias_name) + if not self.client.indices.exists(index=physical_name): + self.client.indices.create( + index=physical_name, + body={ + "settings": get_index_settings(primary_shards), + "mappings": mappings, + "aliases": {alias_name: {}}, + }, + ) + return + + self.client.indices.put_alias(index=physical_name, name=alias_name) + def ensure_usage_indexes(self, collection, access_date, index_prefix=None): index_prefix = index_prefix or getattr( settings, "OPENSEARCH_INDEX_NAME", "usage", ) - year_index = generate_year_index_name(index_prefix, collection, access_date) month_index = generate_month_index_name(index_prefix, collection, access_date) - - self.create_index_if_not_exists( - year_index, - get_index_mappings(collection, "year"), + analytics_index = generate_analytics_index_name( + index_prefix, + collection, + access_date, ) - self.create_index_if_not_exists( + + self.create_alias_if_not_exists( month_index, - get_index_mappings(collection, "month"), + get_index_mappings("counter"), + ) + self.create_alias_if_not_exists( + analytics_index, + get_index_mappings("analytics"), ) - return {"year": year_index, "month": month_index} + return {"counter": month_index, "analytics": analytics_index} def index_documents(self, index_name, documents, ping_client=False): if ping_client and not self.ping(): @@ -130,24 +178,44 @@ def index_documents(self, index_name, documents, ping_client=False): ), ) - def increment_document_items_for_daily_job( + def index_document_items(self, index_name, document_items, ping_client=False): + if ping_client and not self.ping(): + return 0 + + succeeded, _failed = helpers.bulk( + self.client, + ( + {"_index": index_name, "_id": doc_id, "_source": document} + for doc_id, document in document_items + ), + chunk_size=self.bulk_chunk_size, + ) + return succeeded + + def increment_document_items_for_day( self, index_name, document_items, - job_id, + access_day, + annual=False, ping_client=False, ): if ping_client and not self.ping(): return + action_builder = ( + build_idempotent_annual_day_increment_action + if annual + else build_idempotent_day_increment_action + ) succeeded, _failed = helpers.bulk( self.client, ( - build_idempotent_job_increment_action( + action_builder( index_name=index_name, doc_id=doc_id, document=document, - job_id=job_id, + access_day=access_day, ) for doc_id, document in document_items ), diff --git a/metrics/tests/opensearch/test_client.py b/metrics/tests/opensearch/test_client.py index 227202b..651e1ce 100644 --- a/metrics/tests/opensearch/test_client.py +++ b/metrics/tests/opensearch/test_client.py @@ -1,163 +1,149 @@ -from unittest import TestCase from unittest.mock import Mock, patch -from django.test import override_settings +from django.test import SimpleTestCase, override_settings from metrics.opensearch.client import OpenSearchUsageClient from metrics.opensearch.mappings import ( - BOOKS_MONTH_INDEX_MAPPINGS, - BOOKS_YEAR_INDEX_MAPPINGS, + ANALYTICS_INDEX_MAPPINGS, + DOCUMENT_INDEX_MAPPINGS, MONTH_INDEX_MAPPINGS, - YEAR_INDEX_MAPPINGS, - get_index_mappings, + SOURCE_INDEX_MAPPINGS, + get_index_settings, ) -class OpenSearchUsageClientTests(TestCase): - @patch.object(OpenSearchUsageClient, "get_opensearch_client") - def test_create_index_sends_mappings_in_request_body(self, mock_get_client): - mock_client = Mock() - mock_get_client.return_value = mock_client +class OpenSearchUsageClientTests(SimpleTestCase): + def test_rejects_non_positive_primary_shard_count(self): + with self.assertRaisesMessage( + ValueError, + "OpenSearch primary shards must be greater than zero.", + ): + get_index_settings(0) + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_creates_physical_index_with_alias_and_compression(self, get_client): + raw_client = Mock() + raw_client.indices.exists_alias.return_value = False + raw_client.indices.exists.return_value = False + get_client.return_value = raw_client client = OpenSearchUsageClient(url="https://example.org:9200") - client.create_index( - index_name="usage_monthly_books_202506", - mappings=MONTH_INDEX_MAPPINGS, + + client.create_alias_if_not_exists( + "usage_monthly_scl_2026", + MONTH_INDEX_MAPPINGS, + primary_shards=3, ) - mock_client.indices.create.assert_called_once_with( - index="usage_monthly_books_202506", + raw_client.indices.create.assert_called_once_with( + index="usage_monthly_scl_2026_000001", body={ - "settings": {"index": {"number_of_replicas": 0}}, + "settings": get_index_settings(3), "mappings": MONTH_INDEX_MAPPINGS, + "aliases": {"usage_monthly_scl_2026": {}}, }, ) - @override_settings( - OPENSEARCH_VERIFY_CERTS=True, - OPENSEARCH_BASIC_AUTH=None, - OPENSEARCH_API_KEY=None, - OPENSEARCH_HTTP_COMPRESS=True, - ) - @patch("metrics.opensearch.client.OpenSearch") - def test_verify_certs_false_explicitly_overrides_settings(self, mock_opensearch): - OpenSearchUsageClient( - url="https://example.org:9200", - verify_certs=False, - ) + def test_mappings_are_explicit_and_separate_metadata_from_facts(self): + assert "source_key" in MONTH_INDEX_MAPPINGS["properties"] + assert "document_key" in MONTH_INDEX_MAPPINGS["properties"] + assert "year" in ANALYTICS_INDEX_MAPPINGS["properties"] + assert "source_key" in ANALYTICS_INDEX_MAPPINGS["properties"] + assert "document_key" in ANALYTICS_INDEX_MAPPINGS["properties"] + assert set(ANALYTICS_INDEX_MAPPINGS["_source"]["excludes"]) == { + "year", + "source_key", + "document_key", + "metric_scope", + "data_type", + "parent_data_type", + "article_version", + "access_type", + "access_method", + "country_code", + "content_language", + } + for mapping in (MONTH_INDEX_MAPPINGS, ANALYTICS_INDEX_MAPPINGS): + assert mapping["dynamic"] is False + assert "source" not in mapping["properties"] + assert "document" not in mapping["properties"] + assert "applied_days" in MONTH_INDEX_MAPPINGS["properties"] + assert "applied_day_masks" in ANALYTICS_INDEX_MAPPINGS["properties"] + assert DOCUMENT_INDEX_MAPPINGS["dynamic"] is False + assert SOURCE_INDEX_MAPPINGS["dynamic"] is False - mock_opensearch.assert_called_once_with( - "https://example.org:9200", - verify_certs=False, - http_compress=True, + @override_settings(OPENSEARCH_BULK_CHUNK_SIZE=2000) + @patch("metrics.opensearch.client.helpers.bulk") + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_daily_increment_uses_short_access_day(self, get_client, bulk): + get_client.return_value = Mock() + bulk.return_value = (1, 0) + client = OpenSearchUsageClient(url="https://example.org:9200") + + succeeded = client.increment_document_items_for_day( + index_name="usage_monthly_scl_2026", + document_items=iter([("key", {"total_requests": 1})]), + access_day="2026-08-20", ) - @override_settings( - OPENSEARCH_BASIC_AUTH=None, - OPENSEARCH_API_KEY=None, - OPENSEARCH_HTTP_COMPRESS=False, - ) - @patch("metrics.opensearch.client.OpenSearch") - def test_http_compression_can_be_disabled(self, mock_opensearch): - OpenSearchUsageClient(url="https://example.org:9200") + action = list(bulk.call_args.args[1])[0] + assert action["script"]["params"]["access_day"] == "2026-08-20" + assert action["upsert"] == {"applied_days": []} + assert bulk.call_args.kwargs["chunk_size"] == 2000 + assert succeeded == 1 - mock_opensearch.assert_called_once_with( - "https://example.org:9200", - verify_certs=False, - http_compress=False, - ) + @patch("metrics.opensearch.client.helpers.bulk") + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_annual_increment_uses_compact_day_mask(self, get_client, bulk): + get_client.return_value = Mock() + bulk.return_value = (1, 0) + client = OpenSearchUsageClient(url="https://example.org:9200") - def test_get_index_mappings_returns_books_specific_mappings(self): - self.assertIs( - get_index_mappings("books", "month"), - BOOKS_MONTH_INDEX_MAPPINGS, - ) - self.assertIs( - get_index_mappings("books", "year"), - BOOKS_YEAR_INDEX_MAPPINGS, + client.increment_document_items_for_day( + index_name="usage_yearly_analytics_scl_2026", + document_items=iter([("key", {"total_requests": 1})]), + access_day="2026-08-20", + annual=True, ) - self.assertIn("counter", BOOKS_MONTH_INDEX_MAPPINGS["properties"]) - self.assertIn("access", BOOKS_YEAR_INDEX_MAPPINGS["properties"]) - self.assertIn("applied_jobs", BOOKS_MONTH_INDEX_MAPPINGS["properties"]) - for mappings in ( - MONTH_INDEX_MAPPINGS, - YEAR_INDEX_MAPPINGS, - BOOKS_MONTH_INDEX_MAPPINGS, - BOOKS_YEAR_INDEX_MAPPINGS, - ): - for removed_field in ( - "document_type", - "scielo_document_type", - "pid", - "pid_v2", - "pid_v3", - "pid_generic", - "title_pid_generic", - "counter_data_type", - "access_month", - "access_year", - ): - self.assertNotIn(removed_field, mappings["properties"]) - document_mapping = mappings["properties"]["document"] - source_mapping = mappings["properties"]["source"] - self.assertEqual(document_mapping["properties"]["id"]["type"], "keyword") - self.assertEqual(document_mapping["properties"]["title"]["type"], "text") - self.assertFalse(document_mapping["properties"]["title"]["index"]) - self.assertEqual(source_mapping["properties"]["id"]["type"], "keyword") - self.assertEqual(source_mapping["properties"]["title"]["type"], "text") - self.assertFalse(source_mapping["properties"]["title"]["index"]) - self.assertEqual( - source_mapping["properties"]["publisher_name"]["type"], "text" - ) - self.assertFalse(source_mapping["properties"]["publisher_name"]["index"]) - @override_settings(OPENSEARCH_BULK_CHUNK_SIZE=2000) + action = list(bulk.call_args.args[1])[0] + params = action["script"]["params"] + assert params["mask_index"] == 3 + assert params["day_mask"] == 1 << 42 + assert action["upsert"] == {"applied_day_masks": [0] * 6} + @patch("metrics.opensearch.client.helpers.bulk") @patch.object(OpenSearchUsageClient, "get_opensearch_client") - def test_increment_documents_for_daily_job_uses_applied_jobs( + def test_annual_day_mask_covers_last_day_of_leap_year( self, - mock_get_client, - mock_bulk, + get_client, + bulk, ): - mock_get_client.return_value = Mock() + get_client.return_value = Mock() + bulk.return_value = (1, 0) client = OpenSearchUsageClient(url="https://example.org:9200") - mock_bulk.return_value = (1, 0) - documents = iter( - [ - ( - "doc-1", - { - "collection": "books", - "document": {"id": "BOOK:WD"}, - "access": {"month": "2025-06"}, - "total_requests": 3, - "total_investigations": 4, - "unique_requests": 2, - "unique_investigations": 3, - }, - ) - ] + client.increment_document_items_for_day( + index_name="usage_yearly_analytics_scl_2024", + document_items=iter([("key", {"total_requests": 1})]), + access_day="2024-12-31", + annual=True, ) - succeeded = client.increment_document_items_for_daily_job( - index_name="usage_monthly_books_202506", - document_items=documents, - job_id="books|2025-06-03|abc123", - ) + action = list(bulk.call_args.args[1])[0] + params = action["script"]["params"] + assert params["mask_index"] == 5 + assert params["day_mask"] == 1 << 50 - actions = list(mock_bulk.call_args.args[1]) - self.assertEqual(len(actions), 1) - action = actions[0] - self.assertEqual(action["_op_type"], "update") - self.assertEqual( - action["script"]["params"]["job_id"], "books|2025-06-03|abc123" + @override_settings( + OPENSEARCH_BASIC_AUTH=None, + OPENSEARCH_API_KEY=None, + OPENSEARCH_HTTP_COMPRESS=True, + ) + @patch("metrics.opensearch.client.OpenSearch") + def test_http_compression_remains_configurable(self, opensearch): + OpenSearchUsageClient(url="https://example.org:9200") + opensearch.assert_called_once_with( + "https://example.org:9200", + verify_certs=False, + http_compress=True, ) - self.assertEqual(action["upsert"], {"applied_jobs": []}) - self.assertEqual(succeeded, 1) - self.assertEqual(mock_bulk.call_args.kwargs["chunk_size"], 2000) - - @override_settings(OPENSEARCH_BULK_CHUNK_SIZE=0) - def test_bulk_chunk_size_must_be_positive(self): - with self.assertRaisesRegex(ValueError, "greater than zero"): - OpenSearchUsageClient(url="https://example.org:9200") From a611b880cc933b2d17730e330505f3102cc54c86 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:50 -0300 Subject: [PATCH 07/23] =?UTF-8?q?Grava=20fatos=20normalizados=20no=20paylo?= =?UTF-8?q?ad=20di=C3=A1rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/daily_payloads.py | 26 ++++++------ metrics/tests/services/test_daily_payloads.py | 40 ++++++++++--------- 2 files changed, 35 insertions(+), 31 deletions(-) diff --git a/metrics/services/daily_payloads.py b/metrics/services/daily_payloads.py index 2580447..ae321ab 100644 --- a/metrics/services/daily_payloads.py +++ b/metrics/services/daily_payloads.py @@ -44,7 +44,7 @@ def __init__(self, storage_path, collection, access_date): ) self.payload_hash = hashlib.sha256() self.output = None - self.next_granularity = "month" + self.next_dataset = "counter" self.completed = False def __enter__(self): @@ -54,13 +54,13 @@ def __enter__(self): self._write_json(self.access_date) self._write_text(',"collection":') self._write_json(self.collection) - self._write_text(',"documents":{"month":') + self._write_text(',"documents":{"counter":') return self - def write_document_items(self, granularity, document_items): - if granularity != self.next_granularity: + def write_document_items(self, dataset, document_items): + if dataset != self.next_dataset: raise RuntimeError( - f"Expected {self.next_granularity} documents, got {granularity}." + f"Expected {self.next_dataset} documents, got {dataset}." ) document_count = 0 @@ -73,16 +73,16 @@ def write_document_items(self, granularity, document_items): self._write_json(document) document_count += 1 self._write_text("}") - if granularity == "month": - self._write_text(',"year":') - self.next_granularity = "year" + if dataset == "counter": + self._write_text(',"analytics":') + self.next_dataset = "analytics" else: - self.next_granularity = None + self.next_dataset = None return document_count def finalize(self, input_log_hashes, summary): - if self.next_granularity is not None: - raise RuntimeError("Month and year documents must be written first.") + if self.next_dataset is not None: + raise RuntimeError("Counter and analytics documents must be written first.") self._write_text('},"input_log_hashes":') self._write_json(input_log_hashes) @@ -115,10 +115,10 @@ def _write_text(self, value): self.output.write(encoded_value) -def iter_document_items(storage_path, granularity): +def iter_document_items(storage_path, dataset): resolved_path = resolve_storage_path(storage_path) with resolved_path.open("rb") as payload_file: - yield from ijson.kvitems(payload_file, f"documents.{granularity}") + yield from ijson.kvitems(payload_file, f"documents.{dataset}") def cleanup_exported_payloads(collections=None, older_than_days=7): diff --git a/metrics/tests/services/test_daily_payloads.py b/metrics/tests/services/test_daily_payloads.py index 826fa9d..3ffaae6 100644 --- a/metrics/tests/services/test_daily_payloads.py +++ b/metrics/tests/services/test_daily_payloads.py @@ -23,19 +23,19 @@ def tearDown(self): def test_incremental_writer_preserves_canonical_bytes_and_hash(self): storage_path = Path("scl/2026/08/2026-08-25.json") payload = { - "collection": "scl", "access_date": "2026-08-25", - "input_log_hashes": ["abc"], + "collection": "scl", "documents": { - "month": {"á": {"total_requests": 2}}, - "year": {"z": {"total_requests": 3}}, + "counter": {"á": {"total_requests": 2}}, + "analytics": {"z": {"total_requests": 3}}, }, + "input_log_hashes": ["abc"], "summary": {"valid_lines": 1}, } expected = json.dumps( payload, ensure_ascii=True, - sort_keys=True, + sort_keys=False, separators=(",", ":"), ).encode("utf-8") @@ -45,10 +45,10 @@ def test_incremental_writer_preserves_canonical_bytes_and_hash(self): payload["access_date"], ) as writer: writer.write_document_items( - "month", sorted(payload["documents"]["month"].items()) + "counter", sorted(payload["documents"]["counter"].items()) ) writer.write_document_items( - "year", sorted(payload["documents"]["year"].items()) + "analytics", sorted(payload["documents"]["analytics"].items()) ) payload_hash = writer.finalize( payload["input_log_hashes"], @@ -66,8 +66,8 @@ def test_iter_document_items_reads_each_granularity_incrementally(self): "access_date": "2026-08-25", "input_log_hashes": ["abc"], "documents": { - "month": {"month-1": {"total_requests": 2}}, - "year": {"year-1": {"total_requests": 3}}, + "counter": {"month-1": {"total_requests": 2}}, + "analytics": {"analytics-1": {"total_requests": 3}}, }, "summary": {}, } @@ -76,17 +76,21 @@ def test_iter_document_items_reads_each_granularity_incrementally(self): payload["collection"], payload["access_date"], ) as writer: - writer.write_document_items("month", payload["documents"]["month"].items()) - writer.write_document_items("year", payload["documents"]["year"].items()) + writer.write_document_items( + "counter", payload["documents"]["counter"].items() + ) + writer.write_document_items( + "analytics", payload["documents"]["analytics"].items() + ) writer.finalize(payload["input_log_hashes"], payload["summary"]) self.assertEqual( - list(daily_payloads.iter_document_items(storage_path, "month")), + list(daily_payloads.iter_document_items(storage_path, "counter")), [("month-1", {"total_requests": 2})], ) self.assertEqual( - list(daily_payloads.iter_document_items(storage_path, "year")), - [("year-1", {"total_requests": 3})], + list(daily_payloads.iter_document_items(storage_path, "analytics")), + [("analytics-1", {"total_requests": 3})], ) def test_incremental_writer_removes_temporary_file_after_error(self): @@ -101,8 +105,8 @@ def test_incremental_writer_removes_temporary_file_after_error(self): "scl", "2026-08-25", ) as writer: - writer.write_document_items("month", iter(())) - writer.write_document_items("year", [("invalid", object())]) + writer.write_document_items("counter", iter(())) + writer.write_document_items("analytics", [("invalid", object())]) self.assertEqual(resolved_path.read_bytes(), b"previous canonical payload") self.assertFalse(resolved_path.with_suffix(".json.tmp").exists()) @@ -119,10 +123,10 @@ def test_document_item_order_produces_deterministic_payload(self): "2026-08-25", ) as writer: writer.write_document_items( - "month", + "counter", [("b", {"total": 2}), ("a", {"total": 1})], ) - writer.write_document_items("year", [("c", {"total": 3})]) + writer.write_document_items("analytics", [("c", {"total": 3})]) hashes.append(writer.finalize(["abc"], {"valid_lines": 1})) contents.append( daily_payloads.resolve_storage_path(storage_path).read_bytes() From 8d982e74916eb1421b99a5a76647eda455dec9b6 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:17:59 -0300 Subject: [PATCH 08/23] =?UTF-8?q?Converte=20os=20conjuntos=20do=20job=20di?= =?UTF-8?q?=C3=A1rio=20sequencialmente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/parsing/job_payloads.py | 46 +++++++++++++---------- metrics/tests/services/test_daily_jobs.py | 3 +- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/metrics/services/parsing/job_payloads.py b/metrics/services/parsing/job_payloads.py index f4213c7..8964203 100644 --- a/metrics/services/parsing/job_payloads.py +++ b/metrics/services/parsing/job_payloads.py @@ -149,8 +149,8 @@ def _write_job_payload(job, results, summary): job.collection, job.access_date, ) - month_document_count = 0 - year_document_count = 0 + counter_document_count = 0 + analytics_document_count = 0 payload_started = monotonic() with daily_payloads.DailyPayloadWriter( @@ -158,39 +158,45 @@ def _write_job_payload(job, results, summary): collection=job.collection.acron3, access_date=job.access_date.isoformat(), ) as writer: - month_started = monotonic() - month_documents = index_docs.iter_partitioned_documents( + counter_started = monotonic() + counter_documents = index_docs.iter_partitioned_documents( results, - "month", + "counter", ) - month_document_count = writer.write_document_items("month", month_documents) - month_conversion_seconds = monotonic() - month_started + counter_document_count = writer.write_document_items( + "counter", + counter_documents, + ) + counter_conversion_seconds = monotonic() - counter_started gc.collect() logging.info( - "Daily metric job %s monthly conversion and serialization completed " + "Daily metric job %s COUNTER conversion and serialization completed " "in %.3f seconds; %s documents; %s.", job.pk, - month_conversion_seconds, - month_document_count, + counter_conversion_seconds, + counter_document_count, memory.format_snapshot(), ) - year_started = monotonic() - year_documents = index_docs.iter_partitioned_documents( + analytics_started = monotonic() + analytics_documents = index_docs.iter_partitioned_documents( results, - "year", + "analytics", + ) + analytics_document_count = writer.write_document_items( + "analytics", + analytics_documents, ) - year_document_count = writer.write_document_items("year", year_documents) del results payload_hash = writer.finalize(summary["input_log_hashes"], summary) - year_conversion_seconds = monotonic() - year_started + analytics_conversion_seconds = monotonic() - analytics_started gc.collect() logging.info( - "Daily metric job %s yearly conversion and serialization completed " + "Daily metric job %s analytics conversion and serialization completed " "in %.3f seconds; %s documents; %s.", job.pk, - year_conversion_seconds, - year_document_count, + analytics_conversion_seconds, + analytics_document_count, memory.format_snapshot(), ) @@ -206,8 +212,8 @@ def _write_job_payload(job, results, summary): job.payload_hash = payload_hash job.summary = { **summary, - "month_document_count": month_document_count, - "year_document_count": year_document_count, + "counter_document_count": counter_document_count, + "analytics_document_count": analytics_document_count, } job.save( update_fields=[ diff --git a/metrics/tests/services/test_daily_jobs.py b/metrics/tests/services/test_daily_jobs.py index 57ef4d6..ed54c50 100644 --- a/metrics/tests/services/test_daily_jobs.py +++ b/metrics/tests/services/test_daily_jobs.py @@ -234,4 +234,5 @@ def test_payload_generation_releases_accumulator(self): _write_job_payload(job, accumulator, summary) self.assertEqual(len(accumulator), 0) - self.assertEqual(accumulator._documents, []) + self.assertEqual(accumulator._strings, []) + self.assertEqual(accumulator._sessions, {}) From ffd0c0b7c8ff0e4078d071941fdcfc13e4b3d119 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:18:09 -0300 Subject: [PATCH 09/23] =?UTF-8?q?Exporta=20fatos=20di=C3=A1rios=20para=20a?= =?UTF-8?q?liases=20anuais?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/export.py | 47 +++++++----- .../services/test_daily_metric_exports.py | 72 +++++++++++++------ 2 files changed, 78 insertions(+), 41 deletions(-) diff --git a/metrics/services/export.py b/metrics/services/export.py index dce465f..8430d5a 100644 --- a/metrics/services/export.py +++ b/metrics/services/export.py @@ -5,7 +5,10 @@ from django.conf import settings from metrics.opensearch.mappings import get_index_mappings -from metrics.opensearch.names import generate_month_index_name, generate_year_index_name +from metrics.opensearch.names import ( + generate_analytics_index_name, + generate_month_index_name, +) from metrics.services import daily_payloads, memory @@ -24,24 +27,24 @@ def export_daily_metric_payload(search_client, job): if not daily_metric_payload_exists(job): raise RuntimeError(f"Daily metric payload not found for job {job.pk}.") - for granularity in ("month", "year"): + for dataset in ("counter", "analytics"): started = monotonic() exported = _sync_documents_group( search_client=search_client, - collection=job.collection.acron3, + collection=job.collection, access_date=job.access_date, document_items=daily_payloads.iter_document_items( job.storage_path, - granularity, + dataset, ), - granularity=granularity, - job_id=job.job_id, + dataset=dataset, + access_day=job.access_date.isoformat(), ) logging.info( "Daily metric job %s %s OpenSearch export completed in %.3f " "seconds; %s documents; %s.", job.pk, - granularity, + dataset, monotonic() - started, exported, memory.format_snapshot(), @@ -53,8 +56,8 @@ def _sync_documents_group( collection, access_date, document_items, - granularity, - job_id, + dataset, + access_day, ): try: first_item = next(document_items) @@ -63,25 +66,33 @@ def _sync_documents_group( index_prefix = settings.OPENSEARCH_INDEX_NAME index_date = access_date.isoformat() - if granularity == "month": + collection_code = collection.acron3 + if dataset == "counter": index_name = generate_month_index_name( index_prefix=index_prefix, - collection=collection, + collection=collection_code, date=index_date, ) else: - index_name = generate_year_index_name( + index_name = generate_analytics_index_name( index_prefix=index_prefix, - collection=collection, + collection=collection_code, date=index_date, ) - search_client.create_index_if_not_exists( - index_name=index_name, - mappings=get_index_mappings(collection, granularity), + search_client.create_alias_if_not_exists( + alias_name=index_name, + mappings=get_index_mappings(dataset), + primary_shards=_get_primary_shards(collection), ) - return search_client.increment_document_items_for_daily_job( + return search_client.increment_document_items_for_day( index_name=index_name, document_items=chain((first_item,), document_items), - job_id=job_id, + access_day=access_day, + annual=dataset == "analytics", ) + + +def _get_primary_shards(collection): + config = getattr(collection, "log_manager_config", None) + return getattr(config, "opensearch_primary_shards", 1) diff --git a/metrics/tests/services/test_daily_metric_exports.py b/metrics/tests/services/test_daily_metric_exports.py index e6850b3..e61c40f 100644 --- a/metrics/tests/services/test_daily_metric_exports.py +++ b/metrics/tests/services/test_daily_metric_exports.py @@ -22,7 +22,10 @@ def setUp(self): self.storage_path = Path("scl/2026/08/2026-08-25.json") self.job = SimpleNamespace( pk=1, - collection=SimpleNamespace(acron3="scl"), + collection=SimpleNamespace( + acron3="scl", + log_manager_config=SimpleNamespace(opensearch_primary_shards=3), + ), access_date=date(2026, 8, 25), storage_path=self.storage_path.as_posix(), payload_hash="payload-hash", @@ -40,25 +43,32 @@ def _write_payload(self): "2026-08-25", ) as writer: writer.write_document_items( - "month", - [("month-doc", {"access": {"month": "2026-08"}})], + "counter", + [("month-doc", {"month": "2026-08"})], ) writer.write_document_items( - "year", - [("year-doc", {"access": {"year": "2026"}})], + "analytics", + [ + ( + "analytics-doc", + {"year": "2026", "country_code": "BR"}, + ) + ], ) writer.finalize(["abc"], {"valid_lines": 1}) - def test_export_streams_each_granularity_as_document_items(self): + def test_export_streams_each_dataset_as_document_items(self): self._write_payload() search_client = Mock() exported_groups = [] - def consume_items(index_name, document_items, job_id): - exported_groups.append((index_name, list(document_items), job_id)) + def consume_items(index_name, document_items, access_day, annual=False): + exported_groups.append( + (index_name, list(document_items), access_day, annual) + ) return 1 - search_client.increment_document_items_for_daily_job.side_effect = consume_items + search_client.increment_document_items_for_day.side_effect = consume_items export_daily_metric_payload(search_client, self.job) @@ -67,40 +77,56 @@ def consume_items(index_name, document_items, job_id): [ ( "usage_monthly_scl_2026", - [("month-doc", {"access": {"month": "2026-08"}})], - self.job.job_id, + [("month-doc", {"month": "2026-08"})], + "2026-08-25", + False, ), ( - "usage_yearly_scl_2026", - [("year-doc", {"access": {"year": "2026"}})], - self.job.job_id, + "usage_yearly_analytics_scl_2026", + [ + ( + "analytics-doc", + {"year": "2026", "country_code": "BR"}, + ) + ], + "2026-08-25", + True, ), ], ) + self.assertEqual( + [ + call.kwargs["primary_shards"] + for call in search_client.create_alias_if_not_exists.call_args_list + ], + [3, 3], + ) - def test_retry_after_partial_export_reuses_same_payload_and_job_id(self): + def test_retry_after_partial_export_reuses_same_payload_and_access_day(self): self._write_payload() first_client = Mock() - first_client.increment_document_items_for_daily_job.side_effect = [ + first_client.increment_document_items_for_day.side_effect = [ 1, - RuntimeError("year export failed"), + RuntimeError("analytics export failed"), ] - with self.assertRaisesMessage(RuntimeError, "year export failed"): + with self.assertRaisesMessage(RuntimeError, "analytics export failed"): export_daily_metric_payload(first_client, self.job) second_client = Mock() - second_client.increment_document_items_for_daily_job.side_effect = ( - lambda index_name, document_items, job_id: len(list(document_items)) + second_client.increment_document_items_for_day.side_effect = ( + lambda index_name, document_items, access_day, annual=False: len( + list(document_items) + ) ) export_daily_metric_payload(second_client, self.job) self.assertEqual( [ - call.kwargs["job_id"] - for call in second_client.increment_document_items_for_daily_job.call_args_list + call.kwargs["access_day"] + for call in second_client.increment_document_items_for_day.call_args_list ], - [self.job.job_id, self.job.job_id], + ["2026-08-25", "2026-08-25"], ) @patch("metrics.services.daily_metric_exports.fetch_required_resources") From ae0b8670ffe754979af12167a6649984a4b4112c Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:18:25 -0300 Subject: [PATCH 10/23] =?UTF-8?q?Remove=20configura=C3=A7=C3=A3o=20obsolet?= =?UTF-8?q?a=20de=20=C3=ADndices=20anuais?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/settings/base.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/config/settings/base.py b/config/settings/base.py index 0ca49ed..1df0d07 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -33,7 +33,6 @@ "wid", ) DEFAULT_PARSING_METADATA_CACHE_RELEASE_COLLECTIONS = ("scl",) -DEFAULT_YEAR_PARTITIONED_COLLECTIONS = ("chl", "col", "mex", "scl") ROOT_DIR = Path(__file__).resolve(strict=True).parent.parent.parent # core/ @@ -532,7 +531,3 @@ "PARSING_METADATA_CACHE_RELEASE_COLLECTIONS", default=list(DEFAULT_PARSING_METADATA_CACHE_RELEASE_COLLECTIONS), ) -YEAR_PARTITIONED_COLLECTIONS = env.list( - "YEAR_PARTITIONED_COLLECTIONS", - default=list(DEFAULT_YEAR_PARTITIONED_COLLECTIONS), -) From 12a4ca51c696eff2b0f78f2ef1d2362d5884714d Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:18:50 -0300 Subject: [PATCH 11/23] =?UTF-8?q?Modela=20cursor=20e=20outbox=20da=20sincr?= =?UTF-8?q?oniza=C3=A7=C3=A3o=20de=20metadados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/migrations/0002_metadata_sync.py | 73 ++++++++++++++++++++++++ metrics/models.py | 38 ++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 metrics/migrations/0002_metadata_sync.py diff --git a/metrics/migrations/0002_metadata_sync.py b/metrics/migrations/0002_metadata_sync.py new file mode 100644 index 0000000..0b9cce3 --- /dev/null +++ b/metrics/migrations/0002_metadata_sync.py @@ -0,0 +1,73 @@ +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [("metrics", "0001_initial")] + + operations = [ + migrations.CreateModel( + name="MetadataSyncState", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "entity", + models.CharField( + choices=[("source", "Source"), ("document", "Document")], + max_length=16, + unique=True, + ), + ), + ("cursor_updated", models.DateTimeField(blank=True, null=True)), + ("cursor_pk", models.PositiveBigIntegerField(default=0)), + ("lease_until", models.DateTimeField(blank=True, null=True)), + ("heartbeat_at", models.DateTimeField(blank=True, null=True)), + ("completed_at", models.DateTimeField(blank=True, null=True)), + ], + ), + migrations.CreateModel( + name="MetadataSyncOutbox", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "entity", + models.CharField( + choices=[("source", "Source"), ("document", "Document")], + max_length=16, + ), + ), + ("object_key", models.CharField(max_length=25)), + ("payload", models.JSONField()), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ], + options={ + "indexes": [ + models.Index( + fields=["entity", "id"], name="metrics_meta_outbox_entity_idx" + ) + ], + "constraints": [ + models.UniqueConstraint( + fields=("entity", "object_key"), + name="metrics_metadata_outbox_entity_key_uniq", + ) + ], + }, + ), + ] diff --git a/metrics/models.py b/metrics/models.py index 2a7e8b2..e1fbc1d 100644 --- a/metrics/models.py +++ b/metrics/models.py @@ -111,3 +111,41 @@ class Meta: def __str__(self): return f"{self.collection.acron3}-{self.access_date}" + + +class MetadataSyncState(models.Model): + ENTITY_SOURCE = "source" + ENTITY_DOCUMENT = "document" + ENTITY_CHOICES = ( + (ENTITY_SOURCE, _("Source")), + (ENTITY_DOCUMENT, _("Document")), + ) + + entity = models.CharField(max_length=16, choices=ENTITY_CHOICES, unique=True) + cursor_updated = models.DateTimeField(null=True, blank=True) + cursor_pk = models.PositiveBigIntegerField(default=0) + lease_until = models.DateTimeField(null=True, blank=True) + heartbeat_at = models.DateTimeField(null=True, blank=True) + completed_at = models.DateTimeField(null=True, blank=True) + + +class MetadataSyncOutbox(models.Model): + entity = models.CharField(max_length=16, choices=MetadataSyncState.ENTITY_CHOICES) + object_key = models.CharField(max_length=25) + payload = models.JSONField() + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=("entity", "object_key"), + name="metrics_metadata_outbox_entity_key_uniq", + ) + ] + indexes = [ + models.Index( + fields=("entity", "id"), + name="metrics_meta_outbox_entity_idx", + ) + ] From 9abed70835b19d4b0fc6ac3afd1bc3075f6ae396 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:18:59 -0300 Subject: [PATCH 12/23] =?UTF-8?q?Serializa=20fontes=20e=20documentos=20par?= =?UTF-8?q?a=20=C3=ADndices=20de=20metadados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/metadata_documents.py | 114 +++++++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 metrics/services/metadata_documents.py diff --git a/metrics/services/metadata_documents.py b/metrics/services/metadata_documents.py new file mode 100644 index 0000000..7bb521f --- /dev/null +++ b/metrics/services/metadata_documents.py @@ -0,0 +1,114 @@ +from metrics.opensearch.keys import document_key, source_key + + +def build_source_document(source, active=True): + key = source_key( + source.collection.acron3, + source.source_type, + source.source_id, + ) + return key, { + "key": key, + "collection": source.collection.acron3, + "source_id": source.source_id, + "source_type": source.source_type, + "title": source.title, + "scielo_issn": source.scielo_issn, + "acronym": source.acronym, + "publisher_name": source.publisher_name or [], + "access_type": source.access_type, + "country": (source.extra_data or {}).get("country"), + "subject_areas": source.subject_areas or [], + "wos_subject_areas": source.wos_subject_areas or [], + "identifiers": source.identifiers or {}, + "active": active, + "updated": source.updated.isoformat(), + } + + +def build_document_document(document, active=True): + collection = document.collection.acron3 + canonical_id = ( + document.pid_v3 + or document.pid_v2 + or document.pid_generic + or document.document_id + ) + key = document_key( + collection, + document.document_type, + canonical_id, + ) + source_identifier = None + if document.source_id: + source_identifier = source_key( + collection, + document.source.source_type, + document.source.source_id, + ) + parent_identifier = None + if document.parent_document_id: + parent = document.parent_document + parent_id = ( + parent.pid_v3 or parent.pid_v2 or parent.pid_generic or parent.document_id + ) + parent_identifier = document_key( + collection, + parent.document_type, + parent_id, + ) + return key, { + "key": key, + "collection": collection, + "document_id": document.document_id, + "document_type": document.document_type, + "source_key": source_identifier, + "parent_document_key": parent_identifier, + "title": document.title, + "publication_year": _integer_or_none(document.publication_year), + "default_lang": document.default_lang, + "text_langs": document.text_langs or [], + "identifiers": _document_identifiers(document), + "active": active, + "updated": document.updated.isoformat(), + } + + +def build_source_tombstone(source): + key, payload = build_source_document(source, active=False) + return key, { + "key": key, + "collection": payload["collection"], + "source_id": payload["source_id"], + "source_type": payload["source_type"], + "active": False, + "updated": payload["updated"], + } + + +def build_document_tombstone(document): + key, payload = build_document_document(document, active=False) + return key, { + "key": key, + "collection": payload["collection"], + "document_id": payload["document_id"], + "document_type": payload["document_type"], + "active": False, + "updated": payload["updated"], + } + + +def _document_identifiers(document): + identifiers = dict(document.identifiers or {}) + for name in ("pid_v2", "pid_v3", "pid_generic", "scielo_issn"): + value = getattr(document, name) + if value: + identifiers[name] = value + return identifiers + + +def _integer_or_none(value): + try: + return int(value) + except (TypeError, ValueError): + return None From 7c7304fe7bf0546bd792960bd50151307a3a064f Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:19:07 -0300 Subject: [PATCH 13/23] =?UTF-8?q?Registra=20altera=C3=A7=C3=B5es=20de=20me?= =?UTF-8?q?tadados=20na=20outbox?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/apps.py | 3 +++ metrics/signals.py | 41 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 metrics/signals.py diff --git a/metrics/apps.py b/metrics/apps.py index e609fae..7628557 100755 --- a/metrics/apps.py +++ b/metrics/apps.py @@ -4,3 +4,6 @@ class MetricsConfig(AppConfig): default_auto_field = "django.db.models.BigAutoField" name = "metrics" + + def ready(self): + from metrics import signals # noqa: F401 diff --git a/metrics/signals.py b/metrics/signals.py new file mode 100644 index 0000000..f2dda66 --- /dev/null +++ b/metrics/signals.py @@ -0,0 +1,41 @@ +from django.db.models.signals import post_save, pre_delete +from django.dispatch import receiver + +from document.models import Document +from metrics.models import MetadataSyncOutbox, MetadataSyncState +from metrics.services.metadata_documents import ( + build_document_document, + build_document_tombstone, + build_source_document, + build_source_tombstone, +) +from source.models import Source + + +def _put(entity, item): + object_key, payload = item + MetadataSyncOutbox.objects.update_or_create( + entity=entity, + object_key=object_key, + defaults={"payload": payload}, + ) + + +@receiver(post_save, sender=Source) +def source_saved(sender, instance, **kwargs): + _put(MetadataSyncState.ENTITY_SOURCE, build_source_document(instance)) + + +@receiver(pre_delete, sender=Source) +def source_deleted(sender, instance, **kwargs): + _put(MetadataSyncState.ENTITY_SOURCE, build_source_tombstone(instance)) + + +@receiver(post_save, sender=Document) +def document_saved(sender, instance, **kwargs): + _put(MetadataSyncState.ENTITY_DOCUMENT, build_document_document(instance)) + + +@receiver(pre_delete, sender=Document) +def document_deleted(sender, instance, **kwargs): + _put(MetadataSyncState.ENTITY_DOCUMENT, build_document_tombstone(instance)) From 185031f66c23cc45041160039b2718747098350a Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:19:18 -0300 Subject: [PATCH 14/23] Sincroniza metadados incrementalmente com o OpenSearch --- config/settings/base.py | 1 + metrics/services/metadata_sync.py | 150 +++++++++++++++++++ metrics/tasks/metadata_sync.py | 12 ++ metrics/tests/services/test_metadata_sync.py | 85 +++++++++++ 4 files changed, 248 insertions(+) create mode 100644 metrics/services/metadata_sync.py create mode 100644 metrics/tasks/metadata_sync.py create mode 100644 metrics/tests/services/test_metadata_sync.py diff --git a/config/settings/base.py b/config/settings/base.py index 1df0d07..6559876 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -356,6 +356,7 @@ "metrics.tasks.daily_metric_exports", "metrics.tasks.index", "metrics.tasks.log_parsing", + "metrics.tasks.metadata_sync", "metrics.tasks.resume", ) # http://docs.celeryproject.org/en/latest/userguide/configuration.html#std:setting-task_serializer diff --git a/metrics/services/metadata_sync.py b/metrics/services/metadata_sync.py new file mode 100644 index 0000000..c1e4092 --- /dev/null +++ b/metrics/services/metadata_sync.py @@ -0,0 +1,150 @@ +import logging +from datetime import timedelta + +from django.db import transaction +from django.db.models import Q +from django.utils import timezone + +from document.models import Document +from metrics.models import MetadataSyncOutbox, MetadataSyncState +from metrics.opensearch.client import OpenSearchUsageClient +from metrics.opensearch.mappings import get_index_mappings +from metrics.opensearch.names import generate_metadata_alias +from metrics.services.metadata_documents import ( + build_document_document, + build_source_document, +) +from source.models import Source + +DEFAULT_BATCH_SIZE = 1000 +LEASE_MINUTES = 30 + + +def sync_metadata(entity=None, batch_size=DEFAULT_BATCH_SIZE): + entities = ( + [entity] + if entity + else [ + MetadataSyncState.ENTITY_SOURCE, + MetadataSyncState.ENTITY_DOCUMENT, + ] + ) + result = {} + client = OpenSearchUsageClient() + if not client.ping(): + raise RuntimeError("OpenSearch client is not available.") + + for current_entity in entities: + result[current_entity] = _sync_entity(client, current_entity, batch_size) + return result + + +def _sync_entity(client, entity, batch_size): + model, serializer, related = _entity_config(entity) + state = _acquire_lease(entity) + alias = generate_metadata_alias(f"{entity}s") + client.create_alias_if_not_exists(alias, get_index_mappings(f"{entity}s")) + exported = 0 + + try: + while True: + batch = list(_cursor_queryset(model, state, related)[:batch_size]) + if not batch: + break + exported += client.index_document_items( + alias, + (serializer(item) for item in batch), + ) + last = batch[-1] + state.cursor_updated = last.updated + state.cursor_pk = last.pk + _heartbeat(state) + + exported += _flush_outbox(client, entity, alias, batch_size) + state.completed_at = timezone.now() + return exported + finally: + state.lease_until = None + state.heartbeat_at = timezone.now() + state.save( + update_fields=[ + "cursor_updated", + "cursor_pk", + "lease_until", + "heartbeat_at", + "completed_at", + ] + ) + logging.info("Metadata sync completed for %s: %s documents.", entity, exported) + + +def _entity_config(entity): + if entity == MetadataSyncState.ENTITY_SOURCE: + return Source, build_source_document, ("collection",) + if entity == MetadataSyncState.ENTITY_DOCUMENT: + return ( + Document, + build_document_document, + ( + "collection", + "source", + "parent_document", + ), + ) + raise ValueError(f"Unsupported metadata entity: {entity}.") + + +def _cursor_queryset(model, state, related): + queryset = model.objects.select_related(*related).order_by("updated", "pk") + if state.cursor_updated is None: + return queryset + return queryset.filter( + Q(updated__gt=state.cursor_updated) + | Q(updated=state.cursor_updated, pk__gt=state.cursor_pk) + ) + + +def _acquire_lease(entity): + now = timezone.now() + with transaction.atomic(): + state, _created = MetadataSyncState.objects.select_for_update().get_or_create( + entity=entity + ) + if state.lease_until and state.lease_until > now: + raise RuntimeError(f"Metadata sync already running for {entity}.") + state.lease_until = now + timedelta(minutes=LEASE_MINUTES) + state.heartbeat_at = now + state.save(update_fields=["lease_until", "heartbeat_at"]) + return state + + +def _heartbeat(state): + now = timezone.now() + state.heartbeat_at = now + state.lease_until = now + timedelta(minutes=LEASE_MINUTES) + state.save( + update_fields=[ + "cursor_updated", + "cursor_pk", + "heartbeat_at", + "lease_until", + ] + ) + + +def _flush_outbox(client, entity, alias, batch_size): + exported = 0 + while True: + entries = list( + MetadataSyncOutbox.objects.filter(entity=entity).order_by("pk")[:batch_size] + ) + if not entries: + return exported + + exported += client.index_document_items( + alias, + ((entry.object_key, entry.payload) for entry in entries), + ) + MetadataSyncOutbox.objects.filter( + pk__in=[entry.pk for entry in entries] + ).delete() diff --git a/metrics/tasks/metadata_sync.py b/metrics/tasks/metadata_sync.py new file mode 100644 index 0000000..12b3dc1 --- /dev/null +++ b/metrics/tasks/metadata_sync.py @@ -0,0 +1,12 @@ +from config import celery_app +from metrics.services.metadata_sync import sync_metadata + + +@celery_app.task( + bind=True, + name="[Metadata] Sync OpenSearch metadata", + queue="load", + timelimit=-1, +) +def task_sync_opensearch_metadata(self, entity=None, batch_size=1000): + return sync_metadata(entity=entity, batch_size=batch_size) diff --git a/metrics/tests/services/test_metadata_sync.py b/metrics/tests/services/test_metadata_sync.py new file mode 100644 index 0000000..a8b7c4a --- /dev/null +++ b/metrics/tests/services/test_metadata_sync.py @@ -0,0 +1,85 @@ +from unittest.mock import Mock, patch + +from django.test import TestCase + +from collection.models import Collection +from document.models import Document +from metrics.models import MetadataSyncOutbox, MetadataSyncState +from metrics.services.metadata_documents import ( + build_document_document, + build_source_document, +) +from metrics.services.metadata_sync import sync_metadata +from source.models import Source + + +class MetadataDocumentsTests(TestCase): + def setUp(self): + self.collection = Collection.objects.create(acron3="prt", acron2="pt") + self.source = Source.objects.create( + collection=self.collection, + source_type=Source.SOURCE_TYPE_JOURNAL, + source_id="1234-5678", + title="Journal title", + subject_areas=["Medicine"], + ) + self.document = Document.objects.create( + collection=self.collection, + source=self.source, + document_type=Document.DOCUMENT_TYPE_ARTICLE, + document_id="internal-id", + pid_v3="S1234-56782026000100001", + title="Article title", + publication_year="2026", + ) + + def test_metadata_is_stored_once_under_the_same_keys_used_by_facts(self): + source_key, source_payload = build_source_document(self.source) + document_key, document_payload = build_document_document(self.document) + + self.assertTrue(source_key.startswith("k1_")) + self.assertTrue(document_key.startswith("k1_")) + self.assertEqual(document_payload["source_key"], source_key) + self.assertEqual(source_payload["title"], "Journal title") + self.assertEqual(document_payload["title"], "Article title") + self.assertEqual(document_payload["publication_year"], 2026) + self.assertTrue(source_payload["active"]) + self.assertTrue(document_payload["active"]) + + def test_save_and_delete_create_transactional_outbox_entries(self): + source_key, _payload = build_source_document(self.source) + document_key, _payload = build_document_document(self.document) + + self.assertTrue( + MetadataSyncOutbox.objects.filter( + entity=MetadataSyncState.ENTITY_SOURCE, + object_key=source_key, + payload__active=True, + ).exists() + ) + self.document.delete() + entry = MetadataSyncOutbox.objects.get( + entity=MetadataSyncState.ENTITY_DOCUMENT, + object_key=document_key, + ) + self.assertFalse(entry.payload["active"]) + + @patch("metrics.services.metadata_sync.OpenSearchUsageClient") + def test_sync_advances_cursor_and_clears_exported_outbox(self, client_class): + client = Mock() + client.ping.return_value = True + client.index_document_items.side_effect = lambda index, items: len(list(items)) + client_class.return_value = client + + exported = sync_metadata(entity=MetadataSyncState.ENTITY_SOURCE) + + state = MetadataSyncState.objects.get(entity=MetadataSyncState.ENTITY_SOURCE) + self.assertGreaterEqual(exported[MetadataSyncState.ENTITY_SOURCE], 1) + self.assertEqual(state.cursor_pk, self.source.pk) + self.assertIsNone(state.lease_until) + self.assertFalse( + MetadataSyncOutbox.objects.filter( + entity=MetadataSyncState.ENTITY_SOURCE + ).exists() + ) + client.create_alias_if_not_exists.assert_called_once() From dc9723729ec98197aada51f2c0587a0883341c38 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:19:29 -0300 Subject: [PATCH 15/23] =?UTF-8?q?Usa=20a=20data=20do=20log=20na=20fronteir?= =?UTF-8?q?a=20anual=20das=20m=C3=A9tricas?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/parsing/lines.py | 1 + metrics/tests/parsing/test_process_line.py | 20 +++++++++++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/metrics/services/parsing/lines.py b/metrics/services/parsing/lines.py index f7a7f04..eb41c36 100644 --- a/metrics/services/parsing/lines.py +++ b/metrics/services/parsing/lines.py @@ -55,6 +55,7 @@ def process_line(results, line, utm, log_file, track_errors=False): results, counter_access, line, + reporting_date=log_file.date, ) except Exception as exc: logging.error( diff --git a/metrics/tests/parsing/test_process_line.py b/metrics/tests/parsing/test_process_line.py index 71586b1..e4ba46f 100644 --- a/metrics/tests/parsing/test_process_line.py +++ b/metrics/tests/parsing/test_process_line.py @@ -1,4 +1,4 @@ -from datetime import date +from datetime import date, datetime from unittest.mock import Mock from django.test import TestCase @@ -76,8 +76,6 @@ def test_url_translation_error_returns_false_none(self): self.assertIsNone(error) def test_valid_line_accumulates_result(self): - from datetime import datetime - results = DailyAccessAccumulator() is_valid, error = process_line( results=results, @@ -89,6 +87,22 @@ def test_valid_line_accumulates_result(self): self.assertIsNone(error) self.assertEqual(len(results), 1) + def test_uses_validated_log_date_at_year_boundary(self): + self.log_file.date = date(2026, 1, 1) + results = DailyAccessAccumulator() + + is_valid, error = process_line( + results=results, + line=self._line(local_datetime=datetime(2025, 12, 31, 23, 59, 59)), + utm=self._fake_utm(), + log_file=self.log_file, + ) + + record = next(results.iter_materialized_values()) + self.assertTrue(is_valid) + self.assertIsNone(error) + self.assertEqual(record["access_date"], "2026-01-01") + def test_validation_failure_without_track_errors_returns_no_discarded_line(self): results = DailyAccessAccumulator() utm = self._fake_utm( From 2a1735753edddbca1597c2deba6aede17f8b7922 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 23:13:45 -0300 Subject: [PATCH 16/23] =?UTF-8?q?Configura=20a=20pol=C3=ADtica=20de=20?= =?UTF-8?q?=C3=ADndices=20por=20cole=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- config/collections.py | 7 ++++ config/settings/base.py | 4 ++ log_manager_config/choices.py | 7 ++++ ...ollectionconfig_opensearch_index_policy.py | 38 +++++++++++++++++++ log_manager_config/models.py | 26 +++++++++++++ log_manager_config/tests/test_models.py | 14 +++++++ log_manager_config/tests/test_tasks.py | 10 +++++ 7 files changed, 106 insertions(+) create mode 100644 log_manager_config/choices.py create mode 100644 log_manager_config/migrations/0006_logmanagercollectionconfig_opensearch_index_policy.py diff --git a/config/collections.py b/config/collections.py index 369b0f7..7a8155b 100644 --- a/config/collections.py +++ b/config/collections.py @@ -1,3 +1,5 @@ +from log_manager_config.choices import OpenSearchPartitionStrategy + COLLECTION_OPAC_URL_MAP = { "dom": "https://scielo.do/api/v1/counter_dict", "scl": "https://www.scielo.br/api/v1/counter_dict", @@ -23,6 +25,7 @@ }, { "acronym": "chl", + "opensearch_partition_strategy": OpenSearchPartitionStrategy.YEARLY, "directory_name": "Site clássico", "path": "/app/logs/bkp-ratchet/scielo.cl", "quantity": 1, @@ -33,6 +36,7 @@ }, { "acronym": "col", + "opensearch_partition_strategy": OpenSearchPartitionStrategy.YEARLY, "directory_name": "Site clássico", "path": "/app/logs/bkp-ratchet/scielo.co", "quantity": 1, @@ -100,6 +104,7 @@ }, { "acronym": "mex", + "opensearch_partition_strategy": OpenSearchPartitionStrategy.YEARLY, "directory_name": "Site clássico", "path": "/app/logs/bkp-ratchet/scielo.mx", "quantity": 1, @@ -166,6 +171,7 @@ }, { "acronym": "scl", + "opensearch_partition_strategy": OpenSearchPartitionStrategy.YEARLY, "directory_name": "SciELO Brasil", "path": "/app/logs/bkp-bunnynet/scielo-br", "quantity": 2, @@ -176,6 +182,7 @@ }, { "acronym": "scl", + "opensearch_partition_strategy": OpenSearchPartitionStrategy.YEARLY, "directory_name": "SciELO Brasil 2", "path": "/app/logs/bkp-bunnynet/scielo-br-2", "quantity": 2, diff --git a/config/settings/base.py b/config/settings/base.py index 6559876..63ffcd0 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -467,6 +467,10 @@ "OPENSEARCH_BULK_CHUNK_SIZE", default=500, ) +OPENSEARCH_ROLLOVER_MAX_SIZE = env( + "OPENSEARCH_ROLLOVER_MAX_SIZE", + default="50gb", +) # Resources # ------------------------------------------------------------------------------ diff --git a/log_manager_config/choices.py b/log_manager_config/choices.py new file mode 100644 index 0000000..c9a8279 --- /dev/null +++ b/log_manager_config/choices.py @@ -0,0 +1,7 @@ +from django.db import models +from django.utils.translation import gettext_lazy as _ + + +class OpenSearchPartitionStrategy(models.TextChoices): + ROLLOVER = "rollover", _("Continuous with size rollover") + YEARLY = "yearly", _("One physical index per year") diff --git a/log_manager_config/migrations/0006_logmanagercollectionconfig_opensearch_index_policy.py b/log_manager_config/migrations/0006_logmanagercollectionconfig_opensearch_index_policy.py new file mode 100644 index 0000000..8e39ef2 --- /dev/null +++ b/log_manager_config/migrations/0006_logmanagercollectionconfig_opensearch_index_policy.py @@ -0,0 +1,38 @@ +# Generated by Django 5.2.12 on 2026-09-11 01:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("log_manager_config", "0005_alter_collectionlogdirectory_translator_class"), + ] + + operations = [ + migrations.AddField( + model_name="logmanagercollectionconfig", + name="opensearch_primary_shards", + field=models.PositiveSmallIntegerField( + default=1, + help_text="Applied only when a new physical index is created.", + verbose_name="OpenSearch Primary Shards", + ), + ), + migrations.AddField( + model_name="logmanagercollectionconfig", + name="opensearch_partition_strategy", + field=models.CharField( + choices=[ + ("rollover", "Continuous with size rollover"), + ("yearly", "One physical index per year"), + ], + default="rollover", + help_text=( + "Uses yearly physical indexes or continuous indexes " + "rolled over by size." + ), + max_length=16, + verbose_name="OpenSearch Partition Strategy", + ), + ), + ] diff --git a/log_manager_config/models.py b/log_manager_config/models.py index 80ed804..a82ebe7 100644 --- a/log_manager_config/models.py +++ b/log_manager_config/models.py @@ -11,6 +11,7 @@ from collection.models import Collection from core.models import CommonControlField +from log_manager_config.choices import OpenSearchPartitionStrategy class LogManagerCollectionConfig(ClusterableModel, CommonControlField): @@ -36,12 +37,28 @@ class LogManagerCollectionConfig(ClusterableModel, CommonControlField): verbose_name=_("Expected Logs Per Day"), default=1, ) + opensearch_primary_shards = models.PositiveSmallIntegerField( + verbose_name=_("OpenSearch Primary Shards"), + default=1, + help_text=_("Applied only when a new physical index is created."), + ) + opensearch_partition_strategy = models.CharField( + verbose_name=_("OpenSearch Partition Strategy"), + max_length=16, + choices=OpenSearchPartitionStrategy.choices, + default=OpenSearchPartitionStrategy.ROLLOVER, + help_text=_( + "Uses yearly physical indexes or continuous indexes rolled over by size." + ), + ) panels = [ AutocompletePanel("collection"), FieldPanel("sample_size"), FieldPanel("buffer_size"), FieldPanel("expected_logs_per_day"), + FieldPanel("opensearch_primary_shards"), + FieldPanel("opensearch_partition_strategy"), InlinePanel("directories", label=_("Directories")), InlinePanel("emails", label=_("Emails")), ] @@ -68,6 +85,11 @@ def load(cls, data, user): sample_size=item.get("sample_size", 0.1), buffer_size=item.get("buffer_size", 2048), expected_logs_per_day=item.get("quantity", 1), + opensearch_primary_shards=item.get("opensearch_primary_shards", 1), + opensearch_partition_strategy=item.get( + "opensearch_partition_strategy", + OpenSearchPartitionStrategy.ROLLOVER, + ), ) @classmethod @@ -78,6 +100,8 @@ def create_or_update( sample_size, buffer_size, expected_logs_per_day, + opensearch_primary_shards=1, + opensearch_partition_strategy=OpenSearchPartitionStrategy.ROLLOVER, ): obj, created = cls.objects.get_or_create(collection=collection) if created: @@ -89,6 +113,8 @@ def create_or_update( obj.sample_size = sample_size obj.buffer_size = buffer_size obj.expected_logs_per_day = expected_logs_per_day + obj.opensearch_primary_shards = opensearch_primary_shards + obj.opensearch_partition_strategy = opensearch_partition_strategy obj.save() logging.info(f"Config for {collection.acron3} updated.") return obj diff --git a/log_manager_config/tests/test_models.py b/log_manager_config/tests/test_models.py index fa8fcb7..5e7bcb3 100644 --- a/log_manager_config/tests/test_models.py +++ b/log_manager_config/tests/test_models.py @@ -2,6 +2,7 @@ from collection.models import Collection from core.users.tests.factories import UserFactory +from log_manager_config.choices import OpenSearchPartitionStrategy from log_manager_config.models import CollectionLogDirectory, LogManagerCollectionConfig @@ -17,12 +18,19 @@ def test_create_or_update_creates_config(self): sample_size=0.2, buffer_size=4096, expected_logs_per_day=3, + opensearch_primary_shards=2, + opensearch_partition_strategy=OpenSearchPartitionStrategy.YEARLY, ) self.assertEqual(config.collection, self.collection) self.assertEqual(config.sample_size, 0.2) self.assertEqual(config.buffer_size, 4096) self.assertEqual(config.expected_logs_per_day, 3) + self.assertEqual(config.opensearch_primary_shards, 2) + self.assertEqual( + config.opensearch_partition_strategy, + OpenSearchPartitionStrategy.YEARLY, + ) def test_create_or_update_updates_existing(self): LogManagerCollectionConfig.create_or_update( @@ -38,11 +46,17 @@ def test_create_or_update_updates_existing(self): sample_size=0.5, buffer_size=8192, expected_logs_per_day=5, + opensearch_primary_shards=3, ) self.assertEqual(LogManagerCollectionConfig.objects.count(), 1) self.assertEqual(config.sample_size, 0.5) self.assertEqual(config.buffer_size, 8192) + self.assertEqual(config.opensearch_primary_shards, 3) + self.assertEqual( + config.opensearch_partition_strategy, + OpenSearchPartitionStrategy.ROLLOVER, + ) class CollectionLogDirectoryTests(TestCase): diff --git a/log_manager_config/tests/test_tasks.py b/log_manager_config/tests/test_tasks.py index 0253fd1..bbc7b18 100644 --- a/log_manager_config/tests/test_tasks.py +++ b/log_manager_config/tests/test_tasks.py @@ -53,6 +53,16 @@ def test_default_seed_matches_log_directories(): assert { item["quantity"] for item in LOG_MANAGER_SEED_DATA if item["acronym"] == "scl" } == {2} + assert { + item.get("opensearch_primary_shards", 1) + for item in LOG_MANAGER_SEED_DATA + if item["acronym"] == "scl" + } == {1} + assert { + item["acronym"] + for item in LOG_MANAGER_SEED_DATA + if item.get("opensearch_partition_strategy", "rollover") == "yearly" + } == {"chl", "col", "mex", "scl"} assert not { "dom", "rvt", From bc011beb46110e1ca374fc7229f665d528bb930b Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:19:51 -0300 Subject: [PATCH 17/23] Atualiza bibliotecas auxiliares de processamento --- requirements/base.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements/base.txt b/requirements/base.txt index e6da798..affcfd3 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -63,13 +63,13 @@ minio==7.2.7 reverse-geocode==1.6 # https://pypi.org/project/reverse-geocode/ # SciELO Log Validator -git+https://github.com/scieloorg/scielo_log_validator@2.0.2#egg=scielo_log_validator +git+https://github.com/scieloorg/scielo_log_validator@2.0.3#egg=scielo_log_validator # SciELO Scholarly Data git+https://github.com/scieloorg/scielo_scholarly_data@v0.1.4#egg=scielo_scholarly_data # SciELO Usage COUNTER -git+https://github.com/scieloorg/scielo_usage_counter@2.2.1#egg=scielo_usage_counter +git+https://github.com/scieloorg/scielo_usage_counter@2.2.2#egg=scielo_usage_counter # Incremental JSON ijson==3.5.1 From e12e88cdf8c557744c25f9551e36e64eaedab722 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 23:11:19 -0300 Subject: [PATCH 18/23] =?UTF-8?q?Refina=20convers=C3=A3o=20e=20defini?= =?UTF-8?q?=C3=A7=C3=B5es=20do=20OpenSearch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/counter/indexing/converter.py | 14 +++--- metrics/counter/indexing/engines/book.py | 4 +- metrics/opensearch/mappings.py | 54 ++++++++++++------------ metrics/opensearch/painless.py | 3 +- 4 files changed, 38 insertions(+), 37 deletions(-) diff --git a/metrics/counter/indexing/converter.py b/metrics/counter/indexing/converter.py index 153d536..f35c4d8 100644 --- a/metrics/counter/indexing/converter.py +++ b/metrics/counter/indexing/converter.py @@ -13,9 +13,9 @@ "book": BookPipeline(), "chapter": BookPipeline(), } -_DEFAULT = DocumentPipeline() +_DEFAULT_PIPELINE = DocumentPipeline() _DEFAULT_PARTITION_COUNT = 64 -ANALYTICS_PROJECTION = "country_language" +_ANALYTICS_PROJECTION = "country_language" def iter_partitioned_documents( @@ -34,7 +34,7 @@ def iter_partitioned_documents( partition = _partition_for_value( value, partition_count, - projection=ANALYTICS_PROJECTION if dataset == "analytics" else None, + projection=_ANALYTICS_PROJECTION if dataset == "analytics" else None, ) partitions[partition].append(record_key) @@ -52,7 +52,7 @@ def iter_partitioned_documents( ) yield from _convert_partition( materialized, - projection=ANALYTICS_PROJECTION, + projection=_ANALYTICS_PROJECTION, ) materialized.clear() record_keys.clear() @@ -77,7 +77,7 @@ def iter_partitioned_values( partition = _partition_for_value( value, partition_count, - projection=ANALYTICS_PROJECTION if dataset == "analytics" else None, + projection=_ANALYTICS_PROJECTION if dataset == "analytics" else None, ) partitions[partition].append(value) @@ -88,7 +88,7 @@ def iter_partitioned_values( else: yield from _convert_partition( partition_values, - projection=ANALYTICS_PROJECTION, + projection=_ANALYTICS_PROJECTION, ) partition_values.clear() finally: @@ -134,7 +134,7 @@ def _get_pipeline(value): if collection == "books": return _PIPELINES["book"] - return _PIPELINES.get(value.get("document_type"), _DEFAULT) + return _PIPELINES.get(value.get("document_type"), _DEFAULT_PIPELINE) def _initialize_unique_state(): diff --git a/metrics/counter/indexing/engines/book.py b/metrics/counter/indexing/engines/book.py index a94ed91..d985569 100644 --- a/metrics/counter/indexing/engines/book.py +++ b/metrics/counter/indexing/engines/book.py @@ -157,7 +157,7 @@ def _should_create_item_document(value): return True -def _extract_title_pid_generic(value, fallback=None): +def _extract_title_pid_generic(value): title_pid_generic = value.get("title_pid_generic") if title_pid_generic: return title_pid_generic @@ -171,4 +171,4 @@ def _extract_title_pid_generic(value, fallback=None): if source_id: return f"BOOK:{str(source_id).upper()}" - return fallback + return None diff --git a/metrics/opensearch/mappings.py b/metrics/opensearch/mappings.py index dbfddf2..23b84f6 100644 --- a/metrics/opensearch/mappings.py +++ b/metrics/opensearch/mappings.py @@ -1,11 +1,11 @@ -METRIC_PROPERTIES = { +_METRIC_PROPERTIES = { "total_requests": {"type": "long"}, "total_investigations": {"type": "long"}, "unique_requests": {"type": "long"}, "unique_investigations": {"type": "long"}, } -COUNTER_DIMENSIONS = { +_COUNTER_DIMENSIONS = { "metric_scope": {"type": "keyword"}, "data_type": {"type": "keyword"}, "parent_data_type": {"type": "keyword"}, @@ -15,26 +15,25 @@ } -def _metric_properties(): - return { - "collection": {"type": "keyword"}, - "source_key": {"type": "keyword"}, - "document_key": {"type": "keyword"}, - "month": {"type": "date", "format": "yyyy-MM"}, - "applied_days": { - "type": "keyword", - "index": False, - "doc_values": False, - }, - **COUNTER_DIMENSIONS, - **METRIC_PROPERTIES, - } +_BASE_METRIC_PROPERTIES = { + "collection": {"type": "keyword"}, + "source_key": {"type": "keyword"}, + "document_key": {"type": "keyword"}, + "month": {"type": "date", "format": "yyyy-MM"}, + "applied_days": { + "type": "keyword", + "index": False, + "doc_values": False, + }, + **_COUNTER_DIMENSIONS, + **_METRIC_PROPERTIES, +} MONTH_INDEX_MAPPINGS = { "dynamic": False, "properties": { - **_metric_properties(), + **_BASE_METRIC_PROPERTIES, "daily_metrics": {"type": "object", "dynamic": False}, }, } @@ -46,7 +45,7 @@ def _metric_properties(): "year", "source_key", "document_key", - *COUNTER_DIMENSIONS, + *_COUNTER_DIMENSIONS, "country_code", "content_language", ] @@ -55,7 +54,7 @@ def _metric_properties(): "year": {"type": "keyword"}, "source_key": {"type": "keyword"}, "document_key": {"type": "keyword"}, - **COUNTER_DIMENSIONS, + **_COUNTER_DIMENSIONS, "applied_day_masks": { "type": "long", "index": False, @@ -63,7 +62,7 @@ def _metric_properties(): }, "country_code": {"type": "keyword"}, "content_language": {"type": "keyword"}, - **METRIC_PROPERTIES, + **_METRIC_PROPERTIES, }, } @@ -107,16 +106,17 @@ def _metric_properties(): }, } +_DATASET_MAPPINGS = { + "counter": MONTH_INDEX_MAPPINGS, + "analytics": ANALYTICS_INDEX_MAPPINGS, + "sources": SOURCE_INDEX_MAPPINGS, + "documents": DOCUMENT_INDEX_MAPPINGS, +} + def get_index_mappings(dataset): - mappings = { - "counter": MONTH_INDEX_MAPPINGS, - "analytics": ANALYTICS_INDEX_MAPPINGS, - "sources": SOURCE_INDEX_MAPPINGS, - "documents": DOCUMENT_INDEX_MAPPINGS, - } try: - return mappings[dataset] + return _DATASET_MAPPINGS[dataset] except KeyError as exc: raise ValueError(f"Unknown OpenSearch dataset: {dataset}.") from exc diff --git a/metrics/opensearch/painless.py b/metrics/opensearch/painless.py index e950f1b..737672a 100644 --- a/metrics/opensearch/painless.py +++ b/metrics/opensearch/painless.py @@ -1,7 +1,8 @@ from datetime import date ANNUAL_MASK_BITS = 63 -ANNUAL_MASK_BUCKETS = 6 +DAYS_PER_LEAP_YEAR = 366 +ANNUAL_MASK_BUCKETS = (DAYS_PER_LEAP_YEAR + ANNUAL_MASK_BITS - 1) // ANNUAL_MASK_BITS METRIC_FIELDS = ( "total_requests", "total_investigations", From 8fb132a6298e1ae0ce464fe98a101e8997933d1f Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 23:12:34 -0300 Subject: [PATCH 19/23] Implementa aliases anuais com rollover por tamanho --- metrics/opensearch/client.py | 251 ++++++++++++++++++++++++++++++----- metrics/opensearch/names.py | 51 ++++--- 2 files changed, 256 insertions(+), 46 deletions(-) diff --git a/metrics/opensearch/client.py b/metrics/opensearch/client.py index 5547c7b..675b678 100644 --- a/metrics/opensearch/client.py +++ b/metrics/opensearch/client.py @@ -1,13 +1,15 @@ import logging +from itertools import islice from django.conf import settings -from opensearchpy import NotFoundError, OpenSearch, helpers +from opensearchpy import NotFoundError, OpenSearch, RequestError, helpers -from metrics.opensearch.mappings import get_index_mappings, get_index_settings +from log_manager_config.choices import OpenSearchPartitionStrategy +from metrics.opensearch.mappings import get_index_settings from metrics.opensearch.names import ( - generate_analytics_index_name, - generate_month_index_name, - generate_physical_index_name, + generate_initial_index_name, + generate_rollover_index_name, + generate_yearly_write_alias, ) from metrics.opensearch.painless import ( build_idempotent_annual_day_increment_action, @@ -15,6 +17,8 @@ merge_metric_document, ) +DOCUMENT_INDEX_LOOKUP_BATCH_SIZE = 1000 + class OpenSearchUsageClient: def __init__(self, url=None, basic_auth=None, api_key=None, verify_certs=None): @@ -125,43 +129,151 @@ def create_alias_if_not_exists( if self.client.indices.exists_alias(name=alias_name): return - physical_name = generate_physical_index_name(alias_name) + physical_name = generate_initial_index_name(alias_name) + if not self.client.indices.exists(index=physical_name): + self._create_index_with_alias( + physical_name, + alias_name, + mappings, + primary_shards, + ) + if not self.client.indices.exists_alias(index=physical_name, name=alias_name): + self.client.indices.put_alias(index=physical_name, name=alias_name) + + def prepare_usage_index( + self, + alias_name, + mappings, + partition_strategy, + access_date, + primary_shards=1, + ): + if partition_strategy == OpenSearchPartitionStrategy.YEARLY: + write_alias = generate_yearly_write_alias(alias_name, access_date) + self._ensure_rollover_alias( + write_alias, + mappings, + primary_shards, + read_alias=alias_name, + ) + return write_alias + + if partition_strategy != OpenSearchPartitionStrategy.ROLLOVER: + raise ValueError( + f"Unsupported OpenSearch partition strategy: {partition_strategy}." + ) + + self._ensure_rollover_alias(alias_name, mappings, primary_shards) + return alias_name + + def _ensure_rollover_alias( + self, + alias_name, + mappings, + primary_shards, + read_alias=None, + ): + if self.client.indices.exists_alias(name=alias_name): + self._get_write_index(alias_name) + self._ensure_read_alias(alias_name, read_alias) + return + + physical_name = generate_rollover_index_name(alias_name) + alias_definitions = {alias_name: {"is_write_index": True}} + if read_alias and read_alias != alias_name: + alias_definitions[read_alias] = {} if not self.client.indices.exists(index=physical_name): + self._create_index_with_alias_definitions( + physical_name=physical_name, + alias_definitions=alias_definitions, + mappings=mappings, + primary_shards=primary_shards, + ) + if not self.client.indices.exists_alias(index=physical_name, name=alias_name): + self.client.indices.put_alias( + index=physical_name, + name=alias_name, + body={"is_write_index": True}, + ) + if read_alias and not self.client.indices.exists_alias( + index=physical_name, + name=read_alias, + ): + self.client.indices.put_alias(index=physical_name, name=read_alias) + + def _create_index_with_alias( + self, + physical_name, + alias_name, + mappings, + primary_shards, + alias_options=None, + ): + self._create_index_with_alias_definitions( + physical_name=physical_name, + alias_definitions={alias_name: alias_options or {}}, + mappings=mappings, + primary_shards=primary_shards, + ) + + def _create_index_with_alias_definitions( + self, + physical_name, + alias_definitions, + mappings, + primary_shards, + ): + try: self.client.indices.create( index=physical_name, body={ "settings": get_index_settings(primary_shards), "mappings": mappings, - "aliases": {alias_name: {}}, + "aliases": alias_definitions, }, ) - return + except RequestError: + if not self.client.indices.exists(index=physical_name): + raise - self.client.indices.put_alias(index=physical_name, name=alias_name) + def _ensure_read_alias(self, write_alias, read_alias): + if not read_alias or read_alias == write_alias: + return + for index_name in self.client.indices.get_alias(name=write_alias): + if not self.client.indices.exists_alias( + index=index_name, + name=read_alias, + ): + self.client.indices.put_alias(index=index_name, name=read_alias) + + def rollover_usage_index( + self, + alias_name, + mappings, + primary_shards=1, + read_alias=None, + ): + body = { + "conditions": { + "max_size": getattr( + settings, + "OPENSEARCH_ROLLOVER_MAX_SIZE", + "50gb", + ) + }, + "settings": get_index_settings(primary_shards), + "mappings": mappings, + } + if read_alias and read_alias != alias_name: + body["aliases"] = {read_alias: {}} - def ensure_usage_indexes(self, collection, access_date, index_prefix=None): - index_prefix = index_prefix or getattr( - settings, - "OPENSEARCH_INDEX_NAME", - "usage", - ) - month_index = generate_month_index_name(index_prefix, collection, access_date) - analytics_index = generate_analytics_index_name( - index_prefix, - collection, - access_date, + response = self.client.indices.rollover( + alias=alias_name, + body=body, ) - - self.create_alias_if_not_exists( - month_index, - get_index_mappings("counter"), - ) - self.create_alias_if_not_exists( - analytics_index, - get_index_mappings("analytics"), - ) - - return {"counter": month_index, "analytics": analytics_index} + if response.get("rolled_over"): + self._ensure_read_alias(alias_name, read_alias) + return response def index_documents(self, index_name, documents, ping_client=False): if ping_client and not self.ping(): @@ -198,6 +310,7 @@ def increment_document_items_for_day( document_items, access_day, annual=False, + resolve_existing_indexes=False, ping_client=False, ): if ping_client and not self.ping(): @@ -208,6 +321,14 @@ def increment_document_items_for_day( if annual else build_idempotent_day_increment_action ) + if resolve_existing_indexes: + return self._increment_items_in_existing_indexes( + index_name, + document_items, + access_day, + action_builder, + ) + succeeded, _failed = helpers.bulk( self.client, ( @@ -223,6 +344,74 @@ def increment_document_items_for_day( ) return succeeded + def _increment_items_in_existing_indexes( + self, + alias_name, + document_items, + access_day, + action_builder, + ): + write_index = self._get_write_index(alias_name) + succeeded = 0 + batch_size = min(self.bulk_chunk_size, DOCUMENT_INDEX_LOOKUP_BATCH_SIZE) + + while True: + batch = list(islice(document_items, batch_size)) + if not batch: + return succeeded + + existing_indexes = self._find_existing_document_indexes( + alias_name, + [doc_id for doc_id, _document in batch], + ) + batch_succeeded, _failed = helpers.bulk( + self.client, + ( + action_builder( + index_name=existing_indexes.get(doc_id, write_index), + doc_id=doc_id, + document=document, + access_day=access_day, + ) + for doc_id, document in batch + ), + chunk_size=self.bulk_chunk_size, + ) + succeeded += batch_succeeded + + def _get_write_index(self, alias_name): + indexes = self.client.indices.get_alias(name=alias_name) + write_indexes = [ + index_name + for index_name, index_data in indexes.items() + if index_data.get("aliases", {}).get(alias_name, {}).get("is_write_index") + ] + if len(write_indexes) == 1: + return write_indexes[0] + if len(indexes) == 1: + return next(iter(indexes)) + raise RuntimeError(f"Alias {alias_name} has no unique write index.") + + def _find_existing_document_indexes(self, alias_name, doc_ids): + response = self.client.search( + index=alias_name, + body={ + "size": len(doc_ids) * 2, + "_source": False, + "query": {"ids": {"values": doc_ids}}, + }, + ) + indexes = {} + for hit in response.get("hits", {}).get("hits", []): + doc_id = hit["_id"] + if doc_id in indexes and indexes[doc_id] != hit["_index"]: + raise RuntimeError( + f"Document {doc_id} exists in multiple indexes behind " + f"alias {alias_name}." + ) + indexes[doc_id] = hit["_index"] + return indexes + def delete_documents(self, index_name, doc_ids, ping_client=False): if ping_client and not self.ping(): return diff --git a/metrics/opensearch/names.py b/metrics/opensearch/names.py index f90e074..7a6d5c2 100644 --- a/metrics/opensearch/names.py +++ b/metrics/opensearch/names.py @@ -1,32 +1,53 @@ -def _validate_index_inputs(index_prefix, collection, date): - if not date or not isinstance(date, str): +from datetime import date as date_type + +INITIAL_INDEX_GENERATION = 1 + + +def _validate_access_date(access_date): + if not access_date or not isinstance(access_date, str): raise ValueError("Date must be a non-empty string in 'YYYY-MM-DD' format.") + try: + return date_type.fromisoformat(access_date) + except ValueError as exc: + raise ValueError("Date must be in 'YYYY-MM-DD' format.") from exc + + +def extract_access_year(date): + return str(_validate_access_date(date).year) + + +def _validate_index_inputs(index_prefix, collection): if not collection or not isinstance(collection, str): raise ValueError("Collection must be a non-empty string.") if not index_prefix or not isinstance(index_prefix, str): raise ValueError("Index prefix must be a non-empty string.") -def extract_access_year(date): - _validate_index_inputs("usage", "tmp", date) - return date.split("-")[0] +def generate_month_index_name(index_prefix, collection): + _validate_index_inputs(index_prefix, collection) + return f"{index_prefix}_monthly_{collection}" + +def generate_analytics_index_name(index_prefix, collection): + _validate_index_inputs(index_prefix, collection) + return f"{index_prefix}_yearly_analytics_{collection}" -def generate_month_index_name(index_prefix, collection, date): - _validate_index_inputs(index_prefix, collection, date) - return f"{index_prefix}_monthly_{collection}_{extract_access_year(date)}" +def generate_initial_index_name(alias_name): + return f"{alias_name}_{INITIAL_INDEX_GENERATION:06d}" -def generate_analytics_index_name(index_prefix, collection, date): - _validate_index_inputs(index_prefix, collection, date) - return f"{index_prefix}_yearly_analytics_{collection}_{extract_access_year(date)}" +def generate_rollover_index_name(alias_name): + return f"{alias_name}-{INITIAL_INDEX_GENERATION:06d}" -def generate_physical_index_name(alias_name): - return f"{alias_name}_000001" +def generate_yearly_write_alias(alias_name, access_date): + return f"{alias_name}_{extract_access_year(access_date)}" -def generate_metadata_alias(entity): + +def generate_metadata_alias(index_prefix, entity): + if not index_prefix or not isinstance(index_prefix, str): + raise ValueError("Index prefix must be a non-empty string.") if entity not in {"documents", "sources"}: raise ValueError("Metadata entity must be 'documents' or 'sources'.") - return f"usage_{entity}" + return f"{index_prefix}_{entity}" From 4071e689e57141ba236050cd38077ead0de8a246 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 23:12:46 -0300 Subject: [PATCH 20/23] =?UTF-8?q?Respeita=20o=20prefixo=20configurado=20no?= =?UTF-8?q?s=20=C3=ADndices=20de=20metadados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/metadata_sync.py | 6 +++++- metrics/tasks/metadata_sync.py | 4 ++-- metrics/tests/services/test_metadata_sync.py | 7 ++++++- 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/metrics/services/metadata_sync.py b/metrics/services/metadata_sync.py index c1e4092..bc20e9c 100644 --- a/metrics/services/metadata_sync.py +++ b/metrics/services/metadata_sync.py @@ -1,6 +1,7 @@ import logging from datetime import timedelta +from django.conf import settings from django.db import transaction from django.db.models import Q from django.utils import timezone @@ -42,7 +43,10 @@ def sync_metadata(entity=None, batch_size=DEFAULT_BATCH_SIZE): def _sync_entity(client, entity, batch_size): model, serializer, related = _entity_config(entity) state = _acquire_lease(entity) - alias = generate_metadata_alias(f"{entity}s") + alias = generate_metadata_alias( + settings.OPENSEARCH_INDEX_NAME, + f"{entity}s", + ) client.create_alias_if_not_exists(alias, get_index_mappings(f"{entity}s")) exported = 0 diff --git a/metrics/tasks/metadata_sync.py b/metrics/tasks/metadata_sync.py index 12b3dc1..dfc92a8 100644 --- a/metrics/tasks/metadata_sync.py +++ b/metrics/tasks/metadata_sync.py @@ -1,5 +1,5 @@ from config import celery_app -from metrics.services.metadata_sync import sync_metadata +from metrics.services.metadata_sync import DEFAULT_BATCH_SIZE, sync_metadata @celery_app.task( @@ -8,5 +8,5 @@ queue="load", timelimit=-1, ) -def task_sync_opensearch_metadata(self, entity=None, batch_size=1000): +def task_sync_opensearch_metadata(self, entity=None, batch_size=DEFAULT_BATCH_SIZE): return sync_metadata(entity=entity, batch_size=batch_size) diff --git a/metrics/tests/services/test_metadata_sync.py b/metrics/tests/services/test_metadata_sync.py index a8b7c4a..660d396 100644 --- a/metrics/tests/services/test_metadata_sync.py +++ b/metrics/tests/services/test_metadata_sync.py @@ -1,6 +1,6 @@ from unittest.mock import Mock, patch -from django.test import TestCase +from django.test import TestCase, override_settings from collection.models import Collection from document.models import Document @@ -64,6 +64,7 @@ def test_save_and_delete_create_transactional_outbox_entries(self): ) self.assertFalse(entry.payload["active"]) + @override_settings(OPENSEARCH_INDEX_NAME="custom_usage") @patch("metrics.services.metadata_sync.OpenSearchUsageClient") def test_sync_advances_cursor_and_clears_exported_outbox(self, client_class): client = Mock() @@ -83,3 +84,7 @@ def test_sync_advances_cursor_and_clears_exported_outbox(self, client_class): ).exists() ) client.create_alias_if_not_exists.assert_called_once() + self.assertEqual( + client.create_alias_if_not_exists.call_args.args[0], + "custom_usage_sources", + ) From 22f915c0ed0e91b21be4571c58a0695273de645c Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 23:12:54 -0300 Subject: [PATCH 21/23] =?UTF-8?q?Testa=20aliases,=20rollover=20e=20concorr?= =?UTF-8?q?=C3=AAncia=20na=20cria=C3=A7=C3=A3o=20de=20=C3=ADndices?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/tests/opensearch/test_client.py | 178 ++++++++++++++++++++++++ metrics/tests/opensearch/test_names.py | 34 +++-- 2 files changed, 200 insertions(+), 12 deletions(-) diff --git a/metrics/tests/opensearch/test_client.py b/metrics/tests/opensearch/test_client.py index 651e1ce..7a9a374 100644 --- a/metrics/tests/opensearch/test_client.py +++ b/metrics/tests/opensearch/test_client.py @@ -1,6 +1,7 @@ from unittest.mock import Mock, patch from django.test import SimpleTestCase, override_settings +from opensearchpy import RequestError from metrics.opensearch.client import OpenSearchUsageClient from metrics.opensearch.mappings import ( @@ -43,6 +44,183 @@ def test_creates_physical_index_with_alias_and_compression(self, get_client): }, ) + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_yearly_partition_uses_validated_access_year(self, get_client): + raw_client = Mock() + raw_client.indices.exists_alias.return_value = False + raw_client.indices.exists.return_value = False + get_client.return_value = raw_client + client = OpenSearchUsageClient(url="https://example.org:9200") + + write_index = client.prepare_usage_index( + alias_name="usage_monthly_scl", + mappings=MONTH_INDEX_MAPPINGS, + partition_strategy="yearly", + access_date="2025-12-31", + ) + + assert write_index == "usage_monthly_scl_2025" + raw_client.indices.create.assert_called_once_with( + index="usage_monthly_scl_2025-000001", + body={ + "settings": get_index_settings(1), + "mappings": MONTH_INDEX_MAPPINGS, + "aliases": { + "usage_monthly_scl_2025": {"is_write_index": True}, + "usage_monthly_scl": {}, + }, + }, + ) + + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_yearly_rollover_keeps_global_read_alias(self, get_client): + raw_client = Mock() + raw_client.indices.rollover.return_value = { + "rolled_over": False, + } + get_client.return_value = raw_client + client = OpenSearchUsageClient(url="https://example.org:9200") + + client.rollover_usage_index( + "usage_monthly_scl_2026", + MONTH_INDEX_MAPPINGS, + read_alias="usage_monthly_scl", + ) + + raw_client.indices.rollover.assert_called_once_with( + alias="usage_monthly_scl_2026", + body={ + "conditions": {"max_size": "50gb"}, + "settings": get_index_settings(1), + "mappings": MONTH_INDEX_MAPPINGS, + "aliases": {"usage_monthly_scl": {}}, + }, + ) + + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_continuous_partition_creates_a_write_alias(self, get_client): + raw_client = Mock() + raw_client.indices.exists_alias.return_value = False + raw_client.indices.exists.return_value = False + get_client.return_value = raw_client + client = OpenSearchUsageClient(url="https://example.org:9200") + + write_index = client.prepare_usage_index( + alias_name="usage_monthly_books", + mappings=MONTH_INDEX_MAPPINGS, + partition_strategy="rollover", + access_date="2026-01-01", + ) + + assert write_index == "usage_monthly_books" + raw_client.indices.create.assert_called_once_with( + index="usage_monthly_books-000001", + body={ + "settings": get_index_settings(1), + "mappings": MONTH_INDEX_MAPPINGS, + "aliases": { + "usage_monthly_books": {"is_write_index": True}, + }, + }, + ) + + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_concurrent_initial_index_creation_is_idempotent(self, get_client): + raw_client = Mock() + raw_client.indices.exists_alias.return_value = False + raw_client.indices.exists.side_effect = [False, True] + raw_client.indices.create.side_effect = RequestError( + 400, + "resource_already_exists_exception", + {}, + ) + get_client.return_value = raw_client + client = OpenSearchUsageClient(url="https://example.org:9200") + + write_index = client.prepare_usage_index( + alias_name="usage_monthly_books", + mappings=MONTH_INDEX_MAPPINGS, + partition_strategy="rollover", + access_date="2026-01-01", + ) + + assert write_index == "usage_monthly_books" + + @override_settings(OPENSEARCH_ROLLOVER_MAX_SIZE="50gb") + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_rollover_preserves_mappings_and_settings(self, get_client): + raw_client = Mock() + get_client.return_value = raw_client + client = OpenSearchUsageClient(url="https://example.org:9200") + + client.rollover_usage_index( + "usage_monthly_books", + MONTH_INDEX_MAPPINGS, + ) + + raw_client.indices.rollover.assert_called_once_with( + alias="usage_monthly_books", + body={ + "conditions": {"max_size": "50gb"}, + "settings": get_index_settings(1), + "mappings": MONTH_INDEX_MAPPINGS, + }, + ) + + @override_settings(OPENSEARCH_BULK_CHUNK_SIZE=2) + @patch("metrics.opensearch.client.helpers.bulk") + @patch.object(OpenSearchUsageClient, "get_opensearch_client") + def test_continuous_updates_follow_existing_backing_index( + self, + get_client, + bulk, + ): + raw_client = Mock() + raw_client.indices.get_alias.return_value = { + "usage_monthly_books-000001": { + "aliases": { + "usage_monthly_books": {"is_write_index": False}, + } + }, + "usage_monthly_books-000002": { + "aliases": { + "usage_monthly_books": {"is_write_index": True}, + } + }, + } + raw_client.search.return_value = { + "hits": { + "hits": [ + { + "_id": "existing", + "_index": "usage_monthly_books-000001", + } + ] + } + } + get_client.return_value = raw_client + bulk.return_value = (2, 0) + client = OpenSearchUsageClient(url="https://example.org:9200") + + succeeded = client.increment_document_items_for_day( + index_name="usage_monthly_books", + document_items=iter( + [ + ("existing", {"total_requests": 1}), + ("new", {"total_requests": 1}), + ] + ), + access_day="2026-01-01", + resolve_existing_indexes=True, + ) + + actions = list(bulk.call_args.args[1]) + assert succeeded == 2 + assert [action["_index"] for action in actions] == [ + "usage_monthly_books-000001", + "usage_monthly_books-000002", + ] + def test_mappings_are_explicit_and_separate_metadata_from_facts(self): assert "source_key" in MONTH_INDEX_MAPPINGS["properties"] assert "document_key" in MONTH_INDEX_MAPPINGS["properties"] diff --git a/metrics/tests/opensearch/test_names.py b/metrics/tests/opensearch/test_names.py index 8898204..f180f71 100644 --- a/metrics/tests/opensearch/test_names.py +++ b/metrics/tests/opensearch/test_names.py @@ -1,23 +1,33 @@ from metrics.opensearch.names import ( generate_analytics_index_name, + generate_initial_index_name, generate_metadata_alias, generate_month_index_name, - generate_physical_index_name, + generate_rollover_index_name, + generate_yearly_write_alias, ) -def test_metric_aliases_are_partitioned_by_collection_and_year(): - assert generate_month_index_name("usage", "scl", "2026-08-20") == ( - "usage_monthly_scl_2026" - ) - assert generate_analytics_index_name("usage", "scl", "2026-08-20") == ( - "usage_yearly_analytics_scl_2026" +def test_metric_aliases_are_stable_per_collection(): + assert generate_month_index_name("usage", "scl") == "usage_monthly_scl" + assert generate_analytics_index_name("usage", "scl") == ( + "usage_yearly_analytics_scl" ) -def test_physical_index_and_metadata_alias_names_are_explicit(): - assert generate_physical_index_name("usage_monthly_scl_2026") == ( - "usage_monthly_scl_2026_000001" +def test_physical_index_names_express_the_partition_strategy(): + assert generate_rollover_index_name("usage_monthly_books") == ( + "usage_monthly_books-000001" + ) + assert generate_yearly_write_alias("usage_monthly_scl", "2026-08-20") == ( + "usage_monthly_scl_2026" ) - assert generate_metadata_alias("documents") == "usage_documents" - assert generate_metadata_alias("sources") == "usage_sources" + + +def test_non_rollover_alias_uses_a_neutral_physical_name(): + assert generate_initial_index_name("usage_documents") == ("usage_documents_000001") + + +def test_metadata_alias_names_are_explicit(): + assert generate_metadata_alias("usage", "documents") == "usage_documents" + assert generate_metadata_alias("custom", "sources") == "custom_sources" From dbaf57947dbe53428236702aaece37a5770961fc Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 23:13:02 -0300 Subject: [PATCH 22/23] =?UTF-8?q?Roteia=20exporta=C3=A7=C3=B5es=20pelo=20a?= =?UTF-8?q?lias=20anual=20de=20escrita?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- metrics/services/export.py | 36 +++++--- .../services/test_daily_metric_exports.py | 88 ++++++++++++++++--- 2 files changed, 102 insertions(+), 22 deletions(-) diff --git a/metrics/services/export.py b/metrics/services/export.py index 8430d5a..1f6ac1d 100644 --- a/metrics/services/export.py +++ b/metrics/services/export.py @@ -4,6 +4,7 @@ from django.conf import settings +from log_manager_config.choices import OpenSearchPartitionStrategy from metrics.opensearch.mappings import get_index_mappings from metrics.opensearch.names import ( generate_analytics_index_name, @@ -71,28 +72,39 @@ def _sync_documents_group( index_name = generate_month_index_name( index_prefix=index_prefix, collection=collection_code, - date=index_date, ) else: index_name = generate_analytics_index_name( index_prefix=index_prefix, collection=collection_code, - date=index_date, ) - search_client.create_alias_if_not_exists( + config = getattr(collection, "log_manager_config", None) + primary_shards = getattr(config, "opensearch_primary_shards", 1) + partition_strategy = getattr( + config, + "opensearch_partition_strategy", + OpenSearchPartitionStrategy.ROLLOVER, + ) + mappings = get_index_mappings(dataset) + write_index = search_client.prepare_usage_index( alias_name=index_name, - mappings=get_index_mappings(dataset), - primary_shards=_get_primary_shards(collection), + mappings=mappings, + partition_strategy=partition_strategy, + access_date=index_date, + primary_shards=primary_shards, ) - return search_client.increment_document_items_for_day( - index_name=index_name, + exported = search_client.increment_document_items_for_day( + index_name=write_index, document_items=chain((first_item,), document_items), access_day=access_day, annual=dataset == "analytics", + resolve_existing_indexes=True, ) - - -def _get_primary_shards(collection): - config = getattr(collection, "log_manager_config", None) - return getattr(config, "opensearch_primary_shards", 1) + search_client.rollover_usage_index( + alias_name=write_index, + mappings=mappings, + primary_shards=primary_shards, + read_alias=index_name if write_index != index_name else None, + ) + return exported diff --git a/metrics/tests/services/test_daily_metric_exports.py b/metrics/tests/services/test_daily_metric_exports.py index e61c40f..58a1a50 100644 --- a/metrics/tests/services/test_daily_metric_exports.py +++ b/metrics/tests/services/test_daily_metric_exports.py @@ -24,7 +24,10 @@ def setUp(self): pk=1, collection=SimpleNamespace( acron3="scl", - log_manager_config=SimpleNamespace(opensearch_primary_shards=3), + log_manager_config=SimpleNamespace( + opensearch_primary_shards=1, + opensearch_partition_strategy="yearly", + ), ), access_date=date(2026, 8, 25), storage_path=self.storage_path.as_posix(), @@ -60,11 +63,27 @@ def _write_payload(self): def test_export_streams_each_dataset_as_document_items(self): self._write_payload() search_client = Mock() + search_client.prepare_usage_index.side_effect = [ + "usage_monthly_scl_2026", + "usage_yearly_analytics_scl_2026", + ] exported_groups = [] - def consume_items(index_name, document_items, access_day, annual=False): + def consume_items( + index_name, + document_items, + access_day, + annual=False, + resolve_existing_indexes=False, + ): exported_groups.append( - (index_name, list(document_items), access_day, annual) + ( + index_name, + list(document_items), + access_day, + annual, + resolve_existing_indexes, + ) ) return 1 @@ -80,6 +99,7 @@ def consume_items(index_name, document_items, access_day, annual=False): [("month-doc", {"month": "2026-08"})], "2026-08-25", False, + True, ), ( "usage_yearly_analytics_scl_2026", @@ -91,15 +111,32 @@ def consume_items(index_name, document_items, access_day, annual=False): ], "2026-08-25", True, + True, ), ], ) self.assertEqual( [ call.kwargs["primary_shards"] - for call in search_client.create_alias_if_not_exists.call_args_list + for call in search_client.prepare_usage_index.call_args_list + ], + [1, 1], + ) + self.assertEqual( + [ + ( + call.kwargs["alias_name"], + call.kwargs["read_alias"], + ) + for call in search_client.rollover_usage_index.call_args_list + ], + [ + ("usage_monthly_scl_2026", "usage_monthly_scl"), + ( + "usage_yearly_analytics_scl_2026", + "usage_yearly_analytics_scl", + ), ], - [3, 3], ) def test_retry_after_partial_export_reuses_same_payload_and_access_day(self): @@ -114,11 +151,11 @@ def test_retry_after_partial_export_reuses_same_payload_and_access_day(self): export_daily_metric_payload(first_client, self.job) second_client = Mock() - second_client.increment_document_items_for_day.side_effect = ( - lambda index_name, document_items, access_day, annual=False: len( - list(document_items) - ) - ) + + def count_items(**kwargs): + return len(list(kwargs["document_items"])) + + second_client.increment_document_items_for_day.side_effect = count_items export_daily_metric_payload(second_client, self.job) self.assertEqual( @@ -129,6 +166,37 @@ def test_retry_after_partial_export_reuses_same_payload_and_access_day(self): ["2026-08-25", "2026-08-25"], ) + def test_continuous_collection_routes_existing_documents_before_rollover(self): + self._write_payload() + self.job.collection.log_manager_config.opensearch_partition_strategy = ( + "rollover" + ) + search_client = Mock() + search_client.prepare_usage_index.side_effect = [ + "usage_monthly_scl", + "usage_yearly_analytics_scl", + ] + search_client.increment_document_items_for_day.side_effect = ( + lambda **kwargs: len(list(kwargs["document_items"])) + ) + + export_daily_metric_payload(search_client, self.job) + + self.assertEqual( + [ + call.kwargs["resolve_existing_indexes"] + for call in search_client.increment_document_items_for_day.call_args_list + ], + [True, True], + ) + self.assertEqual( + [ + call.kwargs["alias_name"] + for call in search_client.rollover_usage_index.call_args_list + ], + ["usage_monthly_scl", "usage_yearly_analytics_scl"], + ) + @patch("metrics.services.daily_metric_exports.fetch_required_resources") @patch("metrics.services.daily_metric_exports.build_daily_metric_job_payload") def test_resume_uses_persisted_payload_without_parsing( From cb0945b4a3ee28b20bd55907d8e5aa11ffd213b9 Mon Sep 17 00:00:00 2001 From: Rafael Damaceno Date: Thu, 10 Sep 2026 22:19:51 -0300 Subject: [PATCH 23/23] =?UTF-8?q?Atualiza=20a=20vers=C3=A3o=20para=202.4.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index 00355e2..197c4d5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3.7 +2.4.0