diff --git a/article/models.py b/article/models.py index 7ba19be59..d1f6d9c92 100755 --- a/article/models.py +++ b/article/models.py @@ -2099,6 +2099,7 @@ def request_pid(self, user, force_update, auto_solve_pid_conflict): force_update=force_update, is_published=True, auto_solve_pid_conflict=auto_solve_pid_conflict, + collection_acron=self.collection and self.collection.acron3, ) # Obtém a primeira resposta (assumindo apenas uma) diff --git a/collection/migrations/0008_collection_network_classification_and_more.py b/collection/migrations/0008_collection_network_classification_and_more.py index 02335e743..a2a6c8782 100644 --- a/collection/migrations/0008_collection_network_classification_and_more.py +++ b/collection/migrations/0008_collection_network_classification_and_more.py @@ -1,4 +1,4 @@ -# Generated by Django 5.2.7 on 2026-10-02 19:03 +# Generated by Django 5.2.7 on 2026-09-30 22:11 import collection.models from django.db import migrations, models diff --git a/collection/models.py b/collection/models.py index 98cce4dda..a71c7e920 100755 --- a/collection/models.py +++ b/collection/models.py @@ -2,8 +2,11 @@ from functools import cached_property from django import forms +from django.conf import settings from django.contrib.postgres.fields import ArrayField +from django.core.cache import cache from django.db import models +from django.utils import timezone from django.utils.translation import gettext_lazy as _ from modelcluster.fields import ParentalKey from modelcluster.models import ClusterableModel @@ -44,6 +47,26 @@ def formfield(self, **kwargs): return super(ArrayField, self).formfield(**defaults) +ARTICLEMETA_COLLECTIONS_URL = ( + "https://articlemeta.scielo.org/api/v1/collection/identifiers/" +) + +ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY = "collection.ensure_network_classification" +# intervalo mínimo (segundos) entre consultas ao articlemeta feitas por +# Collection.ensure_network_classification +ENSURE_NETWORK_CLASSIFICATION_INTERVAL = 600 + + +def normalize_network_classification(network_classification): + """ + Retorna network_classification como lista ou None + Ex.: "scielonetwork" -> ["scielonetwork"] + """ + if isinstance(network_classification, str): + network_classification = [network_classification] + return [item for item in network_classification or [] if item] or None + + class CollectionName(TextWithLang): collection = ParentalKey( "Collection", @@ -257,7 +280,7 @@ def __str__(self): def load(cls, user, collections_data=None, verify=False): if not collections_data: collections_data = fetch_data( - "https://articlemeta.scielo.org/api/v1/collection/identifiers/", + ARTICLEMETA_COLLECTIONS_URL, json=True, verify=verify, ) @@ -279,6 +302,88 @@ def load(cls, user, collections_data=None, verify=False): network_classification=collection_data.get("network_classification"), ) + @classmethod + def get_collections_without_network_classification(cls): + return cls.objects.filter( + models.Q(network_classification__isnull=True) + | models.Q(network_classification=[]) + ) + + @classmethod + def ensure_network_classification(cls, user): + """ + Garante network_classification das coleções antes de seu uso para + identificar a coleção principal (ex.: PidProviderXML.register de + artigos publicados em mais de uma coleção). + + Consulta o articlemeta somente se há coleções sem esse dado e, no + máximo, uma vez a cada ENSURE_NETWORK_CLASSIFICATION_INTERVAL + segundos, evitando uma consulta por registro quando o articlemeta + está indisponível ou não tem a coleção. + Falhas são registradas no log e não interrompem quem chama. + """ + if not getattr(settings, "COLLECTION_ENSURE_NETWORK_CLASSIFICATION", True): + return None + try: + if not cls.get_collections_without_network_classification().exists(): + return None + # cache.add é atômico: somente um processo consulta o articlemeta + if not cache.add( + ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY, + True, + ENSURE_NETWORK_CLASSIFICATION_INTERVAL, + ): + return None + return cls.complete_network_classification(user) + except Exception as e: + logging.exception(f"Collection.ensure_network_classification: {e}") + return None + + @classmethod + def complete_network_classification(cls, user, collections_data=None, verify=False): + """ + Preenche em lote network_classification das coleções que estão + sem esse dado, a partir dos dados do articlemeta. + Coleções já preenchidas não são alteradas. + """ + queryset = cls.get_collections_without_network_classification() + result = {"updated": [], "not_found": []} + if not queryset.exists(): + return result + + if not collections_data: + collections_data = fetch_data( + ARTICLEMETA_COLLECTIONS_URL, + json=True, + verify=verify, + ) + network_classification_by_acron = { + item.get("acron"): normalize_network_classification( + item.get("network_classification") + ) + for item in collections_data + } + + now = timezone.now() + items = [] + for obj in queryset: + network_classification = network_classification_by_acron.get(obj.acron3) + if not network_classification: + result["not_found"].append(obj.acron3) + continue + obj.network_classification = network_classification + obj.updated_by = user + # bulk_update não aplica auto_now + obj.updated = now + items.append(obj) + result["updated"].append(obj.acron3) + + cls.objects.bulk_update( + items, ["network_classification", "updated_by", "updated"] + ) + logging.info(f"Collection.complete_network_classification: {result}") + return result + @classmethod def get(cls, acron3): return cls.objects.get(acron3=acron3) @@ -330,9 +435,9 @@ def create_or_update( obj.has_analytics = has_analytics obj.collection_type = collection_type obj.is_active = is_active - if isinstance(network_classification, str): - network_classification = [network_classification] - obj.network_classification = network_classification or None + obj.network_classification = normalize_network_classification( + network_classification + ) obj.save() for language in names or {}: lang = Language.get_or_create(code2=language, creator=user) @@ -369,6 +474,18 @@ def get_name_for_language(self, lang_code=None): self.collection_name.first().text if self.collection_name.exists() else "" ) + @classmethod + def get_national_journal_collections(cls): + """ + Retorna as coleções cuja classificação de rede + é exclusivamente scielonetwork + """ + return cls.objects.filter(network_classification=["scielonetwork"]) + + @property + def is_national_journal_collection(self): + return self.network_classification == ["scielonetwork"] + @classmethod def get_acronyms(cls, collection_acron_list): queryset = cls.objects diff --git a/collection/scripts/complete_network_classification.py b/collection/scripts/complete_network_classification.py new file mode 100644 index 000000000..9bb63e50a --- /dev/null +++ b/collection/scripts/complete_network_classification.py @@ -0,0 +1,7 @@ +from collection import tasks + + +def run(username): + tasks.task_complete_network_classification.apply_async( + kwargs={"username": username} + ) diff --git a/collection/tasks.py b/collection/tasks.py index e5244043e..beff260e0 100644 --- a/collection/tasks.py +++ b/collection/tasks.py @@ -7,12 +7,24 @@ User = get_user_model() +def _get_user(user_id=None, username=None): + if user_id: + return User.objects.get(pk=user_id) + if username: + return User.objects.get(username=username) + raise ValueError("user_id or username is required") + + @celery_app.task(bind=True) def task_load_collections(self, user_id=None, username=None): - if user_id: - user = User.objects.get(pk=user_id) - elif username: - user = User.objects.get(username=username) - else: - raise ValueError("user_id or username is required") + user = _get_user(user_id, username) Collection.load(user) + + +@celery_app.task(bind=True) +def task_complete_network_classification(self, user_id=None, username=None): + """ + Preenche network_classification das coleções que estão sem esse dado + """ + user = _get_user(user_id, username) + return Collection.complete_network_classification(user) diff --git a/collection/tests.py b/collection/tests.py deleted file mode 100755 index 7ce503c2d..000000000 --- a/collection/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/collection/tests/__init__.py b/collection/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/collection/tests/test_complete_network_classification.py b/collection/tests/test_complete_network_classification.py new file mode 100755 index 000000000..f0bd15a08 --- /dev/null +++ b/collection/tests/test_complete_network_classification.py @@ -0,0 +1,85 @@ +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.test import TestCase + +from collection.models import Collection + +User = get_user_model() + + +class CompleteNetworkClassificationTest(TestCase): + def setUp(self): + self.user = User.objects.create(username="tester") + self.collections_data = [ + {"acron": "scl", "network_classification": ["scielonetwork"]}, + {"acron": "spa", "network_classification": ["scielonetwork", "thematic"]}, + {"acron": "psi", "network_classification": "thematic"}, + {"acron": "pef", "network_classification": ["independent"]}, + ] + + def test_completes_only_collections_without_network_classification(self): + Collection.objects.create(acron3="scl", creator=self.user) + Collection.objects.create( + acron3="spa", network_classification=[], creator=self.user + ) + Collection.objects.create(acron3="psi", creator=self.user) + Collection.objects.create( + acron3="pef", network_classification=["thematic"], creator=self.user + ) + + result = Collection.complete_network_classification( + self.user, collections_data=self.collections_data + ) + + self.assertEqual(sorted(result["updated"]), ["psi", "scl", "spa"]) + self.assertEqual(result["not_found"], []) + self.assertEqual( + Collection.objects.get(acron3="scl").network_classification, + ["scielonetwork"], + ) + self.assertEqual( + Collection.objects.get(acron3="spa").network_classification, + ["scielonetwork", "thematic"], + ) + self.assertEqual( + Collection.objects.get(acron3="psi").network_classification, + ["thematic"], + ) + # já preenchida: mantém o valor atual + self.assertEqual( + Collection.objects.get(acron3="pef").network_classification, + ["thematic"], + ) + self.assertEqual(Collection.objects.get(acron3="scl").updated_by, self.user) + + def test_reports_collections_absent_from_source(self): + Collection.objects.create(acron3="xyz", creator=self.user) + + result = Collection.complete_network_classification( + self.user, collections_data=self.collections_data + ) + + self.assertEqual(result, {"updated": [], "not_found": ["xyz"]}) + self.assertIsNone(Collection.objects.get(acron3="xyz").network_classification) + + @patch("collection.models.fetch_data") + def test_does_not_fetch_when_all_collections_are_complete(self, mock_fetch_data): + Collection.objects.create( + acron3="scl", network_classification=["scielonetwork"], creator=self.user + ) + + result = Collection.complete_network_classification(self.user) + + self.assertEqual(result, {"updated": [], "not_found": []}) + mock_fetch_data.assert_not_called() + + @patch("collection.models.fetch_data") + def test_fetches_articlemeta_when_data_is_not_given(self, mock_fetch_data): + mock_fetch_data.return_value = self.collections_data + Collection.objects.create(acron3="scl", creator=self.user) + + result = Collection.complete_network_classification(self.user) + + self.assertEqual(result["updated"], ["scl"]) + mock_fetch_data.assert_called_once() diff --git a/collection/tests/test_ensure_network_classification.py b/collection/tests/test_ensure_network_classification.py new file mode 100644 index 000000000..684d60420 --- /dev/null +++ b/collection/tests/test_ensure_network_classification.py @@ -0,0 +1,73 @@ +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.core.cache import cache +from django.test import TestCase, override_settings + +from collection.models import ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY, Collection + +User = get_user_model() + + +@override_settings(COLLECTION_ENSURE_NETWORK_CLASSIFICATION=True) +class EnsureNetworkClassificationTest(TestCase): + def setUp(self): + self.user = User.objects.create(username="tester") + cache.delete(ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY) + self.addCleanup(cache.delete, ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY) + patcher = patch( + "collection.models.fetch_data", + return_value=[ + {"acron": "scl", "network_classification": ["scielonetwork"]}, + ], + ) + self.mock_fetch_data = patcher.start() + self.addCleanup(patcher.stop) + + def test_completes_collections_without_network_classification(self): + Collection.objects.create(acron3="scl", creator=self.user) + + result = Collection.ensure_network_classification(self.user) + + self.assertEqual(result, {"updated": ["scl"], "not_found": []}) + self.assertTrue( + Collection.objects.get(acron3="scl").is_national_journal_collection + ) + self.mock_fetch_data.assert_called_once() + + def test_does_not_fetch_when_all_collections_are_complete(self): + Collection.objects.create( + acron3="scl", network_classification=["scielonetwork"], creator=self.user + ) + + self.assertIsNone(Collection.ensure_network_classification(self.user)) + self.mock_fetch_data.assert_not_called() + + def test_fetches_at_most_once_per_interval(self): + # coleção ausente no articlemeta continua sem network_classification + Collection.objects.create(acron3="xyz", creator=self.user) + + result = Collection.ensure_network_classification(self.user) + self.assertEqual(result, {"updated": [], "not_found": ["xyz"]}) + self.assertIsNone(Collection.ensure_network_classification(self.user)) + self.mock_fetch_data.assert_called_once() + + # após o intervalo, consulta novamente + cache.delete(ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY) + Collection.ensure_network_classification(self.user) + self.assertEqual(self.mock_fetch_data.call_count, 2) + + def test_does_not_raise_when_articlemeta_fails(self): + self.mock_fetch_data.side_effect = Exception("articlemeta indisponível") + Collection.objects.create(acron3="scl", creator=self.user) + + self.assertIsNone(Collection.ensure_network_classification(self.user)) + self.assertIsNone(Collection.objects.get(acron3="scl").network_classification) + + @override_settings(COLLECTION_ENSURE_NETWORK_CLASSIFICATION=False) + def test_does_nothing_when_disabled(self): + Collection.objects.create(acron3="scl", creator=self.user) + + self.assertIsNone(Collection.ensure_network_classification(self.user)) + self.mock_fetch_data.assert_not_called() + self.assertIsNone(Collection.objects.get(acron3="scl").network_classification) diff --git a/collection/tests/test_normalize_network_classification.py b/collection/tests/test_normalize_network_classification.py new file mode 100644 index 000000000..a7ae97cc6 --- /dev/null +++ b/collection/tests/test_normalize_network_classification.py @@ -0,0 +1,29 @@ +from django.test import SimpleTestCase + +from collection.models import normalize_network_classification + + +class NormalizeNetworkClassificationTest(SimpleTestCase): + def test_str_becomes_list(self): + self.assertEqual( + normalize_network_classification("scielonetwork"), ["scielonetwork"] + ) + + def test_list_is_kept(self): + self.assertEqual( + normalize_network_classification(["scielonetwork", "thematic"]), + ["scielonetwork", "thematic"], + ) + + def test_tuple_becomes_list(self): + self.assertEqual(normalize_network_classification(("thematic",)), ["thematic"]) + + def test_empty_values_become_none(self): + for value in (None, "", [], (), [""], [None]): + with self.subTest(value=value): + self.assertIsNone(normalize_network_classification(value)) + + def test_empty_items_are_removed(self): + self.assertEqual( + normalize_network_classification(["", "thematic", None]), ["thematic"] + ) diff --git a/collection/tests/test_tasks.py b/collection/tests/test_tasks.py new file mode 100644 index 000000000..28d93d5b1 --- /dev/null +++ b/collection/tests/test_tasks.py @@ -0,0 +1,33 @@ +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.test import TestCase + +from collection import tasks + +User = get_user_model() + + +@patch("collection.tasks.Collection.complete_network_classification") +class TaskCompleteNetworkClassificationTest(TestCase): + def setUp(self): + self.user = User.objects.create(username="tester") + + def test_runs_with_username(self, mock_complete): + mock_complete.return_value = {"updated": ["scl"], "not_found": []} + + result = tasks.task_complete_network_classification(username="tester") + + mock_complete.assert_called_once_with(self.user) + self.assertEqual(result, {"updated": ["scl"], "not_found": []}) + + def test_runs_with_user_id(self, mock_complete): + tasks.task_complete_network_classification(user_id=self.user.pk) + + mock_complete.assert_called_once_with(self.user) + + def test_requires_user(self, mock_complete): + with self.assertRaises(ValueError): + tasks.task_complete_network_classification() + + mock_complete.assert_not_called() diff --git a/collection/tests/test_wagtail_hooks.py b/collection/tests/test_wagtail_hooks.py new file mode 100644 index 000000000..82e85ceeb --- /dev/null +++ b/collection/tests/test_wagtail_hooks.py @@ -0,0 +1,53 @@ +from django.contrib.auth import get_user_model +from django.test import TestCase + +from collection.models import Collection +from collection.wagtail_hooks import CollectionFilterSet + +User = get_user_model() + + +class CollectionFilterSetTest(TestCase): + def setUp(self): + user = User.objects.create(username="tester") + Collection.objects.create( + acron3="scl", network_classification=["scielonetwork"], creator=user + ) + Collection.objects.create( + acron3="spa", + network_classification=["scielonetwork", "thematic"], + creator=user, + ) + Collection.objects.create( + acron3="psi", network_classification=["thematic"], creator=user + ) + Collection.objects.create(acron3="xyz", creator=user) + + def filter_acronyms(self, data): + filterset = CollectionFilterSet(data, queryset=Collection.objects.all()) + self.assertTrue(filterset.is_valid(), filterset.errors) + return sorted(filterset.qs.values_list("acron3", flat=True)) + + def test_without_filter_returns_all(self): + self.assertEqual(self.filter_acronyms({}), ["psi", "scl", "spa", "xyz"]) + + def test_filter_by_one_network_classification(self): + self.assertEqual( + self.filter_acronyms({"network_classification": ["thematic"]}), + ["psi", "spa"], + ) + + def test_filter_by_any_of_network_classifications(self): + self.assertEqual( + self.filter_acronyms( + {"network_classification": ["scielonetwork", "thematic"]} + ), + ["psi", "scl", "spa"], + ) + + def test_invalid_choice_is_rejected(self): + filterset = CollectionFilterSet( + {"network_classification": ["unknown"]}, + queryset=Collection.objects.all(), + ) + self.assertFalse(filterset.is_valid()) diff --git a/collection/wagtail_hooks.py b/collection/wagtail_hooks.py index 4fb0c1db1..9f2eb2654 100644 --- a/collection/wagtail_hooks.py +++ b/collection/wagtail_hooks.py @@ -1,8 +1,12 @@ +import django_filters +from django import forms from django.http import HttpResponseRedirect from django.utils.translation import gettext_lazy as _ +from wagtail.admin.filters import WagtailFilterSet from wagtail.snippets.models import register_snippet from wagtail.snippets.views.snippets import CreateView, SnippetViewSet +from . import choices from .models import Collection from config.menu import get_menu_order @@ -13,6 +17,33 @@ def form_valid(self, form): return HttpResponseRedirect(self.get_success_url()) +class CollectionFilterSet(WagtailFilterSet): + # django-filter não gera filtro automaticamente para ArrayField + network_classification = django_filters.MultipleChoiceFilter( + label=_("Network classification"), + choices=choices.NETWORK_CLASSIFICATION, + widget=forms.CheckboxSelectMultiple, + method="filter_network_classification", + ) + + class Meta: + model = Collection + fields = [ + "platform_status", + "network_classification", + "status", + "collection_type", + "is_active", + "has_analytics", + ] + + def filter_network_classification(self, queryset, name, value): + if not value: + return queryset + # coleções que tenham ao menos uma das classificações selecionadas + return queryset.filter(**{f"{name}__overlap": value}) + + @register_snippet class CollectionAdmin(SnippetViewSet): model = Collection @@ -26,18 +57,13 @@ class CollectionAdmin(SnippetViewSet): "main_name", "acron3", "platform_status", + "network_classification", "status", "collection_type", "is_active", "updated", ) - list_filter = ( - "platform_status", - "status", - "collection_type", - "is_active", - "has_analytics", - ) + filterset_class = CollectionFilterSet search_fields = ( "acron3", "acron2", @@ -51,6 +77,8 @@ class CollectionAdmin(SnippetViewSet): "code", "domain", "main_name", + "platform_status", + "network_classification", "status", "has_analytics", "collection_type", diff --git a/config/settings/base.py b/config/settings/base.py index a5c2c7fd6..ff76189b5 100644 --- a/config/settings/base.py +++ b/config/settings/base.py @@ -731,3 +731,9 @@ # Score mínimo (percentual_score) para considerar um candidato de PidProviderXML como aprovado PID_PROVIDER_MIN_RATE = env.float("PID_PROVIDER_MIN_RATE", default=0.65) + +# Completa automaticamente (articlemeta) network_classification das coleções +# antes de PidProviderXML.register (ver Collection.ensure_network_classification) +COLLECTION_ENSURE_NETWORK_CLASSIFICATION = env.bool( + "COLLECTION_ENSURE_NETWORK_CLASSIFICATION", default=True +) diff --git a/config/settings/test.py b/config/settings/test.py index 9d9cf4e79..a0f4df2d6 100755 --- a/config/settings/test.py +++ b/config/settings/test.py @@ -28,3 +28,6 @@ # Your stuff... # ------------------------------------------------------------------------------ +# evita consultas ao articlemeta durante os testes +# (os testes de Collection.ensure_network_classification a habilitam) +COLLECTION_ENSURE_NETWORK_CLASSIFICATION = False diff --git a/pid_provider/base_pid_provider.py b/pid_provider/base_pid_provider.py index 14033f9ff..e7f6a10e4 100644 --- a/pid_provider/base_pid_provider.py +++ b/pid_provider/base_pid_provider.py @@ -51,6 +51,7 @@ def provide_pid_for_xml_with_pre( registered_in_core=None, caller=None, auto_solve_pid_conflict=None, + collection_acron=None, ): """ Fornece e valida PIDs para documento XML, retornando dados completos de registro. @@ -77,6 +78,10 @@ def provide_pid_for_xml_with_pre( Identificador do sistema chamador auto_solve_pid_conflict : bool, optional Resolve conflitos de PID automaticamente + collection_acron : str, optional + Acrônimo da coleção de origem do XML (custom-meta); necessário + para registrar o pid v2 e a versão do XML por coleção + (CollectionPidV2) quando o periódico está em mais de uma coleção Returns ------- @@ -93,6 +98,10 @@ def provide_pid_for_xml_with_pre( """ self.caller = caller + # identifica a coleção de origem do XML (custom-meta) + if collection_acron and collection_acron != xml_with_pre.collection: + xml_with_pre.collection = collection_acron + registered = PidProviderXML.register( xml_with_pre, name, @@ -122,6 +131,7 @@ def provide_pid_for_xml_zip( registered_in_core=None, caller=None, auto_solve_pid_conflict=True, + collection_acron=None, ): """ Fornece / Valida PID para o XML em um arquivo compactado @@ -143,6 +153,7 @@ def provide_pid_for_xml_zip( registered_in_core=registered_in_core, caller=caller, auto_solve_pid_conflict=auto_solve_pid_conflict, + collection_acron=collection_acron, ) except Exception as e: exc_type, exc_value, exc_traceback = sys.exc_info() @@ -179,6 +190,7 @@ def provide_pid_for_xml_uri( registered_in_core=None, detail=None, auto_solve_pid_conflict=None, + collection_acron=None, ): """ Fornece / Valida PID de um XML disponível por um URI @@ -211,6 +223,7 @@ def provide_pid_for_xml_uri( origin=xml_uri, registered_in_core=registered_in_core, auto_solve_pid_conflict=auto_solve_pid_conflict, + collection_acron=collection_acron, ) # Handle response based on success or failure diff --git a/pid_provider/exceptions.py b/pid_provider/exceptions.py index a8ff9624e..a3c83d076 100644 --- a/pid_provider/exceptions.py +++ b/pid_provider/exceptions.py @@ -45,4 +45,7 @@ class PidProviderXMLFixPidV2Error(Exception): ... class APIPidProviderFixPidV2Error(Exception): ... -class UnmatchedPidProviderXMLError(Exception): ... \ No newline at end of file +class UnmatchedPidProviderXMLError(Exception): ... + + +class RequiredCollectionErrorToGetPidProviderXMLError(Exception): ... \ No newline at end of file diff --git a/pid_provider/migrations/0020_collectionpidv2.py b/pid_provider/migrations/0020_collectionpidv2.py new file mode 100644 index 000000000..e6e1d791a --- /dev/null +++ b/pid_provider/migrations/0020_collectionpidv2.py @@ -0,0 +1,116 @@ +# Generated by Django 5.2.7 on 2026-09-26 12:00 + +import django.db.models.deletion +import modelcluster.fields +from django.conf import settings +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("collection", "0008_collection_network_classification_and_more"), + ("pid_provider", "0019_alter_pidproviderxmlregistration_event_status"), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name="CollectionPidV2", + fields=[ + ( + "id", + models.BigAutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ( + "created", + models.DateTimeField( + auto_now_add=True, verbose_name="Creation date" + ), + ), + ( + "updated", + models.DateTimeField( + auto_now=True, verbose_name="Last update date" + ), + ), + ( + "pid_v2", + models.CharField( + blank=True, max_length=24, null=True, verbose_name="v2" + ), + ), + ( + "journal_acron", + models.CharField( + blank=True, + max_length=25, + null=True, + verbose_name="Journal acronym", + ), + ), + ( + "collection", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="collection.collection", + ), + ), + ( + "current_version", + models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + to="pid_provider.xmlversion", + ), + ), + ( + "creator", + models.ForeignKey( + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_creator", + to=settings.AUTH_USER_MODEL, + verbose_name="Creator", + ), + ), + ( + "pid_provider_xml", + modelcluster.fields.ParentalKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="collection_pids_v2", + to="pid_provider.pidproviderxml", + ), + ), + ( + "updated_by", + models.ForeignKey( + blank=True, + editable=False, + null=True, + on_delete=django.db.models.deletion.SET_NULL, + related_name="%(class)s_last_mod_user", + to=settings.AUTH_USER_MODEL, + verbose_name="Updater", + ), + ), + ], + options={ + "indexes": [ + models.Index( + fields=["pid_v2"], name="pid_provide_pid_v2_98b12b_idx" + ) + ], + "unique_together": {("pid_provider_xml", "collection")}, + }, + ), + ] diff --git a/pid_provider/models.py b/pid_provider/models.py index b58a9e4a8..472dba67e 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -32,6 +32,10 @@ profile_staticmethod, ) from pid_provider import choices, exceptions +from pid_provider.multi_collection import ( + get_journal_pid_from_v2, + get_xml_collections, +) from pid_provider.query_params import ( zero_to_none, QueryBuilderPidProviderXML, @@ -110,6 +114,9 @@ def xml_directory_path(instance, filename): class XMLVersion(CommonControlField): """ Tem função de guardar a versão do XML + + A versão do XML de cada coleção, quando o periódico está em mais de uma + coleção, é indicada por CollectionPidV2.current_version """ pid_provider_xml = models.ForeignKey( @@ -363,6 +370,166 @@ def created_updated(self): return self.updated or self.created +class CollectionPidV2(CommonControlField): + """ + Registro do PID v2 e da versão do XML do documento em cada coleção, + quando o periódico está em mais de uma coleção. + + Um mesmo periódico pode estar em mais de uma coleção com PIDs + (issn_scielo) e/ou acrônimos diferentes e, consequentemente, seus + artigos têm PIDs v2 e XML diferentes em cada coleção. + Ex.: Psicologia USP + - scl: S0103-65642009000300003 + - psi: S1678-51772009000300003 + + PidProviderXML.v2 e PidProviderXML.current_version mantêm os dados da + coleção principal (network_classification="scielonetwork"). Na falta + dela, PidProviderXML.v2 e PidProviderXML.current_version ficam vazios + (ver get_current_version). + """ + + pid_provider_xml = ParentalKey( + "PidProviderXML", + on_delete=models.CASCADE, + related_name="collection_pids_v2", + ) + collection = models.ForeignKey( + Collection, null=True, blank=True, on_delete=models.SET_NULL + ) + pid_v2 = models.CharField(_("v2"), max_length=24, null=True, blank=True) + journal_acron = models.CharField( + _("Journal acronym"), max_length=25, null=True, blank=True + ) + # versão atual do XML recebido desta coleção + current_version = models.ForeignKey( + XMLVersion, on_delete=models.SET_NULL, null=True, blank=True + ) + + panels = [ + FieldPanel("collection", read_only=True), + FieldPanel("journal_acron", read_only=True), + FieldPanel("pid_v2", read_only=True), + AutocompletePanel("current_version", read_only=True), + ] + + class Meta: + unique_together = [("pid_provider_xml", "collection")] + indexes = [ + models.Index(fields=["pid_v2"]), + ] + + def __str__(self): + return f"{self.collection} {self.journal_acron} {self.pid_v2}" + + @staticmethod + def is_equal_to(pid_provider_xml, xml_with_pre): + if not pid_provider_xml: + return False + for item in pid_provider_xml.collection_pids_v2.filter(collection__acron3=xml_with_pre.collection): + version = item.current_version + if version: + return bool(version and version.is_equal_to(xml_with_pre)) + return False + + @property + def journal_pid(self): + # ISSN adotado como PID do periódico na coleção (issn_scielo) + return get_journal_pid_from_v2(self.pid_v2) + + @classmethod + def get(cls, pid_provider_xml, collection): + return cls.objects.get(pid_provider_xml=pid_provider_xml, collection=collection) + + @classmethod + def create( + cls, + user, + pid_provider_xml, + collection, + pid_v2, + journal_acron=None, + current_version=None, + ): + try: + obj = cls() + obj.pid_v2 = pid_v2 + obj.journal_acron = journal_acron + obj.creator = user + obj.pid_provider_xml = pid_provider_xml + obj.collection = collection + obj.current_version = current_version + obj.save() + return obj + except IntegrityError: + return cls.get(pid_provider_xml, collection) + + @classmethod + @profile_classmethod + def create_or_update( + cls, + user, + pid_provider_xml, + collection, + pid_v2, + journal_acron=None, + current_version=None, + ): + if not (user and pid_provider_xml and collection and pid_v2 and journal_acron and current_version): + raise ValueError( + f"CollectionPidV2.create_or_update requires user ({user}) and pid_provider_xml ({pid_provider_xml}) and collection ({collection}) and pid_v2 ({pid_v2})" + ) + try: + obj = cls.get(pid_provider_xml, collection) + obj.pid_v2 = pid_v2 + obj.journal_acron = journal_acron + obj.updated_by = user + obj.current_version = current_version + obj.save() + return obj + except cls.DoesNotExist: + return cls.create( + user, + pid_provider_xml, + collection, + pid_v2, + journal_acron, + current_version, + ) + + @staticmethod + def get_current_version(pid_provider_xml, collection=None, collection_acron=None): + """ + Retorna a versão atual do XML da coleção solicitada ou, se a coleção + não foi fornecida, a versão da coleção principal + (PidProviderXML.current_version) + """ + if not pid_provider_xml: + return + if not collection and not collection_acron: + return pid_provider_xml.current_version + + params = {"current_version__isnull": False} + if collection: + params["collection"] = collection + else: + params["collection__acron3"] = collection_acron + item = pid_provider_xml.collection_pids_v2.filter( + **params + ).select_related("current_version").first() + if item: + return item.current_version + return None + + @property + def data(self): + return { + "pid_v2": self.pid_v2, + "journal_acron": self.journal_acron, + "collection_acron": self.collection and self.collection.acron3, + "is_main": bool(self.collection and self.collection.is_national_journal_collection), + } + + class PidProviderXMLManager(models.Manager): """ Manager customizado: aplica select_related("current_version") em toda @@ -478,6 +645,7 @@ class PidProviderXML(BasePidProviderXML, CommonControlField, ClusterableModel): panel_b = [ AutocompletePanel("current_version", read_only=True), InlinePanel("other_pid", label=_("Other PID")), + InlinePanel("collection_pids_v2", label=_("PID v2 by collection")), ] panel_c = [ FieldPanel("z_surnames", read_only=True), @@ -607,6 +775,7 @@ def get_queryset( @classmethod def delete_queryset(cls, qs): OtherPid.objects.filter(pid_provider_xml__in=qs).delete() + CollectionPidV2.objects.filter(pid_provider_xml__in=qs).delete() qs.delete() @classmethod @@ -614,16 +783,42 @@ def delete_queryset(cls, qs): def public_items(cls, from_date): now = datetime.utcnow().isoformat()[:10] # select_related("current_version") já vem do manager + # documento pode ter somente a versão de uma coleção + # (ver CollectionPidV2.get_current_version) return cls.objects.filter( (Q(available_since__isnull=True) | Q(available_since__lte=now)) - & (Q(created__gte=from_date) | Q(updated__gte=from_date)), - current_version__pid_provider_xml__v3__isnull=False, - ).iterator() + & (Q(created__gte=from_date) | Q(updated__gte=from_date)) + & ( + Q(current_version__isnull=False) + | Q(collection_pids_v2__current_version__isnull=False) + ), + v3__isnull=False, + ).distinct().iterator() @property def created_updated(self): return self.updated or self.created + @property + def collection_pids_v2_data(self): + items = [] + for item in self.collection_pids_v2.all(): + items.append(item.data) + return items + + @property + def pid_v2_list(self): + """ + Retorna os pids v2 do documento: o principal (self.v2) seguido + dos pids v2 do documento nas demais coleções + """ + items = [self.v2] + if self.pk: + items.extend( + self.collection_pids_v2.values_list("pid_v2", flat=True) + ) + return list(dict.fromkeys(item for item in items if item)) + @property @profile_property def data(self): @@ -633,7 +828,8 @@ def data(self): "aop_pid": self.aop_pid, "pkg_name": self.pkg_name, "registered_in_core": self.registered_in_core, - "ppx_id": self.id + "ppx_id": self.id, + "collection_pids_v2": self.collection_pids_v2_data, } _data["registered_data"] = self.get_readable_data() _data.update(self.record_status) @@ -661,10 +857,14 @@ def record_status(self): @classmethod @profile_classmethod - def get_xml_with_pre(cls, v3): + def get_xml_with_pre(cls, v3, collection=None, collection_acron=None): + """ + Retorna o XML da versão atual da coleção ou da coleção principal + (ver CollectionPidV2.get_current_version) + """ try: # select_related("current_version") já vem do manager - return cls.objects.get(v3=v3).xml_with_pre + return cls.objects.get(v3=v3).get_current_version(collection, collection_acron).xml_with_pre except cls.DoesNotExist: return None except Exception: @@ -735,7 +935,8 @@ def data_to_compare(self): # caso xml esteja ilegível # usar pid_v2 para reduzir a disambiguidade, pois sem # article_titles, body_fragment, surnames pode ser insuficiente - data["pid_v2"] = self.v2 + # lista, pois o documento pode ter pid v2 diferente em cada coleção + data["pid_v2"] = self.pid_v2_list or None data.update({ "z_surnames": self.z_surnames, "z_collab": self.z_collab, @@ -818,13 +1019,21 @@ def register( error_type = None select_record_response = None + # garante network_classification das coleções antes de + # identificar a coleção principal (ver add_collections) + Collection.ensure_network_classification(user) + # inputs pkg_name = filename input_data = None xml_adapter_data = None - input_data = {} - input_data.update(fix_xml_with_pre_data(xml_with_pre)) + input_data = fix_xml_with_pre_data(xml_with_pre) + xml_collections = get_xml_collections(xml_with_pre) + if not xml_collections: + raise exceptions.RequiredCollectionErrorToGetPidProviderXMLError( + "Unable to identify collection from XML. Please provide collection_acron or ensure XML has valid collection information." + ) input_data.update(fix_get_article_data(xml_with_pre)) input_data["origin"] = origin response["input_data"] = input_data @@ -878,7 +1087,6 @@ def register( except PidProviderXMLPidV3ConflictError as exc: event_status = "conflict" raise exc - # analisa se continua o registro try: PidProviderXML.is_updated( @@ -895,9 +1103,12 @@ def register( origin_date, available_since, registered_in_core, + xml_collections, ) # data to return response.update(registered.data) + # garante que o valor de v2 seja o da entrada, da coleção processada, e não o da coleção principal (que pode ser diferente) + response["v2"] = xml_with_pre.v2 except exceptions.ForbiddenPidProviderXMLRegistrationError: event_status = "forbidden" raise @@ -905,6 +1116,8 @@ def register( event_status = "skipped" response["skipped"] = True response.update(registered.data) + # garante que o valor de v2 seja o da entrada, da coleção processada, e não o da coleção principal (que pode ser diferente) + response["v2"] = xml_with_pre.v2 # do not raise except Exception as exc: event_status = event_status or "error" @@ -950,7 +1163,6 @@ def complete_missing_xml_pids( registered_pid=registered and registered.v3, auto_solve_pid_conflict=auto_solve_pid_conflict, ) - if valid_pid != xml_with_pre.v3: xml_with_pre.v3 = valid_pid xml_changed["pid_v3"] = valid_pid @@ -1002,6 +1214,7 @@ def _save( origin_date=None, available_since=None, registered_in_core=None, + xml_collections=None, ): if registered: registered_changed = registered.check_registered_pids_changed( @@ -1023,33 +1236,57 @@ def _save( if registered_changed: registered._add_other_pid(registered_changed, user) - registered._add_current_version(xml_adapter.xml_with_pre, user) - - registered.add_collections(xml_adapter) + registered.save() + + registered.add_collections(user, xml_adapter, xml_collections) return registered - def add_collections(self, xml_adapter): - q = Q() - issn_print = xml_adapter.journal_issn_print - issn_electronic = xml_adapter.journal_issn_electronic + def get_current_version(self, collection=None, collection_acron=None): + """ + Retorna a versão atual do XML da coleção ou da coleção principal + (ver CollectionPidV2.get_current_version) + """ + if not collection and not collection_acron: + return self.current_version + return CollectionPidV2.get_current_version(self, collection, collection_acron) - try: - Collection.objects.filter(scielojournal__isnull=True).exists() - issn_path = "scielojournal__journal__official" - except FieldError: - issn_path = "journalproc__journal__official_journal" + def add_collections(self, user, xml_adapter, xml_collections): + """ + Adiciona as coleções do periódico e registra o pid v2 e a versão do + XML da coleção de origem do XML (CollectionPidV2) - if issn_print: - q |= Q(**{f"{issn_path}__issn_print": issn_print}) - if issn_electronic: - q |= Q(**{f"{issn_path}__issn_electronic": issn_electronic}) + Somente o XML da coleção principal é a versão atual do documento + (self.current_version) + """ + xml_with_pre = xml_adapter.xml_with_pre + collection_acron = xml_with_pre.collection - for collection in Collection.objects.filter(q): + for xml_collection in xml_collections: + collection = xml_collection["collection"] self.collections.add(collection) + if collection.acron3 != collection_acron: + continue + if xml_collection["is_main"]: + self.v2 = xml_with_pre.v2 + self._add_current_version(xml_with_pre, user) + current_version = XMLVersion.get_or_create(user, self, xml_with_pre) + CollectionPidV2.create_or_update( + user, + self, + collection, + xml_with_pre.v2, + journal_acron=xml_collection["journal_acron"], + current_version=current_version, + ) + @staticmethod def is_updated( - xml_with_pre, registered, force_update, origin_date, registered_in_core + xml_with_pre, + registered, + force_update, + origin_date, + registered_in_core, ): """ XML é versão AOP, mas @@ -1075,6 +1312,7 @@ def is_updated( return # verifica se é necessário atualizar + # compara com a versão da mesma coleção (legado) if registered.is_equal_to(xml_with_pre): # XML fornecido é igual ao registrado, não precisa continuar logging.info(f"Skip update: equal") @@ -1098,9 +1336,7 @@ def is_updated( @profile_method def is_equal_to(self, xml_with_pre): - return bool( - self.current_version and self.current_version.is_equal_to(xml_with_pre) - ) + return CollectionPidV2.is_equal_to(self, xml_with_pre) @classmethod @profile_classmethod @@ -1120,14 +1356,14 @@ def select_records(cls, xml_adapter): # 1) busca por v3, v2, aop_pid, DOI identifier_queries = qbuilder.identifier_queries + # distinct: identifier_queries inclui busca por collection_pids_v2 yield "ids", ( - list(objects.filter(identifier_queries)) + list(objects.filter(identifier_queries).distinct()) if identifier_queries != Q() else [] ) selected_journal = objects.filter(qbuilder.issn_query) - # 2) busca exata com journal + issue + dados do artigo yield ( "journal-issue-article-strict", @@ -1182,7 +1418,6 @@ def _add_data(self, xml_adapter, registered_in_core): self.pkg_name = xml_adapter.sps_pkg_name self.article_pub_year = xml_adapter.article_pub_year self.v3 = xml_adapter.v3 - self.v2 = xml_adapter.v2 self.aop_pid = xml_adapter.aop_pid self.fpage = xml_adapter.fpage @@ -1228,7 +1463,16 @@ def _add_issue(self, xml_adapter): self.pub_year = xml_adapter.pub_year or xml_adapter.article_pub_year @profile_method - def _add_current_version(self, xml_with_pre, user, delete=False): + def _add_current_version( + self, xml_with_pre, user, delete=False + ): + """ + Registra a versão do XML + + Se o periódico está em mais de uma coleção, registra a versão para as + coleções do XML (CollectionPidV2) e somente a versão da coleção + principal é a versão atual do documento (self.current_version) + """ if delete: try: self.current_version.delete() @@ -1239,7 +1483,7 @@ def _add_current_version(self, xml_with_pre, user, delete=False): self.save() @profile_method - def check_registered_pids_changed(self, xml_with_pre): + def check_registered_pids_changed(self, xml_with_pre, ignore_v2=False): registered_changed = [] if self.v3 != xml_with_pre.v3: registered_changed.append( @@ -1250,15 +1494,18 @@ def check_registered_pids_changed(self, xml_with_pre): "registered": self.v3, } ) - if self.v2 != xml_with_pre.v2: - registered_changed.append( - { - "pid_type": "pid_v2", - "pid_in_xml": xml_with_pre.v2, - "version": self.current_version, - "registered": self.v2, - } - ) + for item in self.collection_pids_v2.filter(collection__acron3=xml_with_pre.collection): + if not ignore_v2 and item.pid_v2 != xml_with_pre.v2: + registered_changed.append( + { + "pid_type": "pid_v2", + "pid_in_xml": xml_with_pre.v2, + "version": item.current_version, + "registered": item.pid_v2, + } + ) + break + if self.aop_pid != xml_with_pre.aop_pid: registered_changed.append( { @@ -1303,7 +1550,11 @@ def _is_registered_pid(cls, v2=None, v3=None, aop_pid=None): if v3: qs = Q(v3=v3) | Q(other_pid__pid_in_xml=v3) elif v2: - qs = Q(v2=v2) | Q(other_pid__pid_in_xml=v2) + qs = ( + Q(v2=v2) + | Q(other_pid__pid_in_xml=v2) + | Q(collection_pids_v2__pid_v2=v2) + ) elif aop_pid: qs = Q(v2=aop_pid) | Q(other_pid__pid_in_xml=aop_pid) | Q(aop_pid=aop_pid) else: @@ -1372,6 +1623,9 @@ def is_registered( raise exc response["registered"] = True response.update(registered.data) + # pid v2 do XML de entrada (como em register), completado com o + # registrado quando ausente (ver complete_missing_xml_pids) + response["v2"] = xml_with_pre.v2 or registered.v2 response["is_equal"] = registered.is_equal_to(xml_with_pre) if not registered.readable_data: response["is_equal"] = False @@ -1411,7 +1665,7 @@ def fix_pid_v2(cls, user, pid_v3, correct_pid_v2): try: if correct_pid_v2 == item.v2: return item.data - xml_with_pre = item.current_version.xml_with_pre + xml_with_pre = item.get_current_version().xml_with_pre xml_with_pre.v2 = correct_pid_v2 item._add_current_version(xml_with_pre, user, delete=True) item.v2 = correct_pid_v2 @@ -1439,7 +1693,7 @@ def mark_as_done(self): def mark_items_as_invalid(cls, issns): # select_related("current_version") já vem do manager # (necessário aqui pois o loop acessa item.xml_with_pre, que usa - # self.current_version) + # self.get_current_version()) items = cls.objects.filter( Q(issn_print__in=issns) | Q(issn_electronic__in=issns), ) @@ -1560,7 +1814,7 @@ def fix_duplicated_pkg_name(cls, pkg_name, user): user=user, pid_type="pid_v3", pid_in_xml=item.v3, - version=item.current_version, + version=item.get_current_version(), pid_provider_xml=most_recent_item, ) except Exception as exception: diff --git a/pid_provider/multi_collection.py b/pid_provider/multi_collection.py new file mode 100644 index 000000000..e27a362db --- /dev/null +++ b/pid_provider/multi_collection.py @@ -0,0 +1,101 @@ +""" +Periódicos presentes em mais de uma coleção, possivelmente com dados +diferentes (acrônimo e/ou PID) e, consequentemente, artigos com PIDs v2 e +XML diferentes em cada coleção. + +Ex.: Psicologia USP +- scl: acron=pusp, pid=0103-6564, artigo S0103-65642009000300003 +- psi: acron=psicousp, pid=1678-5177, artigo S1678-51772009000300003 + +No site novo, o acrônimo é a chave do periódico e o PID compõe o pid v2 +(link legado), então os dados de cada coleção devem ser preservados. + +Código que não é modelo de banco de dados; o registro por coleção está em +pid_provider.models.CollectionPidV2 +""" + +from django.apps import apps +from django.db.models import Q + + +def get_journal_pid_from_v2(pid_v2): + """ + Retorna o PID do periódico (issn_scielo) contido no PID v2 + Ex.: S0103-65642009000300003 -> 0103-6564 + """ + if pid_v2 and len(pid_v2) == 23: + return pid_v2[1:10] + return None + + +def normalize_acron(acron): + return acron and acron.strip().lower() + + +def get_xml_collections(xml_with_pre): + """ + Retorna os dados do periódico em cada coleção em que está presente, + identificado pelos ISSNs e pelo PID do periódico contido no pid v2, + com a coleção principal primeiro + + Returns + ------- + list of dict + [{"collection": Collection, "journal_acron": str, "journal_pid": str, + "is_main": bool}] + + Se o XML informa a coleção de origem (custom-meta), retorna somente + os dados do periódico nesta coleção + """ + + issn_print = xml_with_pre.journal_issn_print + issn_electronic = xml_with_pre.journal_issn_electronic + pid_v2 = xml_with_pre.v2 + journal_acron = normalize_acron(xml_with_pre.journal_acron) + journal_pid = get_journal_pid_from_v2(pid_v2) + if not issn_print and not issn_electronic and not journal_pid: + return [] + try: + # Core + model = apps.get_model("journal", "SciELOJournal") + issn_path = "journal__official" + acron_field = "journal_acron" + pid_field = "issn_scielo" + except LookupError: + # Upload + model = apps.get_model("proc", "JournalProc") + issn_path = "journal__official_journal" + acron_field = "acron" + pid_field = "pid" + + q = Q() + if issn_print: + q |= Q(**{f"{issn_path}__issn_print": issn_print}) + if issn_electronic: + q |= Q(**{f"{issn_path}__issn_electronic": issn_electronic}) + if journal_pid: + q |= Q(**{pid_field: journal_pid}) + if journal_acron: + q |= Q(**{acron_field: journal_acron}) + + items = [] + main_item = None + for item in model.objects.filter(q, collection__isnull=False).select_related( + "collection" + ): + collection = item.collection + is_main = collection.is_national_journal_collection + data = { + "is_main": is_main, + "collection": collection, + "journal_acron": getattr(item, acron_field), + "journal_pid": getattr(item, pid_field), + } + if is_main: + main_item = data + else: + items.append(data) + if main_item: + items.insert(0, main_item) + + return items diff --git a/pid_provider/query_params.py b/pid_provider/query_params.py index c42efcf42..fc7619a24 100644 --- a/pid_provider/query_params.py +++ b/pid_provider/query_params.py @@ -123,14 +123,32 @@ def compare_lists(registered, xml_adapter_titles): return how_similar(" ".join(sorted(words1)), " ".join(sorted(words2))) +def compare_pid_v2(registered_pids_v2, input_pid_v2): + """ + Compara o pid v2 de entrada com os pids v2 registrados do documento + (um mesmo documento pode ter pid v2 diferente em cada coleção, ex.: + S0103-65642009000300003 em scl e S1678-51772009000300003 em psi). + + Retorna 1 se o pid v2 de entrada é um dos registrados; caso contrário, 0. + """ + if input_pid_v2 in registered_pids_v2: + return 1 + return 0 + + def compare_items(label, registered, input_data): """ Compara um único item entre o valor registrado e o valor de entrada. - - Se `registered` for uma lista (ex.: títulos), delega a - compare_lists(). - - Caso os dois valores, normalizados (falsy vira None), sejam - iguais, o score é 1. + Regras, na ordem em que são avaliadas: + - Se os dois valores forem iguais (sem normalização: None != ""), + o score é 1. + - Se `label` começar com "z_" ou contiver "finger" (hashes / + fingerprints), o score é 0, pois só a igualdade exata importa. + - Se `registered` for uma lista: + - "pid_v2" (pids v2 do documento em cada coleção): delega a + compare_pid_v2(); + - demais (ex.: títulos): delega a compare_lists(). - Caso contrário, o score vem de how_similar() entre os dois valores (None é tratado como string vazia). @@ -138,12 +156,15 @@ def compare_items(label, registered, input_data): "registered" quando o score não é 1 — útil para inspecionar divergências. """ - if isinstance(registered, list): - score = compare_lists(registered, input_data) - elif input_data == registered: + if input_data == registered: score = 1 elif label.startswith("z_") or 'finger' in label: score = 0 + elif isinstance(registered, list): + if label == "pid_v2": + score = compare_pid_v2(registered, input_data) + else: + score = compare_lists(registered, input_data) else: score = how_similar(input_data or "", registered or "") response = {"label": label, "score": score} @@ -318,7 +339,9 @@ def identifier_queries(self): q |= Q(v3=v3) if v2: - q |= Q(v2=v2) + # o pid v2 do XML pode ser o pid v2 do documento em outra coleção + # (periódico com PIDs diferentes em coleções diferentes) + q |= Q(v2=v2) | Q(collection_pids_v2__pid_v2=v2) if aop_pid: q |= Q(v2=aop_pid) | Q(aop_pid=aop_pid) @@ -396,9 +419,6 @@ def article_location_params(self): "fpage_seq": self.adapter_data.get("fpage_seq"), "lpage": self.adapter_data.get("lpage"), } - order = self.xml_adapter.order - if order: - data["v2__endswith"] = order return data @property @@ -598,9 +618,7 @@ def select_record(xml_adapter, selection_results): for label, results in selection_results: if not results: continue - result = get_best_match(results, xml_adapter_data_to_compare) - matched = result.get("matched") multiple_matched = result.get("multiple_matched") unmatched = result.get("unmatched") diff --git a/pid_provider/sources/harvesting.py b/pid_provider/sources/harvesting.py index ea3f73c29..b09e72939 100644 --- a/pid_provider/sources/harvesting.py +++ b/pid_provider/sources/harvesting.py @@ -57,6 +57,7 @@ def provide_pid_for_opac_and_am_xml( force_update=force_update, is_published=True, detail=detail, + collection_acron=collection_acron, ) CollectionPidRequest.create_or_update( user=user, diff --git a/pid_provider/tests/test_collection_pid_v2.py b/pid_provider/tests/test_collection_pid_v2.py new file mode 100644 index 000000000..852554526 --- /dev/null +++ b/pid_provider/tests/test_collection_pid_v2.py @@ -0,0 +1,405 @@ +""" +Testes de unidade para periódicos presentes em mais de uma coleção com +dados diferentes (PID e/ou acrônimo) e, consequentemente, artigos com +PIDs v2 e XML diferentes em cada coleção. + +Ex.: Psicologia USP +- scl: acron=pusp, pid=0103-6564, artigo S0103-65642009000300003 +- psi: acron=psicousp, pid=1678-5177, artigo S1678-51772009000300003 + +Os cenários de registro com XML real estão em +test_register_multi_collection.py +""" + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from django.contrib.auth import get_user_model +from django.test import SimpleTestCase, TestCase + +from collection.models import Collection +from journal.models import Journal, OfficialJournal, SciELOJournal +from pid_provider.models import CollectionPidV2, PidProviderXML, XMLVersion +from pid_provider.multi_collection import ( + get_journal_pid_from_v2, + get_xml_collections, + normalize_acron, +) +from pid_provider.query_params import compare + +User = get_user_model() + +SCL_V2 = "S0103-65642009000300003" +PSI_V2 = "S1678-51772009000300003" +ISSN_PRINT = "0103-6564" +ISSN_ELECTRONIC = "1678-5177" + +# CollectionPidV2.data +SCL_DATA = { + "pid_v2": SCL_V2, + "journal_acron": "pusp", + "collection_acron": "scl", + "is_main": True, +} +PSI_DATA = { + "pid_v2": PSI_V2, + "journal_acron": "psicousp", + "collection_acron": "psi", + "is_main": False, +} + + +def make_xml_with_pre( + v2=None, + journal_acron=None, + collection=None, + issn_print=ISSN_PRINT, + issn_electronic=ISSN_ELECTRONIC, +): + return SimpleNamespace( + v2=v2, + v3="V3-PSICOUSP-0000000001", + aop_pid=None, + journal_acron=journal_acron, + collection=collection, + journal_issn_print=issn_print, + journal_issn_electronic=issn_electronic, + ) + + +class MultiCollectionJournalTestBase(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="multicol", password="x") + self.scl = Collection.objects.create( + acron3="scl", network_classification=["scielonetwork"], creator=self.user + ) + self.psi = Collection.objects.create( + acron3="psi", network_classification=["thematic"], creator=self.user + ) + journal = Journal.objects.create( + official=OfficialJournal.objects.create( + title="Psicologia USP", + issn_print=ISSN_PRINT, + issn_electronic=ISSN_ELECTRONIC, + creator=self.user, + ), + creator=self.user, + ) + # psi é criada antes de scl para verificar que a principal vem primeiro + for collection, acron, pid in ( + (self.psi, "psicousp", ISSN_ELECTRONIC), + (self.scl, "pusp", ISSN_PRINT), + ): + SciELOJournal.objects.create( + journal=journal, + collection=collection, + journal_acron=acron, + issn_scielo=pid, + creator=self.user, + ) + + def create_pid_provider_xml(self, v2=SCL_V2): + return PidProviderXML.objects.create( + v3="V3-PSICOUSP-0000000001", v2=v2, creator=self.user + ) + + def create_version(self, registered, finger_print): + return XMLVersion.objects.create( + pid_provider_xml=registered, finger_print=finger_print, creator=self.user + ) + + def create_collection_pid_v2(self, registered, collection, pid_v2, version): + return CollectionPidV2.create_or_update( + self.user, + pid_provider_xml=registered, + collection=collection, + pid_v2=pid_v2, + journal_acron="pusp" if collection == self.scl else "psicousp", + current_version=version, + ) + + +class GetJournalPidFromV2Test(SimpleTestCase): + def test_get_journal_pid_from_v2(self): + self.assertEqual(get_journal_pid_from_v2(SCL_V2), ISSN_PRINT) + self.assertEqual(get_journal_pid_from_v2(PSI_V2), ISSN_ELECTRONIC) + self.assertIsNone(get_journal_pid_from_v2(None)) + self.assertIsNone(get_journal_pid_from_v2("S0103-6564")) + + +class NormalizeAcronTest(SimpleTestCase): + def test_normalize_acron(self): + self.assertEqual(normalize_acron(" PUSP "), "pusp") + self.assertIsNone(normalize_acron(None)) + + +class GetXMLCollectionsTest(MultiCollectionJournalTestBase): + def result(self, xml_with_pre): + return [ + ( + item["collection"].acron3, + item["journal_acron"], + item["journal_pid"], + item["is_main"], + ) + for item in get_xml_collections(xml_with_pre) + ] + + def test_returns_journal_data_by_collection_with_main_first(self): + self.assertEqual( + self.result(make_xml_with_pre(PSI_V2, "psicousp")), + [ + ("scl", "pusp", ISSN_PRINT, True), + ("psi", "psicousp", ISSN_ELECTRONIC, False), + ], + ) + + def test_identifies_journal_by_pid_v2_or_acron(self): + # sem ISSN, somente a coleção com o PID ou o acrônimo do periódico + expected = [("psi", "psicousp", ISSN_ELECTRONIC, False)] + cases = ( + # somente pelo PID do periódico contido no pid v2 + make_xml_with_pre(PSI_V2, issn_print=None, issn_electronic=None), + # pelo acrônimo (normalizado), PID do periódico desconhecido + make_xml_with_pre( + "S0000-00002009000300003", + " PSICOUSP ", + issn_print=None, + issn_electronic=None, + ), + ) + for xml_with_pre in cases: + with self.subTest(v2=xml_with_pre.v2, acron=xml_with_pre.journal_acron): + self.assertEqual(self.result(xml_with_pre), expected) + + def test_no_main_collection(self): + Collection.objects.update(network_classification=None) + self.assertEqual( + {item[3] for item in self.result(make_xml_with_pre(SCL_V2, "pusp"))}, + {False}, + ) + + def test_empty_without_journal_identification(self): + xml_with_pre = make_xml_with_pre( + None, "pusp", issn_print=None, issn_electronic=None + ) + self.assertEqual(get_xml_collections(xml_with_pre), []) + + def test_empty_when_journal_is_not_found(self): + xml_with_pre = make_xml_with_pre( + "S0000-00002009000300003", + "xxx", + issn_print="0000-0000", + issn_electronic="0000-0001", + ) + self.assertEqual(get_xml_collections(xml_with_pre), []) + + +class CollectionPidV2Test(MultiCollectionJournalTestBase): + def test_journal_pid(self): + self.assertEqual(CollectionPidV2(pid_v2=PSI_V2).journal_pid, ISSN_ELECTRONIC) + self.assertIsNone(CollectionPidV2().journal_pid) + + def test_create_or_update_requires_all_data(self): + registered = self.create_pid_provider_xml() + version = self.create_version(registered, "v1") + params = dict( + user=self.user, + pid_provider_xml=registered, + collection=self.psi, + pid_v2=PSI_V2, + journal_acron="psicousp", + current_version=version, + ) + for name in params: + with self.subTest(missing=name): + with self.assertRaises(ValueError): + CollectionPidV2.create_or_update(**{**params, name: None}) + self.assertFalse(CollectionPidV2.objects.exists()) + + def test_create_or_update(self): + registered = self.create_pid_provider_xml() + first = self.create_version(registered, "v1") + second = self.create_version(registered, "v2") + + self.create_collection_pid_v2(registered, self.psi, PSI_V2, first) + obj = self.create_collection_pid_v2(registered, self.psi, PSI_V2, second) + + obj.refresh_from_db() + self.assertEqual(obj.current_version, second) + self.assertEqual(obj.journal_acron, "psicousp") + self.assertEqual( + CollectionPidV2.objects.filter(pid_provider_xml=registered).count(), 1 + ) + + def test_get_current_version(self): + registered = self.create_pid_provider_xml() + psi_version = self.create_version(registered, "psi") + self.create_collection_pid_v2(registered, self.psi, PSI_V2, psi_version) + + # sem versão da coleção principal + self.assertIsNone(CollectionPidV2.get_current_version(registered)) + self.assertIsNone(CollectionPidV2.get_current_version(registered, self.scl)) + self.assertEqual( + CollectionPidV2.get_current_version(registered, self.psi), psi_version + ) + self.assertEqual( + CollectionPidV2.get_current_version(registered, collection_acron="psi"), + psi_version, + ) + + registered.current_version = self.create_version(registered, "scl") + registered.save() + self.assertEqual( + CollectionPidV2.get_current_version(registered), + registered.current_version, + ) + self.assertIsNone(CollectionPidV2.get_current_version(None)) + + def test_is_equal_to_compares_with_version_of_the_same_collection(self): + registered = self.create_pid_provider_xml() + psi_version = self.create_version(registered, "psi") + self.create_collection_pid_v2(registered, self.psi, PSI_V2, psi_version) + + with patch.object(XMLVersion, "is_equal_to", return_value=True) as is_equal: + self.assertTrue( + CollectionPidV2.is_equal_to(registered, make_xml_with_pre(collection="psi")) + ) + self.assertEqual(is_equal.call_count, 1) + self.assertFalse( + CollectionPidV2.is_equal_to(registered, make_xml_with_pre(collection="scl")) + ) + self.assertFalse( + CollectionPidV2.is_equal_to(registered, make_xml_with_pre(collection=None)) + ) + self.assertFalse( + CollectionPidV2.is_equal_to(None, make_xml_with_pre(collection="psi")) + ) + self.assertEqual(is_equal.call_count, 1) + + +class PidProviderXMLCollectionsTest(MultiCollectionJournalTestBase): + def test_get_current_version(self): + registered = self.create_pid_provider_xml() + psi_version = self.create_version(registered, "psi") + self.create_collection_pid_v2(registered, self.psi, PSI_V2, psi_version) + registered.current_version = self.create_version(registered, "scl") + registered.save() + + self.assertEqual(registered.get_current_version(), registered.current_version) + self.assertEqual(registered.get_current_version(self.psi), psi_version) + self.assertEqual( + registered.get_current_version(collection_acron="psi"), psi_version + ) + self.assertIsNone(registered.get_current_version(self.scl)) + + def test_add_collections_registers_only_the_xml_collection(self): + registered = self.create_pid_provider_xml(PSI_V2) + xml_adapter = MagicMock( + xml_with_pre=make_xml_with_pre(PSI_V2, "psicousp", collection="psi") + ) + version = self.create_version(registered, "psi") + + with patch.object(XMLVersion, "get_or_create", return_value=version): + registered.add_collections( + self.user, xml_adapter, get_xml_collections(xml_adapter.xml_with_pre) + ) + + registered.refresh_from_db() + self.assertEqual( + set(registered.collections.values_list("acron3", flat=True)), + {"scl", "psi"}, + ) + self.assertEqual(registered.collection_pids_v2_data, [PSI_DATA]) + self.assertEqual(registered.get_current_version(self.psi), version) + # somente o XML da coleção principal é a versão atual do documento + self.assertIsNone(registered.current_version) + + def test_add_collections_main_collection_xml_is_the_current_version(self): + registered = self.create_pid_provider_xml(SCL_V2) + xml_adapter = MagicMock( + xml_with_pre=make_xml_with_pre(SCL_V2, "pusp", collection="scl") + ) + version = self.create_version(registered, "scl") + + with patch.object(XMLVersion, "get_or_create", return_value=version): + registered.add_collections( + self.user, xml_adapter, get_xml_collections(xml_adapter.xml_with_pre) + ) + + registered.refresh_from_db() + self.assertEqual(registered.current_version, version) + self.assertEqual(registered.get_current_version(self.scl), version) + + def test_add_collections_without_xml_collection(self): + registered = self.create_pid_provider_xml(SCL_V2) + xml_adapter = MagicMock(xml_with_pre=make_xml_with_pre(SCL_V2, "pusp")) + + with patch.object(XMLVersion, "get_or_create") as get_or_create: + registered.add_collections( + self.user, xml_adapter, get_xml_collections(xml_adapter.xml_with_pre) + ) + + get_or_create.assert_not_called() + registered.refresh_from_db() + self.assertEqual(registered.collections.count(), 2) + self.assertFalse(registered.collection_pids_v2.exists()) + self.assertIsNone(registered.current_version) + + def test_check_registered_pids_changed_compares_pid_v2_of_the_same_collection(self): + registered = self.create_pid_provider_xml(SCL_V2) + self.create_collection_pid_v2( + registered, self.psi, PSI_V2, self.create_version(registered, "psi") + ) + cases = ( + # (pid v2 do XML, coleção do XML, houve mudança) + (PSI_V2, "psi", False), + ("S1678-51772009000300099", "psi", True), + # coleção sem pid v2 registrado + (PSI_V2, "scl", False), + ) + for pid_v2, collection, expected in cases: + with self.subTest(pid_v2=pid_v2, collection=collection): + xml_with_pre = make_xml_with_pre(pid_v2, collection=collection) + changed = registered.check_registered_pids_changed(xml_with_pre) + self.assertEqual( + any(item["pid_type"] == "pid_v2" for item in changed), expected + ) + + def test_check_registered_pids_changed_registers_pid_v2_and_version_of_the_collection(self): + registered = self.create_pid_provider_xml(SCL_V2) + registered.current_version = self.create_version(registered, "scl") + registered.save() + psi_version = self.create_version(registered, "psi") + self.create_collection_pid_v2(registered, self.psi, PSI_V2, psi_version) + xml_with_pre = make_xml_with_pre( + "S1678-51772009000300099", collection="psi" + ) + + changed = registered.check_registered_pids_changed(xml_with_pre) + + pid_v2_changes = [item for item in changed if item["pid_type"] == "pid_v2"] + self.assertEqual(len(pid_v2_changes), 1) + self.assertEqual(pid_v2_changes[0]["registered"], PSI_V2) + self.assertEqual(pid_v2_changes[0]["version"], psi_version) + + +class DataToCompareTest(MultiCollectionJournalTestBase): + def test_uses_all_pids_v2_when_unreadable(self): + registered = self.create_pid_provider_xml(SCL_V2) + for collection, pid_v2 in ((self.scl, SCL_V2), (self.psi, PSI_V2)): + self.create_collection_pid_v2( + registered, collection, pid_v2, self.create_version(registered, pid_v2) + ) + self.assertEqual(registered.pid_v2_list, [SCL_V2, PSI_V2]) + self.assertCountEqual( + registered.collection_pids_v2_data, [SCL_DATA, PSI_DATA] + ) + + with patch.object(PidProviderXML, "get_readable_data", return_value={}): + registered_data = registered.data_to_compare + self.assertEqual(registered_data["pid_v2"], [SCL_V2, PSI_V2]) + + result = compare(registered_data, {**registered_data, "pid_v2": PSI_V2}) + pid_v2_item = next(i for i in result["items"] if i["label"] == "pid_v2") + self.assertEqual(pid_v2_item["score"], 1) diff --git a/pid_provider/tests/test_pid_provider_xml_helpers.py b/pid_provider/tests/test_pid_provider_xml_helpers.py index dd16ecf80..838ea39ec 100644 --- a/pid_provider/tests/test_pid_provider_xml_helpers.py +++ b/pid_provider/tests/test_pid_provider_xml_helpers.py @@ -121,7 +121,9 @@ def test_newer_origin_date_returns_none(self): class CheckRegisteredPidsChangedTests(SimpleTestCase): def test_reports_only_fields_that_differ(self): ppx = PidProviderXML(v3="V3-OLD", v2="V2-SAME", aop_pid=None) - xml_with_pre = SimpleNamespace(v3="V3-NEW", v2="V2-SAME", aop_pid="AOP-NEW") + xml_with_pre = SimpleNamespace( + v3="V3-NEW", v2="V2-SAME", aop_pid="AOP-NEW", collection=None + ) changed = ppx.check_registered_pids_changed(xml_with_pre) @@ -134,7 +136,7 @@ def test_reports_only_fields_that_differ(self): def test_returns_empty_list_when_nothing_changed(self): ppx = PidProviderXML(v3="V3", v2="V2", aop_pid="AOP") - xml_with_pre = SimpleNamespace(v3="V3", v2="V2", aop_pid="AOP") + xml_with_pre = SimpleNamespace(v3="V3", v2="V2", aop_pid="AOP", collection=None) self.assertEqual(ppx.check_registered_pids_changed(xml_with_pre), []) @@ -438,7 +440,8 @@ def test_falls_back_to_pid_v2_when_readable_data_is_empty(self): creator=self.user, v3="A", v2="V2-FALLBACK", readable_data=None ) data = ppx.data_to_compare - self.assertEqual(data["pid_v2"], "V2-FALLBACK") + # lista: o documento pode ter pid v2 diferente em cada coleção + self.assertEqual(data["pid_v2"], ["V2-FALLBACK"]) class DataPropertyTests(TestCase): diff --git a/pid_provider/tests/test_query_params.py b/pid_provider/tests/test_query_params.py index 140f3f9d9..6347e1f26 100644 --- a/pid_provider/tests/test_query_params.py +++ b/pid_provider/tests/test_query_params.py @@ -41,6 +41,7 @@ QueryBuilderPidProviderXML, compare, compare_items, + compare_pid_v2, compare_lists, fix_get_data_to_compare, fix_xml_with_pre_data, @@ -328,7 +329,9 @@ def test_v3_only(self): def test_v2_and_aop_pid_combine_with_or(self): adapter = make_xml_adapter(data={}, v2="V2-1", aop_pid="AOP-1") qbuilder = QueryBuilderPidProviderXML(adapter) - expected = Q(v2="V2-1") | (Q(v2="AOP-1") | Q(aop_pid="AOP-1")) + expected = (Q(v2="V2-1") | Q(collection_pids_v2__pid_v2="V2-1")) | ( + Q(v2="AOP-1") | Q(aop_pid="AOP-1") + ) self.assertEqual(qbuilder.identifier_queries, expected) def test_includes_main_doi(self): @@ -428,7 +431,7 @@ def test_with_order_adds_v2_endswith(self): adapter = make_xml_adapter(data={}, order="00003") qbuilder = QueryBuilderPidProviderXML(adapter) params = qbuilder.article_location_params - self.assertEqual(params["v2__endswith"], "00003") + self.assertNotIn("v2__endswith", params) class PartialBodyQueryTests(SimpleTestCase): @@ -931,3 +934,39 @@ def test_all_fields_falsy_on_both_sides_returns_zero_percentual_score(self): self.assertEqual(compared_item["score"], 0) self.assertEqual(result["total_score"], 0.0) self.assertEqual(result["percentual_score"], 0.0) + + +class ComparePidV2Tests(SimpleTestCase): + """ + Um mesmo documento pode ter pid v2 diferente em cada coleção + (ex.: S0103-65642009000300003 em scl e S1678-51772009000300003 em psi) + """ + + REGISTERED = ["S0103-65642009000300003", "S1678-51772009000300003"] + + def test_one_when_input_is_any_registered_pid_v2(self): + self.assertEqual(compare_pid_v2(self.REGISTERED, "S0103-65642009000300003"), 1) + self.assertEqual(compare_pid_v2(self.REGISTERED, "S1678-51772009000300003"), 1) + + def test_zero_when_any_side_is_empty(self): + self.assertEqual(compare_pid_v2([], "S0103-65642009000300003"), 0) + self.assertEqual(compare_pid_v2(self.REGISTERED, None), 0) + + def test_zero_when_input_is_not_registered(self): + # pid v2 é identificador: sem similaridade parcial + self.assertEqual( + compare_pid_v2(self.REGISTERED, "S0103-65642009000300004"), 0 + ) + + def test_compare_items_uses_compare_pid_v2_for_pid_v2_list(self): + result = compare_items("pid_v2", self.REGISTERED, "S1678-51772009000300003") + self.assertEqual(result, {"label": "pid_v2", "score": 1}) + + @patch("pid_provider.query_params.compare_lists") + def test_compare_items_does_not_use_compare_lists_for_pid_v2(self, mock_compare_lists): + compare_items("pid_v2", self.REGISTERED, "S1678-51772009000300003") + mock_compare_lists.assert_not_called() + + def test_compare_items_keeps_string_comparison_for_single_pid_v2(self): + result = compare_items("pid_v2", "S0103-65642009000300003", "S0103-65642009000300003") + self.assertEqual(result["score"], 1) diff --git a/pid_provider/tests/test_register.py b/pid_provider/tests/test_register.py index 49401c624..d631469bb 100644 --- a/pid_provider/tests/test_register.py +++ b/pid_provider/tests/test_register.py @@ -163,6 +163,21 @@ def setUp(self): self.m_record = self.p_record.start() self.addCleanup(self.p_record.stop) + # register() exige que a coleção do XML seja identificável + # (RequiredCollectionErrorToGetPidProviderXMLError); _save está + # mockado nos testes, então o conteúdo da lista não é consumido. + self.p_xml_collections = patch(f"{PATCH_BASE}.get_xml_collections") + self.m_xml_collections = self.p_xml_collections.start() + self.m_xml_collections.return_value = [ + { + "is_main": True, + "collection": MagicMock(name="collection"), + "journal_acron": "acron", + "journal_pid": "0000-0000", + } + ] + self.addCleanup(self.p_xml_collections.stop) + # Mantém os testes independentes da configuração persistida. self.p_setting = patch(f"{PATCH_BASE}.PidProviderSetting.load") self.m_setting_load = self.p_setting.start() @@ -475,6 +490,27 @@ def test_not_enough_parameters_becomes_response(self): self.assert_recorded_status("bad_request") +class RequiredCollectionPathTest(RegisterTestBase): + """ + Sem coleção identificável para o XML, register() não seleciona nem + grava o registro: a exceção vira resposta de erro + """ + + def test_unidentified_collection_becomes_error_response(self): + self.m_xml_collections.return_value = [] + with patch(f"{PATCH_BASE}.select_record") as m_select, \ + patch(f"{PATCH_BASE}.PidProviderXML._save") as m_save: + response = PidProviderXML.register(self.xml, "file.xml", self.user) + + m_select.assert_not_called() + m_save.assert_not_called() + self.assertEqual( + response["error_type"], + str(exceptions.RequiredCollectionErrorToGetPidProviderXMLError), + ) + self.assert_recorded_status("error") + + class UnexpectedErrorPathTest(RegisterTestBase): """ ATENÇÃO — MUDANÇA DE CONTRATO: @@ -664,6 +700,7 @@ def test_top_level_keys_on_clean_success(self): "xml_adapter_data", "xml_changed", "v3", + "v2", "record_status", "created", "event_status", @@ -748,6 +785,7 @@ def test_skipped_path_keys(self): "input_data", "xml_adapter_data", "v3", + "v2", "record_status", "skipped", "event_status", diff --git a/pid_provider/tests/test_register_multi_collection.py b/pid_provider/tests/test_register_multi_collection.py new file mode 100644 index 000000000..901f53b3d --- /dev/null +++ b/pid_provider/tests/test_register_multi_collection.py @@ -0,0 +1,871 @@ +""" +Testes de integração de PidProviderXML.register com 2 XML que representam +o mesmo artigo em coleções diferentes. + +Ex.: Psicologia USP +- scl: acron=pusp, pid=0103-6564, artigo S0103-65642009000300003 +- psi: acron=psicousp, pid=1678-5177, artigo S1678-51772009000300003 + +Os XML têm o mesmo conteúdo, diferem no pid v2 (e no acrônimo do +periódico), informam a coleção de origem (custom-meta, como em +ArticleProc) e não têm pid v3 (como os XML gerados a partir do site clássico). + +- somente o XML da coleção principal é a versão atual do documento + (PidProviderXML.current_version) +- o pid v2 principal (PidProviderXML.v2) é somente o da coleção principal; + a resposta do registro traz o pid v2 do XML recebido +- o pid v2 e a versão do XML de cada coleção ficam em CollectionPidV2 +""" + +from tempfile import TemporaryDirectory +from unittest.mock import patch + +from django.contrib.auth import get_user_model +from django.core.cache import cache +from django.test import TestCase, override_settings +from packtools.sps.pid_provider.xml_sps_lib import XMLWithPre + +from collection.models import ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY, Collection +from journal.models import Journal, OfficialJournal, SciELOJournal +from pid_provider import exceptions +from pid_provider.models import ( + CollectionPidV2, + OtherPid, + PidProviderXML, + XMLVersion, +) + +User = get_user_model() + +SCL_V2 = "S0103-65642009000300003" +PSI_V2 = "S1678-51772009000300003" +ISSN_PRINT = "0103-6564" +ISSN_ELECTRONIC = "1678-5177" + + +def pid_v2_data(pid_v2, journal_acron, collection_acron, is_main): + # CollectionPidV2.data + return { + "pid_v2": pid_v2, + "journal_acron": journal_acron, + "collection_acron": collection_acron, + "is_main": is_main, + } + + +SCL_DATA = pid_v2_data(SCL_V2, "pusp", "scl", True) +PSI_DATA = pid_v2_data(PSI_V2, "psicousp", "psi", False) + +XML_TEMPLATE = """ +
+ + + {journal_acron} + + Psicologia USP + + 0103-6564 + 1678-5177 + + + {pid_v2} + 00003 + + + Artigos + + + + A constituição do sujeito e o laço social + + + + + Silva + Maria + + + + + Souza + João + + + + + 01 + 09 + 2009 + + + 2009 + + 20 + 3 + 333 + 350 + + + +

Este artigo discute a constituição do sujeito a partir do laço social, considerando as contribuições da psicanálise para a compreensão das relações entre o indivíduo e a cultura contemporânea.

+

Discute-se, ainda, o lugar do outro na formação da subjetividade e as consequências clínicas dessa perspectiva.

+ +
+""" + + +TITLE = "A constituição do sujeito e o laço social" +UPDATED_TITLE = "A constituição do sujeito e do laço social" + +# Errata: documento próprio (article-type="correction"), com pid v2 próprio, +# sem resumo e sem autores, contendo somente os trechos "onde se lê X, +# leia-se Y" e o related-article que aponta para o artigo corrigido +SCL_ERRATUM_V2 = "S0103-65642009000300010" +PSI_ERRATUM_V2 = "S1678-51772009000300010" + +ERRATUM_TEMPLATE = """ +
+ + + {journal_acron} + + Psicologia USP + + 0103-6564 + 1678-5177 + + + {pid_v2} + 00010 + + + Errata + + + + Errata + + + 15 + 10 + 2009 + + + 2009 + + 20 + 3 + 351 + 351 + + + + +

No artigo "A constituição do sujeito e o laço social", publicado no v. 20, n. 3, p. 333-350, 2009:

+

Onde se lê: "A constituição do sujeito e o laço social"

+

Leia-se: "A constituição do sujeito e do laço social"

+ +
+""" + + +def make_xml_with_pre(pid_v2, journal_acron, collection=None, content=None): + xml_content = content or XML_TEMPLATE.format( + pid_v2=pid_v2, journal_acron=journal_acron + ) + xml_with_pre = list(XMLWithPre.create(xml_content=xml_content))[0] + if collection: + # coleção de origem do XML (ver ArticleProc) + xml_with_pre.collection = collection + return xml_with_pre + + +def scl_xml(): + return make_xml_with_pre(SCL_V2, "pusp", "scl") + + +def psi_xml(): + return make_xml_with_pre(PSI_V2, "psicousp", "psi") + + +def updated_xml(pid_v2, journal_acron, collection): + # nova versão do XML do mesmo artigo (mesmo pid v2), com o título corrigido + content = XML_TEMPLATE.format( + pid_v2=pid_v2, journal_acron=journal_acron + ).replace(TITLE, UPDATED_TITLE) + return make_xml_with_pre(pid_v2, journal_acron, collection, content) + + +def erratum_xml(pid_v2, journal_acron, collection): + content = ERRATUM_TEMPLATE.format( + pid_v2=pid_v2, journal_acron=journal_acron + ) + return make_xml_with_pre(pid_v2, journal_acron, collection, content) + + +# não consulta o articlemeta, independente das settings usadas (ex.: settings.local +# com articlemeta acessível completaria network_classification das coleções +# e os testes sem coleção principal passariam a ter scl como principal); +# subclasses podem habilitar com override_settings +@override_settings(COLLECTION_ENSURE_NETWORK_CLASSIFICATION=False) +class RegisterSameArticleInDifferentCollectionsTestBase(TestCase): + def setUp(self): + self.media_directory = TemporaryDirectory() + self.override_settings = override_settings( + MEDIA_ROOT=self.media_directory.name + ) + self.override_settings.enable() + self.addCleanup(self.override_settings.disable) + self.addCleanup(self.media_directory.cleanup) + + self.user = User.objects.create_user(username="multicol-register") + self.scl = Collection.objects.create( + acron3="scl", network_classification=["scielonetwork"], creator=self.user + ) + self.psi = Collection.objects.create( + acron3="psi", network_classification=["thematic"], creator=self.user + ) + official_journal = OfficialJournal.objects.create( + title="Psicologia USP", + issn_print=ISSN_PRINT, + issn_electronic=ISSN_ELECTRONIC, + creator=self.user, + ) + journal = Journal.objects.create( + official=official_journal, creator=self.user + ) + SciELOJournal.objects.create( + journal=journal, + collection=self.scl, + journal_acron="pusp", + issn_scielo=ISSN_PRINT, + creator=self.user, + ) + SciELOJournal.objects.create( + journal=journal, + collection=self.psi, + journal_acron="psicousp", + issn_scielo=ISSN_ELECTRONIC, + creator=self.user, + ) + + def register(self, xml_with_pre, filename): + response = PidProviderXML.register( + xml_with_pre, filename, self.user, origin_date="2026-01-01" + ) + self.assertIsNone( + response.get("error_type"), + f"{response.get('error_msg')} {response.get('traceback')}", + ) + return response + + def assert_same_article_registered_once(self, first, second): + self.assertEqual(PidProviderXML.objects.count(), 1) + registered = PidProviderXML.objects.get() + self.assertEqual(first["v3"], registered.v3) + self.assertEqual(second["v3"], registered.v3) + self.assertEqual(first["ppx_id"], second["ppx_id"]) + return registered + + def assert_pids_v2_by_collection(self, registered, main="scl"): + self.assertCountEqual( + registered.collection_pids_v2_data, + [ + pid_v2_data(SCL_V2, "pusp", "scl", main == "scl"), + pid_v2_data(PSI_V2, "psicousp", "psi", main == "psi"), + ], + ) + self.assertEqual( + { + (item.collection.acron3, item.journal_acron, item.journal_pid, item.pid_v2) + for item in CollectionPidV2.objects.filter( + pid_provider_xml=registered + ) + }, + { + ("scl", "pusp", ISSN_PRINT, SCL_V2), + ("psi", "psicousp", ISSN_ELECTRONIC, PSI_V2), + }, + ) + + def collection_current_version(self, registered, collection): + return CollectionPidV2.objects.get( + pid_provider_xml=registered, collection=collection + ).current_version + + def assert_current_version_is_from(self, registered, collection): + registered.refresh_from_db() + self.assertEqual( + registered.current_version, + self.collection_current_version(registered, collection), + ) + + def assert_no_pid_v2_change(self, registered): + # pid v2 de outra coleção não é mudança de pid v2 + self.assertFalse( + OtherPid.objects.filter( + pid_provider_xml=registered, pid_type="pid_v2" + ).exists() + ) + + +class RegisterMainCollectionFirstTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """scl (coleção principal) é registrado antes de psi (temática)""" + + def test_second_xml_is_identified_as_the_same_article(self): + first = self.register(scl_xml(), "scl") + second = self.register(psi_xml(), "psi") + + self.assertEqual(first["event_status"], "created") + self.assertEqual(second["event_status"], "updated") + registered = self.assert_same_article_registered_once(first, second) + # xml de psi recebe o mesmo pid v3 + self.assertEqual(second["xml_changed"], {"pid_v3": registered.v3}) + + def test_main_v2_is_kept_and_v2_by_collection_is_registered(self): + self.register(scl_xml(), "scl") + second = self.register(psi_xml(), "psi") + + registered = PidProviderXML.objects.get() + self.assertEqual(registered.v2, SCL_V2) + # a resposta traz o pid v2 do XML recebido + self.assertEqual(second["v2"], PSI_V2) + self.assertCountEqual(second["collection_pids_v2"], [SCL_DATA, PSI_DATA]) + self.assert_pids_v2_by_collection(registered) + self.assert_no_pid_v2_change(registered) + + def test_registering_alternately_keeps_main_v2(self): + responses = [ + self.register(scl_xml(), "scl"), + self.register(psi_xml(), "psi"), + self.register(scl_xml(), "scl"), + self.register(psi_xml(), "psi"), + ] + + registered = PidProviderXML.objects.get() + self.assertEqual({item["v3"] for item in responses}, {registered.v3}) + self.assertEqual(registered.v2, SCL_V2) + self.assertEqual(CollectionPidV2.objects.count(), 2) + self.assert_no_pid_v2_change(registered) + + def test_is_registered_identifies_xml_of_both_collections(self): + created = self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + for xml_with_pre in (scl_xml(), psi_xml()): + with self.subTest(v2=xml_with_pre.v2): + response = PidProviderXML.is_registered(xml_with_pre) + self.assertTrue(response["registered"]) + self.assertEqual(response["v3"], created["v3"]) + # a resposta traz o pid v2 do XML de entrada + self.assertEqual(response["v2"], xml_with_pre.v2) + + def test_pids_v2_of_both_collections_are_not_free(self): + self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + self.assertTrue(PidProviderXML._is_registered_pid(v2=SCL_V2)) + self.assertTrue(PidProviderXML._is_registered_pid(v2=PSI_V2)) + + +class RegisterThematicCollectionFirstTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """psi (temática) é registrado antes de scl (coleção principal)""" + + def test_second_xml_is_identified_as_the_same_article(self): + first = self.register(psi_xml(), "psi") + second = self.register(scl_xml(), "scl") + + self.assertEqual(first["event_status"], "created") + self.assertEqual(second["event_status"], "updated") + self.assert_same_article_registered_once(first, second) + + def test_main_collection_v2_becomes_the_main_v2(self): + first = self.register(psi_xml(), "psi") + self.assertEqual(first["v2"], PSI_V2) + + second = self.register(scl_xml(), "scl") + + registered = PidProviderXML.objects.get() + self.assertEqual(registered.v2, SCL_V2) + self.assertEqual(second["v2"], SCL_V2) + self.assert_pids_v2_by_collection(registered) + self.assert_no_pid_v2_change(registered) + + def test_thematic_collection_does_not_replace_main_v2_afterwards(self): + self.register(psi_xml(), "psi") + self.register(scl_xml(), "scl") + self.register(updated_xml(PSI_V2, "psicousp", "psi"), "psi") + + registered = PidProviderXML.objects.get() + self.assertEqual(registered.v2, SCL_V2) + self.assert_pids_v2_by_collection(registered) + + +class RegisterWithoutMainCollectionTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """nenhuma das coleções é classificada como scielonetwork""" + + def setUp(self): + super().setUp() + Collection.objects.update(network_classification=None) + + def test_main_v2_is_not_registered(self): + first = self.register(psi_xml(), "psi") + second = self.register(scl_xml(), "scl") + + # a resposta traz o pid v2 do XML recebido + self.assertEqual(first["v2"], PSI_V2) + self.assertEqual(second["v2"], SCL_V2) + registered = PidProviderXML.objects.get() + # sem coleção principal, v2 não é completado + self.assertIsNone(registered.v2) + for xml_with_pre in (psi_xml(), scl_xml()): + with self.subTest(v2=xml_with_pre.v2): + response = PidProviderXML.is_registered(xml_with_pre) + self.assertTrue(response["registered"]) + self.assertEqual(response["v2"], xml_with_pre.v2) + self.assert_pids_v2_by_collection(registered, main=None) + self.assert_no_pid_v2_change(registered) + + +@override_settings(COLLECTION_ENSURE_NETWORK_CLASSIFICATION=True) +class RegisterCompletesNetworkClassificationTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """ + coleções sem network_classification: o registro completa o dado + (articlemeta) antes de identificar a coleção principal + """ + + def setUp(self): + super().setUp() + Collection.objects.update(network_classification=None) + cache.delete(ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY) + self.addCleanup(cache.delete, ENSURE_NETWORK_CLASSIFICATION_CACHE_KEY) + patcher = patch( + "collection.models.fetch_data", + return_value=[ + {"acron": "scl", "network_classification": ["scielonetwork"]}, + {"acron": "psi", "network_classification": ["thematic"]}, + ], + ) + self.mock_fetch_data = patcher.start() + self.addCleanup(patcher.stop) + + def test_main_collection_is_identified(self): + first = self.register(psi_xml(), "psi") + second = self.register(scl_xml(), "scl") + + # consulta o articlemeta somente uma vez + self.mock_fetch_data.assert_called_once() + self.assertTrue( + Collection.objects.get(acron3="scl").is_national_journal_collection + ) + self.assertFalse( + Collection.objects.get(acron3="psi").is_national_journal_collection + ) + + registered = self.assert_same_article_registered_once(first, second) + self.assertEqual(registered.v2, SCL_V2) + self.assert_pids_v2_by_collection(registered) + self.assert_current_version_is_from(registered, self.scl) + + +class RegisterWithoutCollectionInXMLTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """XML não informa a coleção de origem""" + + def test_no_version_is_registered(self): + response = self.register(make_xml_with_pre(SCL_V2, "pusp"), "scl") + + registered = PidProviderXML.objects.get() + self.assertEqual(response["v2"], SCL_V2) + self.assertEqual( + set(registered.collections.values_list("acron3", flat=True)), + {"scl", "psi"}, + ) + self.assertFalse(CollectionPidV2.objects.exists()) + self.assertIsNone(registered.current_version) + self.assertFalse(XMLVersion.objects.exists()) + + def test_unidentified_collection_is_not_registered(self): + SciELOJournal.objects.all().delete() + + response = PidProviderXML.register( + make_xml_with_pre(SCL_V2, "pusp"), + "scl", + self.user, + origin_date="2026-01-01", + ) + + self.assertEqual(response["event_status"], "error") + self.assertEqual( + response["error_type"], + str(exceptions.RequiredCollectionErrorToGetPidProviderXMLError), + ) + self.assertFalse(PidProviderXML.objects.exists()) + self.assertFalse(CollectionPidV2.objects.exists()) + self.assertFalse(XMLVersion.objects.exists()) + + +class XMLVersionByCollectionTest(RegisterSameArticleInDifferentCollectionsTestBase): + """ + O XML de cada coleção chega em momentos diferentes na migração: + cada coleção tem sua versão atual e a versão atual do documento é a da + coleção principal + """ + + def test_each_collection_has_its_own_current_version(self): + self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + registered = PidProviderXML.objects.get() + scl_version = self.collection_current_version(registered, self.scl) + psi_version = self.collection_current_version(registered, self.psi) + self.assertNotEqual(scl_version, psi_version) + self.assertEqual(scl_version.xml_with_pre.v2, SCL_V2) + self.assertEqual(psi_version.xml_with_pre.v2, PSI_V2) + self.assertEqual(scl_version.xml_with_pre.collection, "scl") + self.assertEqual(psi_version.xml_with_pre.collection, "psi") + + def test_current_version_is_the_main_collection_version(self): + self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + registered = PidProviderXML.objects.get() + self.assert_current_version_is_from(registered, self.scl) + self.assertEqual(registered.xml_with_pre.v2, SCL_V2) + + def test_main_collection_version_becomes_current_version(self): + self.register(psi_xml(), "psi") + registered = PidProviderXML.objects.get() + # somente a coleção principal completa current_version + self.assertIsNone(registered.current_version) + self.assertIsNotNone(self.collection_current_version(registered, self.psi)) + + self.register(scl_xml(), "scl") + + self.assert_current_version_is_from(registered, self.scl) + + def test_same_xml_of_the_collection_is_skipped(self): + self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + total = XMLVersion.objects.count() + + responses = [self.register(scl_xml(), "scl"), self.register(psi_xml(), "psi")] + + self.assertEqual( + [response["event_status"] for response in responses], + ["skipped", "skipped"], + ) + self.assertEqual(XMLVersion.objects.count(), total) + + def test_is_registered_compares_with_version_of_the_same_collection(self): + response = self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + # XML com o pid v3 atribuído (como no pacote após o registro) + for xml_with_pre in (scl_xml(), psi_xml()): + xml_with_pre.v3 = response["v3"] + with self.subTest(v2=xml_with_pre.v2): + self.assertTrue( + PidProviderXML.is_registered(xml_with_pre)["is_equal"] + ) + + def test_get_xml_with_pre_by_collection(self): + response = self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + v3 = response["v3"] + + self.assertEqual(PidProviderXML.get_xml_with_pre(v3).v2, SCL_V2) + self.assertEqual(PidProviderXML.get_xml_with_pre(v3, self.scl).v2, SCL_V2) + self.assertEqual(PidProviderXML.get_xml_with_pre(v3, self.psi).v2, PSI_V2) + self.assertEqual( + PidProviderXML.get_xml_with_pre(v3, collection_acron="psi").v2, PSI_V2 + ) + + def test_get_xml_with_pre_of_collection_without_version(self): + response = self.register(scl_xml(), "scl") + + self.assertIsNone(PidProviderXML.get_xml_with_pre(response["v3"], self.psi)) + + +class XMLVersionWithoutMainCollectionTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + def setUp(self): + super().setUp() + Collection.objects.update(network_classification=None) + + def test_current_version_is_not_registered(self): + self.register(psi_xml(), "psi") + self.register(scl_xml(), "scl") + + registered = PidProviderXML.objects.get() + # sem coleção principal, current_version não é completado + self.assertIsNone(registered.current_version) + self.assertIsNone(PidProviderXML.get_xml_with_pre(registered.v3)) + self.assertEqual( + PidProviderXML.get_xml_with_pre(registered.v3, self.psi).v2, PSI_V2 + ) + self.assertEqual( + PidProviderXML.get_xml_with_pre(registered.v3, self.scl).v2, SCL_V2 + ) + + def test_update_of_the_same_collection_updates_its_version(self): + self.register(psi_xml(), "psi") + self.register(scl_xml(), "scl") + updated = updated_xml(PSI_V2, "psicousp", "psi") + + self.register(updated, "psi") + + registered = PidProviderXML.objects.get() + self.assertEqual( + registered.get_current_version(self.psi).finger_print, + updated.finger_print, + ) + self.assertNotEqual( + registered.get_current_version(self.scl).finger_print, + updated.finger_print, + ) + + +class XMLVersionOfJournalWithSameDataInCollectionsTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """ + Periódico com o mesmo PID e acrônimo em todas as coleções: o XML de + cada coleção difere somente na coleção de origem + """ + + def setUp(self): + super().setUp() + SciELOJournal.objects.filter(collection=self.psi).update( + issn_scielo=ISSN_PRINT, journal_acron="pusp" + ) + + def test_each_collection_is_registered_when_its_xml_arrives(self): + self.register(scl_xml(), "scl") + + registered = PidProviderXML.objects.get() + self.assertEqual(registered.collections.count(), 2) + self.assertEqual(registered.collection_pids_v2_data, [SCL_DATA]) + + self.register(make_xml_with_pre(SCL_V2, "pusp", "psi"), "psi") + + registered = PidProviderXML.objects.get() + self.assertEqual( + { + (item.collection.acron3, item.pid_v2, item.current_version.xml_with_pre.collection) + for item in CollectionPidV2.objects.all() + }, + {("scl", SCL_V2, "scl"), ("psi", SCL_V2, "psi")}, + ) + self.assert_current_version_is_from(registered, self.scl) + self.assert_no_pid_v2_change(registered) + + +class XMLVersionOfJournalWithDifferentAcronInCollectionsTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """ + Periódico com o mesmo PID, mas acrônimos diferentes nas coleções + (no site novo, o acrônimo é a chave do periódico): o XML de cada + coleção difere no journal-id + """ + + def setUp(self): + super().setUp() + SciELOJournal.objects.filter(collection=self.psi).update(issn_scielo=ISSN_PRINT) + + def test_different_article_pid_v2_with_same_journal_pid(self): + # pid v2 do artigo depende também do pid do fascículo e da ordem + psi_v2 = "S0103-65642009000300007" + self.register(make_xml_with_pre(SCL_V2, "pusp", "scl"), "scl") + self.register(make_xml_with_pre(psi_v2, "psicousp", "psi"), "psi") + + registered = PidProviderXML.objects.get() + self.assertEqual(registered.v2, SCL_V2) + self.assert_no_pid_v2_change(registered) + self.assertCountEqual( + registered.collection_pids_v2_data, + [SCL_DATA, pid_v2_data(psi_v2, "psicousp", "psi", False)], + ) + self.assertEqual( + PidProviderXML.get_xml_with_pre(registered.v3, self.psi).v2, psi_v2 + ) + self.assertEqual( + PidProviderXML.get_xml_with_pre(registered.v3, self.scl).v2, SCL_V2 + ) + + def test_same_pid_v2_with_different_journal_acron(self): + self.register(make_xml_with_pre(SCL_V2, "pusp", "scl"), "scl") + self.register(make_xml_with_pre(SCL_V2, "psicousp", "psi"), "psi") + + registered = PidProviderXML.objects.get() + self.assertEqual(registered.v2, SCL_V2) + self.assertEqual( + { + (item.collection.acron3, item.journal_acron, item.pid_v2) + for item in CollectionPidV2.objects.all() + }, + {("scl", "pusp", SCL_V2), ("psi", "psicousp", SCL_V2)}, + ) + self.assertEqual( + PidProviderXML.get_xml_with_pre(registered.v3, self.psi).journal_acron, + "psicousp", + ) + self.assertEqual( + PidProviderXML.get_xml_with_pre(registered.v3, self.scl).journal_acron, + "pusp", + ) + + +class MigrationCollectionVersionsTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """ + Migração: o XML de cada coleção (pid v2 e journal-id da coleção) + cria / atualiza somente a versão desta coleção + """ + + def assert_collection_xml(self, registered, collection, acron, pid_v2): + xml_with_pre = PidProviderXML.get_xml_with_pre(registered.v3, collection) + self.assertEqual(xml_with_pre.journal_acron, acron) + self.assertEqual(xml_with_pre.v2, pid_v2) + self.assertEqual(xml_with_pre.v3, registered.v3) + self.assertEqual(xml_with_pre.collection, collection.acron3) + return xml_with_pre + + def test_other_collections_are_registered_when_their_xml_arrives(self): + self.register(scl_xml(), "scl") + + registered = PidProviderXML.objects.get() + self.assertFalse( + CollectionPidV2.objects.filter(collection=self.psi).exists() + ) + + self.register(psi_xml(), "psi") + + self.assert_collection_xml(registered, self.scl, "pusp", SCL_V2) + self.assert_collection_xml(registered, self.psi, "psicousp", PSI_V2) + + def test_xml_of_a_collection_updates_only_its_version(self): + self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + self.register(updated_xml(SCL_V2, "pusp", "scl"), "scl") + + registered = PidProviderXML.objects.get() + scl = self.assert_collection_xml(registered, self.scl, "pusp", SCL_V2) + psi = self.assert_collection_xml(registered, self.psi, "psicousp", PSI_V2) + self.assertIn(UPDATED_TITLE, scl.tostring()) + self.assertNotIn(UPDATED_TITLE, psi.tostring()) + self.assert_current_version_is_from(registered, self.scl) + + def test_xml_collection_is_identified_by_custom_meta(self): + self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + + # acrônimo não corresponde a nenhuma coleção + self.register(updated_xml(SCL_V2, "psicologiausp", "scl"), "scl") + + registered = PidProviderXML.objects.get() + scl = self.assert_collection_xml(registered, self.scl, "psicologiausp", SCL_V2) + psi = self.assert_collection_xml(registered, self.psi, "psicousp", PSI_V2) + self.assertIn(UPDATED_TITLE, scl.tostring()) + self.assertNotIn(UPDATED_TITLE, psi.tostring()) + + def test_same_xml_is_skipped(self): + first = scl_xml() + self.register(first, "scl") + total = XMLVersion.objects.count() + + again = scl_xml() + again.v3 = first.v3 + response = self.register(again, "scl") + + self.assertEqual(response["event_status"], "skipped") + self.assertEqual(XMLVersion.objects.count(), total) + + +class ErratumInDifferentCollectionsTest( + RegisterSameArticleInDifferentCollectionsTestBase +): + """ + A errata não é uma nova versão do artigo: é outro documento, com pid v2 + próprio, que aponta para o artigo corrigido (related-article). + Registrar a errata não altera o registro do artigo. + """ + + def setUp(self): + super().setUp() + self.register(scl_xml(), "scl") + self.register(psi_xml(), "psi") + self.article = PidProviderXML.objects.get() + + def get_erratum(self): + return PidProviderXML.objects.exclude(pk=self.article.pk).get() + + def test_erratum_is_registered_as_another_document(self): + response = self.register( + erratum_xml(SCL_ERRATUM_V2, "pusp", "scl"), "scl" + ) + + self.assertEqual(PidProviderXML.objects.count(), 2) + erratum = self.get_erratum() + self.assertEqual(response["v3"], erratum.v3) + self.assertNotEqual(erratum.v3, self.article.v3) + self.assertEqual(erratum.v2, SCL_ERRATUM_V2) + + def test_erratum_of_both_collections_is_registered_once(self): + first = self.register(erratum_xml(SCL_ERRATUM_V2, "pusp", "scl"), "scl") + second = self.register( + erratum_xml(PSI_ERRATUM_V2, "psicousp", "psi"), "psi" + ) + + self.assertEqual(PidProviderXML.objects.count(), 2) + erratum = self.get_erratum() + self.assertEqual(first["v3"], erratum.v3) + self.assertEqual(second["v3"], erratum.v3) + self.assertEqual(erratum.v2, SCL_ERRATUM_V2) + self.assertCountEqual( + erratum.collection_pids_v2_data, + [ + pid_v2_data(SCL_ERRATUM_V2, "pusp", "scl", True), + pid_v2_data(PSI_ERRATUM_V2, "psicousp", "psi", False), + ], + ) + self.assert_no_pid_v2_change(erratum) + + def test_erratum_does_not_change_the_article(self): + article_versions = { + collection.acron3: self.collection_current_version( + self.article, collection + ).pk + for collection in (self.scl, self.psi) + } + + self.register(erratum_xml(SCL_ERRATUM_V2, "pusp", "scl"), "scl") + self.register(erratum_xml(PSI_ERRATUM_V2, "psicousp", "psi"), "psi") + + self.article.refresh_from_db() + self.assertEqual(self.article.v2, SCL_V2) + self.assert_pids_v2_by_collection(self.article) + self.assert_no_pid_v2_change(self.article) + self.assertEqual( + { + collection.acron3: self.collection_current_version( + self.article, collection + ).pk + for collection in (self.scl, self.psi) + }, + article_versions, + ) diff --git a/pid_provider/tests/test_register_shared_identifiers.py b/pid_provider/tests/test_register_shared_identifiers.py new file mode 100644 index 000000000..f69707187 --- /dev/null +++ b/pid_provider/tests/test_register_shared_identifiers.py @@ -0,0 +1,317 @@ +""" +Testes de integração de PidProviderXML.register quando artigos diferentes +compartilham um identificador que deveria ser exclusivo. + +Reproduz o problema relatado na execução de +`task_load_records_from_counter_dict` para o periódico BJM da coleção scl: +`PidProvider.provide_pid_for_xml_uri` falhou com +QueryDocumentMultipleObjectsReturnedError porque no PidProviderXML há mais de +um registro com o mesmo: + +- aop_pid: S1517-83822015005040098 está registrado para + vTQdGGLzjLB9t9bkv7KRkhw (S1517-83822015000300759) e para + 6tVtNpdVbNjLPFLFptwhDSH +- pkg_name: o nome de pacote de n4YgC9hXSXqDGJSzrRBBqJM + (S1517-83822013000200047) está registrado também para + 9fWvbZ7pXXKWr4HTCtqjjjq + +Nos dois casos, os demais identificadores (pid v3 e pid v2) e o conteúdo do +XML são suficientes para identificar um único registro, então o registro +deve ser encontrado e atualizado, sem erro. +""" + +from tempfile import TemporaryDirectory + +from django.contrib.auth import get_user_model +from django.test import TestCase, override_settings +from packtools.sps.pid_provider.xml_sps_lib import XMLWithPre + +from collection.models import Collection +from journal.models import Journal, OfficialJournal, SciELOJournal +from pid_provider.models import PidProviderXML + +User = get_user_model() + +ISSN_PRINT = "1517-8382" +ISSN_ELECTRONIC = "1678-4405" + +# mesmo aop_pid registrado para artigos diferentes +AOP_PID = "S1517-83822015005040098" +AOP_ARTICLE = dict( + v3="vTQdGGLzjLB9t9bkv7KRkhw", + v2="S1517-83822015000300759", + aop_pid=AOP_PID, + doi="10.1590/S1517-838246320140359", + year="2015", + volume="46", + issue="3", + fpage="759", + lpage="766", + title="Antimicrobial activity of essential oils against foodborne bacteria", + surnames=("Oliveira", "Pereira"), + body=( + "Essential oils were extracted by hydrodistillation and tested " + "against Staphylococcus aureus and Escherichia coli isolated from " + "food samples." + ), +) +OTHER_AOP_ARTICLE = dict( + v3="6tVtNpdVbNjLPFLFptwhDSH", + v2="S1517-83822015000300767", + aop_pid=AOP_PID, + doi="10.1590/S1517-838246320140412", + year="2015", + volume="46", + issue="3", + fpage="767", + lpage="775", + title="Biofilm formation by Pseudomonas aeruginosa on stainless steel", + surnames=("Santos", "Almeida"), + body=( + "Biofilm formation was evaluated on stainless steel coupons exposed " + "to clinical isolates of Pseudomonas aeruginosa under static " + "conditions." + ), +) + +# mesmo pkg_name registrado para artigos diferentes +PKG_NAME_ARTICLE = dict( + v3="n4YgC9hXSXqDGJSzrRBBqJM", + v2="S1517-83822013000200047", + aop_pid=None, + doi="10.1590/S1517-83822013000200047", + year="2013", + volume="44", + issue="2", + fpage="347", + lpage="352", + title="Detection of Salmonella spp. in poultry carcasses by real-time PCR", + surnames=("Costa", "Ferreira"), + body=( + "Poultry carcasses were collected from slaughterhouses and analysed " + "by real-time PCR targeting the invA gene of Salmonella spp." + ), +) +OTHER_PKG_NAME_ARTICLE = dict( + v3="9fWvbZ7pXXKWr4HTCtqjjjq", + v2="S1517-83822013000200048", + aop_pid=None, + doi="10.1590/S1517-83822013000200048", + year="2013", + volume="44", + issue="2", + fpage="353", + lpage="360", + title="Diversity of endophytic fungi isolated from Amazonian medicinal plants", + surnames=("Lima", "Rodrigues"), + body=( + "Endophytic fungi were isolated from leaves of medicinal plants " + "collected in the Amazon region and identified by ITS sequencing." + ), +) + +XML_TEMPLATE = """ +
+ + + bjm + + Brazilian Journal of Microbiology + + 1517-8382 + 1678-4405 + + + {v3} + {v2} + {aop_pid_node} + {doi} + + + Research Paper + + + + {title} + + + {contribs} + + + 21 + 07 + {year} + + + {year} + + {volume} + {issue} + {fpage} + {lpage} + + + +

{body}

+ +
+""" + +CONTRIB_TEMPLATE = """ + + {surname} + A. + + """ + + +def make_xml_with_pre(article): + aop_pid = article.get("aop_pid") + aop_pid_node = ( + f'{aop_pid}' + if aop_pid + else "" + ) + contribs = "\n ".join( + CONTRIB_TEMPLATE.format(surname=surname) for surname in article["surnames"] + ) + xml_content = XML_TEMPLATE.format( + aop_pid_node=aop_pid_node, + contribs=contribs, + **{k: v for k, v in article.items() if k not in ("aop_pid", "surnames")}, + ) + xml_with_pre = list(XMLWithPre.create(xml_content=xml_content))[0] + xml_with_pre.collection = "scl" + return xml_with_pre + + +class RegisterWithIdentifierSharedByOtherArticleMixin: + # artigo que está sendo carregado e o outro artigo que, no + # PidProviderXML, tem o mesmo identificador + article = None + other_article = None + + def setUp(self): + self.media_directory = TemporaryDirectory() + self.override_settings = override_settings( + MEDIA_ROOT=self.media_directory.name + ) + self.override_settings.enable() + self.addCleanup(self.override_settings.disable) + self.addCleanup(self.media_directory.cleanup) + + self.user = User.objects.create_user(username="shared-identifiers") + scl = Collection.objects.create( + acron3="scl", network_classification=["scielonetwork"], creator=self.user + ) + official_journal = OfficialJournal.objects.create( + title="Brazilian Journal of Microbiology", + issn_print=ISSN_PRINT, + issn_electronic=ISSN_ELECTRONIC, + creator=self.user, + ) + journal = Journal.objects.create( + official=official_journal, creator=self.user + ) + SciELOJournal.objects.create( + journal=journal, + collection=scl, + journal_acron="bjm", + issn_scielo=ISSN_PRINT, + creator=self.user, + ) + + for article in (self.article, self.other_article): + self.assert_no_error( + self.register(make_xml_with_pre(article), f"scl_{article['v3']}") + ) + self.share_identifier() + self.assertEqual(PidProviderXML.objects.count(), 2) + + def share_identifier(self): + """ + Reproduz o estado do PidProviderXML: o outro artigo com o mesmo + identificador do artigo + """ + raise NotImplementedError + + def register(self, xml_with_pre, filename): + # como em provide_pid_for_xml_uri (ver Detail do Unexpected Event) + return PidProviderXML.register( + xml_with_pre, filename, self.user, origin_date="2021-08-31" + ) + + def assert_no_error(self, response): + self.assertIsNone( + response.get("error_type"), + f"{response.get('error_msg')} {response.get('select_record_response')}", + ) + + def test_register_identifies_the_article(self): + response = self.register( + make_xml_with_pre(self.article), f"scl_{self.article['v3']}" + ) + + self.assert_no_error(response) + self.assertNotEqual(response.get("event_status"), "multiple") + self.assertEqual(response["v3"], self.article["v3"]) + self.assertEqual(response["v2"], self.article["v2"]) + # não cria novo registro nem altera o outro artigo + self.assertEqual(PidProviderXML.objects.count(), 2) + other = PidProviderXML.objects.get(v3=self.other_article["v3"]) + self.assertEqual(other.v2, self.other_article["v2"]) + + def test_register_identifies_the_other_article(self): + response = self.register( + make_xml_with_pre(self.other_article), + f"scl_{self.other_article['v3']}", + ) + + self.assert_no_error(response) + self.assertEqual(response["v3"], self.other_article["v3"]) + self.assertEqual(response["v2"], self.other_article["v2"]) + self.assertEqual(PidProviderXML.objects.count(), 2) + + def test_is_registered_identifies_the_article(self): + response = PidProviderXML.is_registered(make_xml_with_pre(self.article)) + + self.assert_no_error(response) + self.assertTrue(response["registered"]) + self.assertEqual(response["v3"], self.article["v3"]) + self.assertEqual(response["v2"], self.article["v2"]) + + +class RegisterWithAopPidSharedByOtherArticleTest( + RegisterWithIdentifierSharedByOtherArticleMixin, TestCase +): + """ + vTQdGGLzjLB9t9bkv7KRkhw e 6tVtNpdVbNjLPFLFptwhDSH com o mesmo aop_pid + """ + + article = AOP_ARTICLE + other_article = OTHER_AOP_ARTICLE + + def share_identifier(self): + PidProviderXML.objects.filter( + v3__in=(self.article["v3"], self.other_article["v3"]) + ).update(aop_pid=AOP_PID) + self.assertEqual(PidProviderXML.objects.filter(aop_pid=AOP_PID).count(), 2) + + +class RegisterWithPkgNameSharedByOtherArticleTest( + RegisterWithIdentifierSharedByOtherArticleMixin, TestCase +): + """ + n4YgC9hXSXqDGJSzrRBBqJM e 9fWvbZ7pXXKWr4HTCtqjjjq com o mesmo pkg_name + """ + + article = PKG_NAME_ARTICLE + other_article = OTHER_PKG_NAME_ARTICLE + + def share_identifier(self): + pkg_name = PidProviderXML.objects.get(v3=self.article["v3"]).pkg_name + self.assertTrue(pkg_name) + PidProviderXML.objects.filter(v3=self.other_article["v3"]).update( + pkg_name=pkg_name + ) + self.assertEqual(PidProviderXML.objects.filter(pkg_name=pkg_name).count(), 2) diff --git a/requirements/base.txt b/requirements/base.txt index 6abe0893b..f25d8a522 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -91,7 +91,7 @@ opensearch-py==3.2.0 # https://github.com/opensearch-project/opensearch-py # ------------------------------------------------------------------------------ tornado>=6.5.2 # not directly required, pinned by Snyk to avoid a vulnerability lxml==6.1.2 # https://github.com/lxml/lxml -git+https://git@github.com/scieloorg/packtools@4.17.6#egg=packtools +git+https://git@github.com/scieloorg/packtools@4.17.8#egg=packtools # pymongo # ------------------------------------------------------------------------------