diff --git a/article/migrations/0051_articlecount_article.py b/article/migrations/0051_articlecount_article.py new file mode 100644 index 000000000..4ab485ac7 --- /dev/null +++ b/article/migrations/0051_articlecount_article.py @@ -0,0 +1,76 @@ +import django.db.models.deletion +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("article", "0050_remove_articlesource_am_article_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="articlecount", + name="article", + field=models.ForeignKey( + blank=True, + null=True, + on_delete=django.db.models.deletion.CASCADE, + related_name="counts", + to="article.article", + ), + ), + migrations.AddIndex( + model_name="articlecount", + index=models.Index( + fields=["article"], name="article_art_article_8968b6_idx" + ), + ), + migrations.AlterUniqueTogether( + name="articlecount", + unique_together={("article", "count_type", "language")}, + ), + migrations.RemoveField( + model_name="articlecount", + name="created", + ), + migrations.RemoveField( + model_name="articlecount", + name="updated", + ), + migrations.RemoveField( + model_name="articlecount", + name="creator", + ), + migrations.RemoveField( + model_name="articlecount", + name="updated_by", + ), + migrations.RemoveField( + model_name="articlecounttype", + name="created", + ), + migrations.RemoveField( + model_name="articlecounttype", + name="creator", + ), + migrations.RemoveField( + model_name="articlecounttype", + name="updated", + ), + migrations.RemoveField( + model_name="articlecounttype", + name="updated_by", + ), + migrations.AlterField( + model_name="articlecounttype", + name="code", + field=models.CharField( + blank=True, + max_length=20, + null=True, + unique=True, + verbose_name="Code", + ), + ), + ] diff --git a/article/models.py b/article/models.py index 8e9b8b543..7ba19be59 100755 --- a/article/models.py +++ b/article/models.py @@ -378,6 +378,28 @@ def xmltree(self): except AttributeError: return PidProvider.get_xmltree(self.pid_v3) + def create_or_update_article_visual_counts(self, user, xmltree=None, errors=None): + """ + Calcula/Recalcula fig, table-wrap, fórmulas e graphics a partir do XML do artigo. + """ + # Evita importação circular. + from article.sources.xmlsps import create_or_update_article_counts + + if errors is None: + errors = [] + if xmltree is None: + pp_xml = self.pp_xml + if not pp_xml: + raise ValueError(f"Article {self.pk} has no associated XML") + xml_with_pre = pp_xml.xml_with_pre + if not xml_with_pre: + raise ValueError( + f"Unable to get XML to count visual elements from {pp_xml}" + ) + xmltree = xml_with_pre.xmltree + create_or_update_article_counts(xmltree, self, user, errors) + return errors + @cached_property def collections(self): """ @@ -1378,8 +1400,23 @@ def data(self): ) -class ArticleCountType(CommonControlField): - code = models.CharField(_("Code"), blank=True, null=True, max_length=20) +class ArticleCountType(models.Model): + TYPE_FIG = "fig" + TYPE_DISP_FORMULA = "disp-formula" + TYPE_TABLE_WRAP = "table-wrap" + TYPE_GRAPHIC = "graphic" + TYPE_INLINE_GRAPHIC = "inline-graphic" + TYPE_INLINE_FORMULA = "inline-formula" + VISUAL_TYPES = ( + TYPE_FIG, + TYPE_DISP_FORMULA, + TYPE_TABLE_WRAP, + TYPE_GRAPHIC, + TYPE_INLINE_GRAPHIC, + TYPE_INLINE_FORMULA, + ) + + code = models.CharField(_("Code"), blank=True, null=True, max_length=20, unique=True) class Meta: indexes = [ @@ -1396,24 +1433,14 @@ def __unicode__(self): def __str__(self): return "%s" % self.code - @property - def data(self): - return dict(article_count_type__code=self.code) - - @classmethod - def get_or_create(cls, code, user): - try: - return cls.objects.get(code=code) - except cls.DoesNotExist: - article_count_type = cls() - article_count_type.code = code - article_count_type.creator = user - article_count_type.save() - - return article_count_type - - -class ArticleCount(CommonControlField): +class ArticleCount(models.Model): + article = models.ForeignKey( + Article, + null=True, + blank=True, + on_delete=models.CASCADE, + related_name="counts", + ) count_type = models.ForeignKey( ArticleCountType, null=True, blank=True, on_delete=models.SET_NULL ) @@ -1426,10 +1453,21 @@ class ArticleCount(CommonControlField): blank=True, ) - panels = [AutocompletePanel("count_type"), AutocompletePanel("language")] + panels = [ + FieldPanel("article", read_only=True), + FieldPanel("count_type", read_only=True), + FieldPanel("count", read_only=True), + FieldPanel("language", read_only=True), + ] class Meta: + unique_together = [("article", "count_type", "language")] indexes = [ + models.Index( + fields=[ + "article", + ] + ), models.Index( fields=[ "count_type", @@ -1443,10 +1481,10 @@ class Meta: ] def __unicode__(self): - return "%s | %s | %s" % (self.count_type, self.count, self.language) + return f"{self.article} | {self.count_type} | {self.count} | {self.language}" def __str__(self): - return "%s | %s | %s" % (self.count_type, self.count, self.language) + return f"{self.article} | {self.count_type} | {self.count} | {self.language}" @property def data(self): diff --git a/article/sources/xmlsps.py b/article/sources/xmlsps.py index be0530988..11e70f2aa 100755 --- a/article/sources/xmlsps.py +++ b/article/sources/xmlsps.py @@ -1,4 +1,4 @@ -import traceback +from collections import Counter from datetime import datetime from itertools import product @@ -12,17 +12,24 @@ from packtools.sps.models.article_license import ArticleLicense from packtools.sps.models.article_titles import ArticleTitles from packtools.sps.models.dates import ArticleDates +from packtools.sps.models.fig import ArticleFigs +from packtools.sps.models.formula import ArticleFormulas from packtools.sps.models.front_articlemeta_issue import ArticleMetaIssue from packtools.sps.models.funding_group import FundingGroup +from packtools.sps.models.graphic import Graphic, InlineGraphic, XmlGraphic from packtools.sps.models.journal_meta import ISSN, Title from packtools.sps.models.kwd_group import ArticleKeywords +from packtools.sps.models.tablewrap import ArticleTableWrappers from packtools.sps.models.v2.article_toc_sections import ArticleTocSections from packtools.sps.models.v2.related_articles import RelatedArticles +from packtools.sps.models.visual_resource_base import XmlVisualResource from article import choices from article.models import ( Article, ArticleAffiliation, + ArticleCount, + ArticleCountType, ArticleFunding, ContribCollab, ContribPerson, @@ -224,6 +231,10 @@ def load_article(user, pp_xml): add_related_articles(xmltree=xmltree, article=article, user=user, errors=errors) + create_or_update_article_counts( + xmltree=xmltree, article=article, user=user, errors=errors + ) + article.create_legacy_keys(user) if not article.pid_v2: raise ValueError(f"Article has no PID v2: {article.pid_v3}") @@ -1064,3 +1075,108 @@ def add_related_articles(xmltree, article, user, errors): except Exception as e: add_error(errors, "add_related_articles", e) + + +class GraphicWithNode(Graphic): + @property + def data(self): + data = super().data + data["node"] = self.node + return data + + +class InlineGraphicWithNode(InlineGraphic): + @property + def data(self): + data = super().data + data["node"] = self.node + return data + + +class XmlGraphicWithNode(XmlGraphic): + def __init__(self, xmltree): + resource_types = [ + ("graphic", GraphicWithNode), + ("inline-graphic", InlineGraphicWithNode), + ] + XmlVisualResource.__init__(self, xmltree, resource_types=resource_types) + + +def count_items_by_parent_lang(items, count_type, counter): + for item in items: + counter[(count_type, item.get("parent_lang") or None)] += 1 + +def count_graphics_and_inline_graphics(xmltree, counter): + """ + Conta graphics e inline-graphics por idioma do artigo, + ignorando graphics que são descendentes de fig/table-wrap. + """ + for item in XmlGraphicWithNode(xmltree).data: + tag = item.get("tag") + lang = item.get("parent_lang") or None + if tag == ArticleCountType.TYPE_INLINE_GRAPHIC: + counter[(ArticleCountType.TYPE_INLINE_GRAPHIC, lang)] += 1 + elif tag == ArticleCountType.TYPE_GRAPHIC: + node = item.get("node") + if node is not None and any( + ancestor.tag in ("fig", "table-wrap") for ancestor in node.iterancestors() + ): + continue + counter[(ArticleCountType.TYPE_GRAPHIC, lang)] += 1 + +def count_visual_and_formula_items(xmltree): + """ + Conta fig, table-wrap, fórmulas e graphics por idioma do article/sub-article. + """ + counter = Counter() + count_items_by_parent_lang( + items=ArticleFigs(xmltree).get_all_figs, + count_type=ArticleCountType.TYPE_FIG, + counter=counter, + ) + count_items_by_parent_lang( + items=ArticleTableWrappers(xmltree).get_all_table_wrappers, + count_type=ArticleCountType.TYPE_TABLE_WRAP, + counter=counter, + ) + formulas = ArticleFormulas(xmltree) + count_items_by_parent_lang( + items=formulas.disp_formula_items, + count_type=ArticleCountType.TYPE_DISP_FORMULA, + counter=counter, + ) + count_items_by_parent_lang( + items=formulas.inline_formula_items, + count_type=ArticleCountType.TYPE_INLINE_FORMULA, + counter=counter, + ) + count_graphics_and_inline_graphics(xmltree, counter) + + return counter + + +def create_or_update_article_counts(xmltree, article, user, errors): + """ + Substitui os totais visuais/fórmulas do artigo a partir do xmltree. + """ + counter = count_visual_and_formula_items(xmltree) + kept_ids = [] + for (count_type_code, lang), count in counter.items(): + language = get_or_create_language(lang, user, errors) if lang else None + if lang and not language: + add_error( + errors, + "create_or_update_article_counts", + ValueError(f"Missing language for {count_type_code}"), + count_type=count_type_code, + lang=lang, + ) + count_type, _ = ArticleCountType.objects.get_or_create(code=count_type_code) + article_count, _ = ArticleCount.objects.update_or_create( + article=article, + count_type=count_type, + language=language, + defaults={"count": count}, + ) + kept_ids.append(article_count.pk) + article.counts.exclude(pk__in=kept_ids).delete() diff --git a/article/tasks.py b/article/tasks.py index bf3dd356c..99f22c65a 100644 --- a/article/tasks.py +++ b/article/tasks.py @@ -966,3 +966,30 @@ def task_process_article_pipeline( }, ) raise + + +@celery_app.task(bind=True) +def task_create_article_visual_counts(self, user_id=None, username=None, article_id=None): + """ + Preenche os totais visuais dos artigos já carregados que têm XML. + """ + user = _get_user(self.request, username=username, user_id=user_id) + articles = Article.objects.filter(pp_xml__isnull=False) + if article_id: + articles = articles.filter(pk=article_id) + for article in articles.iterator(): + logging.info( + f"Starting visual counts for article id={article.pk} ({article})" + ) + try: + article.create_or_update_article_visual_counts(user) + except Exception as e: + exc_type, exc_value, exc_traceback = sys.exc_info() + UnexpectedEvent.create( + action="article.tasks.task_create_article_visual_counts", + item=str(article), + exception=e, + exc_traceback=exc_traceback, + detail={"article_id": article.id}, + ) + diff --git a/article/tests/test_task_update_article_visual_counts.py b/article/tests/test_task_update_article_visual_counts.py new file mode 100644 index 000000000..51bdeefbb --- /dev/null +++ b/article/tests/test_task_update_article_visual_counts.py @@ -0,0 +1,71 @@ +import logging +import unittest +from unittest.mock import MagicMock, patch + +from article.tasks import task_create_article_visual_counts + +MODULE_PATH = "article.tasks" + + +def setUpModule(): + logging.disable(logging.CRITICAL) + + +def tearDownModule(): + logging.disable(logging.NOTSET) + + +def make_user(user_id=1, username="counts-user"): + return MagicMock(id=user_id, username=username) + + +class TestTaskCreateArticleVisualCounts(unittest.TestCase): + def test_fills_counts_for_articles_with_xml_without_load_article(self): + user = make_user() + first = MagicMock(id=10) + second = MagicMock(id=11) + qs = MagicMock() + qs.iterator.return_value = iter([first, second]) + + with patch(f"{MODULE_PATH}._get_user", return_value=user), patch( + f"{MODULE_PATH}.Article" + ) as MockArticle, patch(f"{MODULE_PATH}.load_article") as mock_load_article: + MockArticle.objects.filter.return_value = qs + task_create_article_visual_counts() + + MockArticle.objects.filter.assert_called_once_with(pp_xml__isnull=False) + first.create_or_update_article_visual_counts.assert_called_once_with(user) + second.create_or_update_article_visual_counts.assert_called_once_with(user) + mock_load_article.assert_not_called() + + def test_article_error_is_logged_and_iteration_continues(self): + user = make_user() + failing = MagicMock(id=10) + failing.create_or_update_article_visual_counts.side_effect = ValueError("no xml") + following = MagicMock(id=11) + qs = MagicMock() + qs.iterator.return_value = iter([failing, following]) + + with patch(f"{MODULE_PATH}._get_user", return_value=user), patch( + f"{MODULE_PATH}.Article" + ) as MockArticle, patch(f"{MODULE_PATH}.UnexpectedEvent") as mock_event: + MockArticle.objects.filter.return_value = qs + task_create_article_visual_counts() + + mock_event.create.assert_called_once() + following.create_or_update_article_visual_counts.assert_called_once_with(user) + + def test_article_id_restricts_the_queryset(self): + user = make_user() + qs = MagicMock() + qs.filter.return_value = qs + qs.iterator.return_value = iter([]) + + with patch(f"{MODULE_PATH}._get_user", return_value=user), patch( + f"{MODULE_PATH}.Article" + ) as MockArticle: + MockArticle.objects.filter.return_value = qs + task_create_article_visual_counts(article_id=10) + + MockArticle.objects.filter.assert_called_once_with(pp_xml__isnull=False) + qs.filter.assert_called_once_with(pk=10) diff --git a/article/tests/test_visual_element_counts.py b/article/tests/test_visual_element_counts.py new file mode 100644 index 000000000..cdb373326 --- /dev/null +++ b/article/tests/test_visual_element_counts.py @@ -0,0 +1,198 @@ +import tempfile + +from lxml import etree +from django.contrib.auth import get_user_model +from django.test import SimpleTestCase, TestCase, override_settings + +from article.models import Article, ArticleCount, ArticleCountType +from article.sources.xmlsps import ( + count_visual_and_formula_items, + create_or_update_article_counts, +) +from core.models import Language +from pid_provider.models import PidProviderXML, XMLVersion + +User = get_user_model() + +VISUAL_ELEMENTS_XML = """ +
+ + + T + + + + + +

+

+ + x + + +

+ + y + +

+ + + + + + x + + + +
+""" + +XML_WITHOUT_LANG = """ +
+ + + +
+""" + + +def xmltree_from_string(xml): + return etree.fromstring(xml.encode()) + + +class CountVisualAndFormulaItemsTest(SimpleTestCase): + def test_counts_by_parent_lang_and_excludes_graphic_inside_fig_and_table_wrap(self): + errors = [] + counter = count_visual_and_formula_items( + xmltree_from_string(VISUAL_ELEMENTS_XML) + ) + + self.assertEqual(counter[(ArticleCountType.TYPE_FIG, "en")], 1) + self.assertEqual(counter[(ArticleCountType.TYPE_FIG, "pt")], 1) + self.assertEqual(counter[(ArticleCountType.TYPE_TABLE_WRAP, "en")], 1) + self.assertEqual(counter[(ArticleCountType.TYPE_DISP_FORMULA, "en")], 1) + self.assertEqual(counter[(ArticleCountType.TYPE_DISP_FORMULA, "pt")], 1) + self.assertEqual(counter[(ArticleCountType.TYPE_INLINE_FORMULA, "en")], 1) + self.assertEqual(counter[(ArticleCountType.TYPE_GRAPHIC, "en")], 2) + self.assertEqual(counter[(ArticleCountType.TYPE_INLINE_GRAPHIC, "en")], 1) + self.assertNotIn((ArticleCountType.TYPE_GRAPHIC, "pt"), counter) + self.assertEqual(errors, []) + + def test_missing_parent_lang_is_counted(self): + errors = [] + counter = count_visual_and_formula_items( + xmltree_from_string(XML_WITHOUT_LANG) + ) + + self.assertEqual(counter[(ArticleCountType.TYPE_FIG, None)], 1) + self.assertEqual(errors, []) + + +class CreateOrUpdateArticleCountsTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="counts_user", password="x") + self.article = Article.objects.create( + pid_v3="counts-v3", creator=self.user + ) + + def test_persists_totals_linked_to_article_by_type_and_language(self): + errors = [] + create_or_update_article_counts( + xmltree_from_string(VISUAL_ELEMENTS_XML), + self.article, + self.user, + errors, + ) + + counts = { + (item.count_type.code, item.language.code2): item.count + for item in self.article.counts.select_related("count_type", "language") + } + self.assertEqual(counts[(ArticleCountType.TYPE_FIG, "en")], 1) + self.assertEqual(counts[(ArticleCountType.TYPE_FIG, "pt")], 1) + self.assertEqual(counts[(ArticleCountType.TYPE_GRAPHIC, "en")], 2) + self.assertFalse( + self.article.counts.filter(count=0).exists() + ) + self.assertEqual(errors, []) + + def test_replaces_previous_counts(self): + count_type, _ = ArticleCountType.objects.get_or_create( + code=ArticleCountType.TYPE_FIG + ) + language = Language.get_or_create(code2="es", creator=self.user) + ArticleCount.objects.create( + article=self.article, + count_type=count_type, + language=language, + count=9, + ) + + create_or_update_article_counts( + xmltree_from_string(VISUAL_ELEMENTS_XML), + self.article, + self.user, + [], + ) + + self.assertFalse( + self.article.counts.filter( + count_type=count_type, language=language + ).exists() + ) + self.assertTrue( + self.article.counts.filter( + count_type__code=ArticleCountType.TYPE_FIG, language__code2="en" + ).exists() + ) + + +@override_settings(MEDIA_ROOT=tempfile.mkdtemp()) +class XMLVersionSaveFileUpdatesCountsTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="xml_save_user", password="x") + self.pp_xml = PidProviderXML.objects.create( + creator=self.user, + v3="COUNTSAVEFILEV3XXXXXXX", + pkg_name="count-save-pkg", + ) + self.article = Article.objects.create( + pid_v3="count-save-article", + pp_xml=self.pp_xml, + creator=self.user, + ) + + def test_save_file_persists_counts_for_linked_article(self): + version = XMLVersion.objects.create( + pid_provider_xml=self.pp_xml, + creator=self.user, + ) + version.save_file(f"{self.pp_xml.v3}.xml", VISUAL_ELEMENTS_XML) + + self.assertEqual( + self.article.counts.get( + count_type__code=ArticleCountType.TYPE_FIG, language__code2="en" + ).count, + 1, + ) + self.assertEqual( + self.article.counts.get( + count_type__code=ArticleCountType.TYPE_FIG, language__code2="pt" + ).count, + 1, + ) + self.assertEqual( + self.article.counts.get( + count_type__code=ArticleCountType.TYPE_GRAPHIC, language__code2="en" + ).count, + 2, + ) + + def test_save_file_without_article_does_not_raise(self): + self.article.delete() + version = XMLVersion.objects.create( + pid_provider_xml=self.pp_xml, + creator=self.user, + ) + version.save_file(f"{self.pp_xml.v3}.xml", VISUAL_ELEMENTS_XML) + self.assertFalse(ArticleCount.objects.filter(article__pp_xml=self.pp_xml).exists()) diff --git a/article/wagtail_hooks.py b/article/wagtail_hooks.py index af830e593..0f6a23dfd 100644 --- a/article/wagtail_hooks.py +++ b/article/wagtail_hooks.py @@ -1,4 +1,7 @@ +from django.urls import reverse +from django.utils.functional import cached_property from django.utils.translation import gettext_lazy as _ +from wagtail.admin.viewsets.base import ViewSet from wagtail.snippets.models import register_snippet from wagtail.snippets.views.snippets import SnippetViewSet, SnippetViewSetGroup @@ -225,6 +228,17 @@ class ArticlePeerReviewStatsSnippetViewSet(SnippetViewSet): list_per_page = 25 +class VisualElementTotalsViewSet(ViewSet): + name = "visual_element_totals" + icon = "table" + menu_label = _("Figure, Table and Equation Counts") + add_to_admin_menu = False + + @cached_property + def menu_url(self): + return reverse("visual_element_totals") + + class ArticleSnippetViewSetGroup(SnippetViewSetGroup): menu_label = _("Articles") menu_icon = "folder-open-inverse" @@ -238,6 +252,7 @@ class ArticleSnippetViewSetGroup(SnippetViewSetGroup): ArticleSourceSnippetViewSet, AMArticleSnippetViewSet, ArticlePeerReviewStatsSnippetViewSet, + VisualElementTotalsViewSet, ) diff --git a/core/templates/wagtailadmin/shared/header.html b/core/templates/wagtailadmin/shared/header.html index 4a2981b0e..145b51e50 100644 --- a/core/templates/wagtailadmin/shared/header.html +++ b/core/templates/wagtailadmin/shared/header.html @@ -6,7 +6,7 @@
- {% blocktrans trimmed %}{{ result_count }} out of {{ all_count }}{% endblocktrans %} + {{ result_count }} {% if result_count and all_count %} {% trans "out of" %} {% endif %}{{ all_count }}
diff --git a/pid_provider/models.py b/pid_provider/models.py index d31ef7f83..f0c24c899 100644 --- a/pid_provider/models.py +++ b/pid_provider/models.py @@ -156,6 +156,20 @@ def save_file(self, filename, content): except Exception as e: logging.exception(e) self.file.save(filename, ContentFile(content)) + self._update_linked_article_visual_counts() + + def _update_linked_article_visual_counts(self): + from article.models import Article + + if not self.pid_provider_xml_id or not self.file: + return + articles = Article.objects.filter(pp_xml_id=self.pid_provider_xml_id) + if not articles: + return + xmltree = self.xml_with_pre.xmltree + user = self.updated_by or self.creator + for article in articles: + article.create_or_update_article_visual_counts(user, xmltree=xmltree) def is_equal_to(self, xml_with_pre): return os.path.isfile(self.file.path) and ( diff --git a/report/forms.py b/report/forms.py new file mode 100644 index 000000000..35e9bf3ad --- /dev/null +++ b/report/forms.py @@ -0,0 +1,38 @@ +from django import forms +from django.utils.translation import gettext_lazy as _ + +from collection.models import Collection +from journal.models import Journal +from thematic_areas.models import ThematicArea + + +class VisualElementTotalsFilterForm(forms.Form): + collection = forms.ModelChoiceField( + queryset=Collection.objects.all(), + required=False, + label=_("Collection"), + empty_label="", + ) + thematic_area = forms.ModelChoiceField( + queryset=ThematicArea.objects.all(), + required=False, + label=_("Thematic Area"), + empty_label="", + ) + journal = forms.ModelChoiceField( + queryset=Journal.objects.order_by("title"), + required=False, + label=_("Journal"), + empty_label="", + ) + pid = forms.CharField( + required=False, + label=_("PID v2 or v3"), + max_length=23, + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.fields["journal"].queryset = Journal.objects.order_by("title").prefetch_related( + "scielojournal_set" + ) diff --git a/report/templates/report/visual_element_totals.html b/report/templates/report/visual_element_totals.html new file mode 100644 index 000000000..1c1551c90 --- /dev/null +++ b/report/templates/report/visual_element_totals.html @@ -0,0 +1,328 @@ +{% extends "wagtailadmin/generic/base.html" %} +{% load i18n static %} + +{% block extra_css %} + {{ block.super }} + +{% endblock %} + +{% block main_content %} +
+ {% for field in form %} +
+ {{ field.label_tag }} + {% if field.name == "journal" %} +
+ +
+ + + {% for journal in field.field.queryset %} + + {% endfor %} +
+
+ + {% else %} + {{ field }} + {% endif %} + +
+ {% endfor %} + + + {% trans "Download spreadsheet" %} + +
+ + {% if selected_filters %} +

+ {% for label, value in selected_filters %} + {{ label }}: {{ value }} + {% endfor %} +

+ {% endif %} + + + + + + + + + + + + + + + {% for row in yearly_rows %} + + + + + + + + + + {% empty %} + + + + {% endfor %} + +
{% trans "Year" %}figdisp-formulatable-wrapgraphicinline-graphicinline-formula
{{ row.article__pub_date_year }}{{ row.fig }}{{ row.disp_formula }}{{ row.table_wrap }}{{ row.graphic }}{{ row.inline_graphic }}{{ row.inline_formula }}
{% trans "No totals for the selected filters." %}
+ +{% endblock %} + +{% block extra_js %} + {{ block.super }} + +{% endblock %} diff --git a/report/tests.py b/report/tests.py index 7ce503c2d..e65cd15fd 100644 --- a/report/tests.py +++ b/report/tests.py @@ -1,3 +1,229 @@ +import csv +from io import StringIO + +from django.contrib.auth import get_user_model from django.test import TestCase -# Create your tests here. +from article.models import Article, ArticleCount, ArticleCountType +from collection.models import Collection +from core.models import Language +from journal.models import Journal, SciELOJournal, ThematicAreaJournal +from report.visual_element_totals import ( + filtered_article_ids, + spreadsheet_rows, + yearly_totals, +) +from thematic_areas.models import ThematicArea + +User = get_user_model() + + +class VisualElementTotalsQueryTest(TestCase): + def setUp(self): + self.user = User.objects.create_user(username="report-user", password="x") + self.collection_scl = Collection.objects.create( + acron3="scl", creator=self.user + ) + self.collection_arg = Collection.objects.create( + acron3="arg", creator=self.user + ) + self.area_health = ThematicArea.objects.create( + level0="Health", creator=self.user + ) + self.area_bio = ThematicArea.objects.create( + level0="Biological", creator=self.user + ) + self.journal = Journal.objects.create( + title="Revista Teste", + creator=self.user, + ) + SciELOJournal.objects.create( + journal=self.journal, + collection=self.collection_scl, + journal_acron="rtest", + creator=self.user, + ) + SciELOJournal.objects.create( + journal=self.journal, + collection=self.collection_arg, + journal_acron="rtest", + creator=self.user, + ) + ThematicAreaJournal.objects.create( + journal=self.journal, + thematic_area=self.area_health, + creator=self.user, + ) + ThematicAreaJournal.objects.create( + journal=self.journal, + thematic_area=self.area_bio, + creator=self.user, + ) + self.lang_fr = Language.get_or_create(code2="fr", creator=self.user) + self.lang_pt = Language.get_or_create(code2="pt", creator=self.user) + self.lang_en = Language.get_or_create(code2="en", creator=self.user) + self.lang_es = Language.get_or_create(code2="es", creator=self.user) + self.fig_type, _ = ArticleCountType.objects.get_or_create( + code=ArticleCountType.TYPE_FIG + ) + self.table_type, _ = ArticleCountType.objects.get_or_create( + code=ArticleCountType.TYPE_TABLE_WRAP + ) + self.article = Article.objects.create( + pid_v3="report-v3", + journal=self.journal, + pub_date_year="2024", + creator=self.user, + ) + for language, count in ( + (self.lang_fr, 1), + (self.lang_pt, 2), + (self.lang_en, 3), + (self.lang_es, 4), + (None, 5), + ): + ArticleCount.objects.create( + article=self.article, + count_type=self.fig_type, + language=language, + count=count, + ) + ArticleCount.objects.create( + article=self.article, + count_type=self.table_type, + language=self.lang_en, + count=1, + ) + self.article_without_year = Article.objects.create( + pid_v3="report-no-year", + journal=self.journal, + creator=self.user, + ) + ArticleCount.objects.create( + article=self.article_without_year, + count_type=self.fig_type, + language=self.lang_en, + count=99, + ) + + def test_includes_articles_without_pub_date_year(self): + article_ids = list(filtered_article_ids()) + self.assertIn(self.article.id, article_ids) + self.assertIn(self.article_without_year.id, article_ids) + + def test_collection_join_does_not_double_count(self): + article_ids = list(filtered_article_ids(collection=self.collection_scl)) + self.assertIn(self.article.id, article_ids) + self.assertEqual(len(article_ids), 2) + rows = list(yearly_totals(article_ids)) + row_2024 = next(row for row in rows if row["article__pub_date_year"] == "2024") + self.assertEqual(row_2024["fig"], 15) + self.assertEqual(row_2024["table_wrap"], 1) + + def test_thematic_area_filter_counts_article_once(self): + article_ids = list(filtered_article_ids(thematic_area=self.area_health)) + self.assertIn(self.article.id, article_ids) + rows = list(yearly_totals(article_ids)) + row_2024 = next(row for row in rows if row["article__pub_date_year"] == "2024") + self.assertEqual(row_2024["fig"], 15) + + def test_spreadsheet_columns_and_concatenated_thematic_areas(self): + article_ids = list(filtered_article_ids()) + columns, rows = spreadsheet_rows(article_ids) + self.assertEqual(columns[:4], ("collection", "thematic_area", "journal", "year")) + fig_columns = [ + column + for column in columns + if column.startswith("fig-") or column == "fig_total" + ] + self.assertEqual( + fig_columns, + [ + "fig-en", + "fig-es", + "fig-fr", + "fig-pt", + "fig-nd", + "fig_total", + ], + ) + self.assertIn("table-wrap-en", columns) + self.assertIn("table-wrap-nd", columns) + self.assertNotIn("table-wrap-pt", columns) + self.assertIn("table-wrap_total", columns) + self.assertNotIn("graphic-en", columns) + self.assertEqual(len(rows), 2) + row = next(item for item in rows if item["year"] == "2024") + self.assertEqual(row["collection"], "arg; scl") + self.assertEqual(row["journal"], "Revista Teste") + self.assertEqual(row["year"], "2024") + self.assertEqual(row["fig-fr"], 1) + self.assertEqual(row["fig-pt"], 2) + self.assertEqual(row["fig-en"], 3) + self.assertEqual(row["fig-es"], 4) + self.assertEqual(row["fig-nd"], 5) + self.assertEqual(row["fig_total"], 15) + self.assertEqual(row["table-wrap-en"], 1) + self.assertEqual(row["table-wrap-nd"], 0) + self.assertEqual(row["table-wrap_total"], 1) + self.assertEqual(row["graphic_total"], 0) + self.assertIn("Health", row["thematic_area"]) + self.assertIn("Biological", row["thematic_area"]) + + buffer = StringIO() + writer = csv.writer(buffer, delimiter=";") + writer.writerow(columns) + for row in rows: + writer.writerow( + [ + "" if row.get(column) is None else row.get(column) + for column in columns + ] + ) + header = buffer.getvalue().splitlines()[0] + self.assertEqual(header, ";".join(columns)) + + def test_pid_matches_v2_or_v3(self): + self.article.pid_v2 = "S0100-000020240001" + self.article.save(update_fields=["pid_v2"]) + + by_v3 = list(filtered_article_ids(pid="report-v3")) + by_v2 = list(filtered_article_ids(pid="S0100-000020240001")) + + self.assertEqual(by_v3, [self.article.id]) + self.assertEqual(by_v2, [self.article.id]) + + def test_missing_language_uses_nd_column(self): + ArticleCount.objects.create( + article=self.article, + count_type=self.fig_type, + language=None, + count=4, + ) + lang_nd = Language.get_or_create(code2="nd", creator=self.user) + ArticleCount.objects.create( + article=self.article, + count_type=self.table_type, + language=lang_nd, + count=6, + ) + columns, rows = spreadsheet_rows(list(filtered_article_ids())) + row = next(item for item in rows if item["year"] == "2024") + self.assertIn("fig-nd", columns) + self.assertEqual(row["fig-nd"], 9) + self.assertEqual(row["fig_total"], 19) + self.assertEqual(row["table-wrap-nd"], 6) + self.assertEqual(row["table-wrap_total"], 7) + fig_columns = [ + column + for column in columns + if column.startswith("fig-") or column == "fig_total" + ] + self.assertEqual(fig_columns[-2], "fig-nd") + self.assertEqual(fig_columns[-1], "fig_total") + + def test_filtered_collection_is_used_in_spreadsheet_column(self): + article_ids = list(filtered_article_ids(collection=self.collection_arg)) + columns, rows = spreadsheet_rows(article_ids, collection=self.collection_arg) + self.assertEqual(rows[0]["collection"], "arg") + self.assertTrue(columns) diff --git a/report/views.py b/report/views.py index 91ea44a21..5eab9cd25 100644 --- a/report/views.py +++ b/report/views.py @@ -1,3 +1,74 @@ +import csv + +from django.http import HttpResponse from django.shortcuts import render +from django.utils.translation import gettext_lazy as _ + +from report.forms import VisualElementTotalsFilterForm +from report.visual_element_totals import ( + filtered_article_ids, + spreadsheet_rows, + yearly_totals, +) + + +def _filters_from_form(form): + if not form.is_valid(): + return None + return { + "collection": form.cleaned_data.get("collection"), + "thematic_area": form.cleaned_data.get("thematic_area"), + "journal": form.cleaned_data.get("journal"), + "pid": form.cleaned_data.get("pid"), + } + + +def visual_element_totals_view(request): + form = VisualElementTotalsFilterForm(request.GET) + yearly_rows = [] + filters = _filters_from_form(form) + selected_filters = [] + if filters is not None: + article_ids = filtered_article_ids(**filters) + yearly_rows = yearly_totals(article_ids) + for name in ("collection", "thematic_area", "journal", "pid"): + value = filters.get(name) + if value: + selected_filters.append((form.fields[name].label, value)) + return render( + request, + "report/visual_element_totals.html", + { + "form": form, + "yearly_rows": yearly_rows, + "selected_filters": selected_filters, + "header_title": _("Figure, Table and Equation Counts"), + "page_title": _("Figure, Table and Equation Counts"), + "header_icon": "table", + }, + ) + + +def visual_element_totals_csv_view(request): + form = VisualElementTotalsFilterForm(request.GET) + filters = _filters_from_form(form) + if filters is None: + return visual_element_totals_view(request) + + article_ids = filtered_article_ids(**filters) + columns, rows = spreadsheet_rows(article_ids, collection=filters["collection"]) -# Create your views here. + response = HttpResponse(content_type="text/csv") + response["Content-Disposition"] = ( + 'attachment; filename="visual_element_totals.csv"' + ) + writer = csv.writer(response, delimiter=";") + writer.writerow(columns) + for row in rows: + writer.writerow( + [ + "" if row.get(column) is None else row.get(column) + for column in columns + ] + ) + return response diff --git a/report/visual_element_totals.py b/report/visual_element_totals.py new file mode 100644 index 000000000..1e5b3ff04 --- /dev/null +++ b/report/visual_element_totals.py @@ -0,0 +1,186 @@ +from article.models import Article, ArticleCount, ArticleCountType +from django.db.models import Exists, OuterRef, Q, Sum +from django.db.models.functions import Coalesce +from journal.models import SciELOJournal, ThematicAreaJournal + +THEMATIC_AREA_SEPARATOR = "; " +UNDEFINED_LANG = "nd" +SPREADSHEET_BASE_COLUMNS = ("collection", "thematic_area", "journal", "year") + + +def _count_alias(code, lang=None): + safe = code.replace("-", "_") + if lang is None: + return f"{safe}_total" + return f"{safe}_{lang}" + + +def _sum_count(code, lang=None): + condition = Q(count_type__code=code) + if lang == UNDEFINED_LANG: + condition &= Q(language__isnull=True) | Q(language__code2=UNDEFINED_LANG) + elif lang is not None: + condition &= Q(language__code2=lang) + return Coalesce(Sum("count", filter=condition), 0) + + +def _languages_by_type(article_ids): + rows = ( + ArticleCount.objects.filter( + article_id__in=article_ids, + count_type__code__in=ArticleCountType.VISUAL_TYPES, + ) + .values_list("count_type__code", "language__code2") + .distinct() + ) + langs_by_code = {} + for code, lang in rows: + langs_by_code.setdefault(code, {UNDEFINED_LANG}).add(lang or UNDEFINED_LANG) + return { + code: sorted(langs, key=lambda lang: (lang == UNDEFINED_LANG, lang)) + for code, langs in langs_by_code.items() + } + + +def spreadsheet_columns(langs_by_code): + columns = list(SPREADSHEET_BASE_COLUMNS) + for code in ArticleCountType.VISUAL_TYPES: + for lang in langs_by_code.get(code, []): + columns.append(f"{code}-{lang}") + columns.append(f"{code}_total") + return tuple(columns) + + +def _spreadsheet_annotations(langs_by_code): + annotations = {} + for code in ArticleCountType.VISUAL_TYPES: + for lang in langs_by_code.get(code, []): + annotations[_count_alias(code, lang)] = _sum_count(code, lang) + annotations[_count_alias(code)] = _sum_count(code) + return annotations + + +def filtered_article_ids(collection=None, thematic_area=None, journal=None, pid=None): + qs = Article.objects.filter(journal__isnull=False) + if collection: + qs = qs.filter( + Exists( + SciELOJournal.objects.filter( + journal_id=OuterRef("journal_id"), + collection=collection, + ) + ) + ) + if thematic_area: + qs = qs.filter( + Exists( + ThematicAreaJournal.objects.filter( + journal_id=OuterRef("journal_id"), + thematic_area=thematic_area, + ) + ) + ) + if journal: + qs = qs.filter(journal=journal) + if pid: + pid = pid.strip() + qs = qs.filter(Q(pid_v2=pid) | Q(pid_v3=pid)) + return qs.values_list("id", flat=True) + + +def yearly_totals(article_ids): + return ( + ArticleCount.objects.filter( + article_id__in=article_ids, + count_type__code__in=ArticleCountType.VISUAL_TYPES, + ) + .values("article__pub_date_year") + .annotate( + **{ + code.replace("-", "_"): _sum_count(code) + for code in ArticleCountType.VISUAL_TYPES + } + ) + .order_by("-article__pub_date_year") + ) + + +def _collections_by_journal(journal_ids): + collections_by_journal = {} + rows = ( + SciELOJournal.objects.filter( + journal_id__in=journal_ids, + collection__acron3__gt="", + ) + .order_by("collection__acron3") + .values_list("journal_id", "collection__acron3") + .distinct() + ) + for journal_id, acron in rows: + collections_by_journal.setdefault(journal_id, []).append(acron) + return collections_by_journal + + +def _thematic_areas_by_journal(journal_ids): + areas_by_journal = {} + items = ThematicAreaJournal.objects.filter( + journal_id__in=journal_ids, + thematic_area__isnull=False, + ).select_related("thematic_area") + for item in items: + areas_by_journal.setdefault(item.journal_id, []).append(str(item.thematic_area)) + return areas_by_journal + + +def spreadsheet_rows(article_ids, collection=None): + langs_by_code = _languages_by_type(article_ids) + columns = spreadsheet_columns(langs_by_code) + aggregated = list( + ArticleCount.objects.filter( + article_id__in=article_ids, + count_type__code__in=ArticleCountType.VISUAL_TYPES, + ) + .values( + "article__journal_id", + "article__journal__title", + "article__pub_date_year", + ) + .annotate(**_spreadsheet_annotations(langs_by_code)) + ) + + journal_ids = {row["article__journal_id"] for row in aggregated} + areas_by_journal = _thematic_areas_by_journal(journal_ids) + collections_by_journal = _collections_by_journal(journal_ids) + + rows = [] + for row in aggregated: + journal_id = row["article__journal_id"] + if collection: + collection_label = collection.acron3 + else: + collection_label = THEMATIC_AREA_SEPARATOR.join( + collections_by_journal.get(journal_id, []) + ) + item = { + "collection": collection_label, + "thematic_area": THEMATIC_AREA_SEPARATOR.join( + areas_by_journal.get(journal_id, []) + ), + "journal": row["article__journal__title"], + "year": row["article__pub_date_year"], + } + for code in ArticleCountType.VISUAL_TYPES: + for lang in langs_by_code.get(code, []): + item[f"{code}-{lang}"] = row[_count_alias(code, lang)] + item[f"{code}_total"] = row[_count_alias(code)] + rows.append(item) + + rows = sorted( + rows, + key=lambda row: ( + row["collection"] or "", + row["journal"] or "", + row["year"] or "", + ), + ) + return columns, rows diff --git a/report/wagtail_hooks.py b/report/wagtail_hooks.py index f0f371b3d..bc24d84e4 100644 --- a/report/wagtail_hooks.py +++ b/report/wagtail_hooks.py @@ -1,9 +1,13 @@ +from django.urls import path +from django.utils.translation import gettext_lazy as _ +from wagtail import hooks from wagtail.snippets.models import register_snippet from wagtail.snippets.views.snippets import SnippetViewSet -from django.utils.translation import gettext_lazy as _ -from .models import ReportCSV from config.menu import get_menu_order +from report.views import visual_element_totals_csv_view, visual_element_totals_view + +from .models import ReportCSV @register_snippet class ReportCSVAdmin(SnippetViewSet): @@ -37,4 +41,20 @@ def link_download(self, obj): return None link_download.short_descriptions = 'Download' - link_download.allow_tags = True \ No newline at end of file + link_download.allow_tags = True + + +@hooks.register("register_admin_urls") +def register_visual_element_totals_urls(): + return [ + path( + "report/visual-element-totals/", + visual_element_totals_view, + name="visual_element_totals", + ), + path( + "report/visual-element-totals/csv/", + visual_element_totals_csv_view, + name="visual_element_totals_csv", + ), + ]