feat(report): adiciona totais de elementos visuais por artigo

article/migrations/0051_articlecount_article.py

@@ -0,0 +1,76 @@
+import django.db.models.deletion
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("article", "0050_remove_articlesource_am_article_and_more"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="articlecount",
+            name="article",
+            field=models.ForeignKey(
+                blank=True,
+                null=True,
+                on_delete=django.db.models.deletion.CASCADE,
+                related_name="counts",
+                to="article.article",
+            ),
+        ),
+        migrations.AddIndex(
+            model_name="articlecount",
+            index=models.Index(
+                fields=["article"], name="article_art_article_8968b6_idx"
+            ),
+        ),
+        migrations.AlterUniqueTogether(
+            name="articlecount",
+            unique_together={("article", "count_type", "language")},
+        ),
+        migrations.RemoveField(
+            model_name="articlecount",
+            name="created",
+        ),
+        migrations.RemoveField(
+            model_name="articlecount",
+            name="updated",
+        ),
+        migrations.RemoveField(
+            model_name="articlecount",
+            name="creator",
+        ),
+        migrations.RemoveField(
+            model_name="articlecount",
+            name="updated_by",
+        ),
+        migrations.RemoveField(
+            model_name="articlecounttype",
+            name="created",
+        ),
+        migrations.RemoveField(
+            model_name="articlecounttype",
+            name="creator",
+        ),
+        migrations.RemoveField(
+            model_name="articlecounttype",
+            name="updated",
+        ),
+        migrations.RemoveField(
+            model_name="articlecounttype",
+            name="updated_by",
+        ),
+        migrations.AlterField(
+            model_name="articlecounttype",
+            name="code",
+            field=models.CharField(
+                blank=True,
+                max_length=20,
+                null=True,
+                unique=True,
+                verbose_name="Code",
+            ),
+        ),
+    ]

article/models.py

@@ -378,6 +378,28 @@ def xmltree(self):
         except AttributeError:
             return PidProvider.get_xmltree(self.pid_v3)
 
+    def create_or_update_article_visual_counts(self, user, xmltree=None, errors=None):
+        """
+        Calcula/Recalcula fig, table-wrap, fórmulas e graphics a partir do XML do artigo.
+        """
+        # Evita importação circular.
+        from article.sources.xmlsps import create_or_update_article_counts
+
+        if errors is None:
+            errors = []
+        if xmltree is None:
+            pp_xml = self.pp_xml
+            if not pp_xml:
+                raise ValueError(f"Article {self.pk} has no associated XML")
+            xml_with_pre = pp_xml.xml_with_pre
+            if not xml_with_pre:
+                raise ValueError(
+                    f"Unable to get XML to count visual elements from {pp_xml}"
+                )
+            xmltree = xml_with_pre.xmltree
+        create_or_update_article_counts(xmltree, self, user, errors)
+        return errors
+
     @cached_property
     def collections(self):
         """
@@ -1378,8 +1400,23 @@ def data(self):
         )
 
 
-class ArticleCountType(CommonControlField):
-    code = models.CharField(_("Code"), blank=True, null=True, max_length=20)
+class ArticleCountType(models.Model):
+    TYPE_FIG = "fig"
+    TYPE_DISP_FORMULA = "disp-formula"
+    TYPE_TABLE_WRAP = "table-wrap"
+    TYPE_GRAPHIC = "graphic"
+    TYPE_INLINE_GRAPHIC = "inline-graphic"
+    TYPE_INLINE_FORMULA = "inline-formula"
+    VISUAL_TYPES = (
+        TYPE_FIG,
+        TYPE_DISP_FORMULA,
+        TYPE_TABLE_WRAP,
+        TYPE_GRAPHIC,
+        TYPE_INLINE_GRAPHIC,
+        TYPE_INLINE_FORMULA,
+    )
+
+    code = models.CharField(_("Code"), blank=True, null=True, max_length=20, unique=True)
 
     class Meta:
         indexes = [
@@ -1396,24 +1433,14 @@ def __unicode__(self):
     def __str__(self):
         return "%s" % self.code
 
-    @property
-    def data(self):
-        return dict(article_count_type__code=self.code)
-
-    @classmethod
-    def get_or_create(cls, code, user):
-        try:
-            return cls.objects.get(code=code)
-        except cls.DoesNotExist:
-            article_count_type = cls()
-            article_count_type.code = code
-            article_count_type.creator = user
-            article_count_type.save()
-
-            return article_count_type
-
-
-class ArticleCount(CommonControlField):
+class ArticleCount(models.Model):
+    article = models.ForeignKey(
+        Article,
+        null=True,
+        blank=True,
+        on_delete=models.CASCADE,
+        related_name="counts",
+    )
     count_type = models.ForeignKey(
         ArticleCountType, null=True, blank=True, on_delete=models.SET_NULL
     )
@@ -1426,10 +1453,21 @@ class ArticleCount(CommonControlField):
         blank=True,
     )
 
-    panels = [AutocompletePanel("count_type"), AutocompletePanel("language")]
+    panels = [
+        FieldPanel("article", read_only=True),
+        FieldPanel("count_type", read_only=True),
+        FieldPanel("count", read_only=True),
+        FieldPanel("language", read_only=True),
+    ]
 
     class Meta:
+        unique_together = [("article", "count_type", "language")]
         indexes = [
+            models.Index(
+                fields=[
+                    "article",
+                ]
+            ),
             models.Index(
                 fields=[
                     "count_type",
@@ -1443,10 +1481,10 @@ class Meta:
         ]
 
     def __unicode__(self):
-        return "%s | %s | %s" % (self.count_type, self.count, self.language)
+        return f"{self.article} | {self.count_type} | {self.count} | {self.language}"
 
     def __str__(self):
-        return "%s | %s | %s" % (self.count_type, self.count, self.language)
+        return f"{self.article} | {self.count_type} | {self.count} | {self.language}"
 
     @property
     def data(self):

article/sources/xmlsps.py

@@ -1,4 +1,4 @@
-import traceback
+from collections import Counter
 from datetime import datetime
 from itertools import product
 
@@ -12,17 +12,24 @@
 from packtools.sps.models.article_license import ArticleLicense
 from packtools.sps.models.article_titles import ArticleTitles
 from packtools.sps.models.dates import ArticleDates
+from packtools.sps.models.fig import ArticleFigs
+from packtools.sps.models.formula import ArticleFormulas
 from packtools.sps.models.front_articlemeta_issue import ArticleMetaIssue
 from packtools.sps.models.funding_group import FundingGroup
+from packtools.sps.models.graphic import Graphic, InlineGraphic, XmlGraphic
 from packtools.sps.models.journal_meta import ISSN, Title
 from packtools.sps.models.kwd_group import ArticleKeywords
+from packtools.sps.models.tablewrap import ArticleTableWrappers
 from packtools.sps.models.v2.article_toc_sections import ArticleTocSections
 from packtools.sps.models.v2.related_articles import RelatedArticles
+from packtools.sps.models.visual_resource_base import XmlVisualResource
 
 from article import choices
 from article.models import (
     Article,
     ArticleAffiliation,
+    ArticleCount,
+    ArticleCountType,
     ArticleFunding,
     ContribCollab,
     ContribPerson,
@@ -224,6 +231,10 @@ def load_article(user, pp_xml):
 
         add_related_articles(xmltree=xmltree, article=article, user=user, errors=errors)
 
+        create_or_update_article_counts(
+            xmltree=xmltree, article=article, user=user, errors=errors
+        )
+
         article.create_legacy_keys(user)
         if not article.pid_v2:
             raise ValueError(f"Article has no PID v2: {article.pid_v3}")
@@ -1064,3 +1075,108 @@ def add_related_articles(xmltree, article, user, errors):
 
     except Exception as e:
         add_error(errors, "add_related_articles", e)
+
+
+class GraphicWithNode(Graphic):
+    @property
+    def data(self):
+        data = super().data
+        data["node"] = self.node
+        return data
+
+
+class InlineGraphicWithNode(InlineGraphic):
+    @property
+    def data(self):
+        data = super().data
+        data["node"] = self.node
+        return data
+
+
+class XmlGraphicWithNode(XmlGraphic):
+    def __init__(self, xmltree):
+        resource_types = [
+            ("graphic", GraphicWithNode),
+            ("inline-graphic", InlineGraphicWithNode),
+        ]
+        XmlVisualResource.__init__(self, xmltree, resource_types=resource_types)
+
+
+def count_items_by_parent_lang(items, count_type, counter):
+    for item in items:
+        counter[(count_type, item.get("parent_lang") or None)] += 1
+
+def count_graphics_and_inline_graphics(xmltree, counter):
+    """
+    Conta graphics e inline-graphics por idioma do artigo,
+    ignorando graphics que são descendentes de fig/table-wrap.
+    """
+    for item in XmlGraphicWithNode(xmltree).data:
+        tag = item.get("tag")
+        lang = item.get("parent_lang") or None
+        if tag == ArticleCountType.TYPE_INLINE_GRAPHIC:
+            counter[(ArticleCountType.TYPE_INLINE_GRAPHIC, lang)] += 1
+        elif tag == ArticleCountType.TYPE_GRAPHIC:
+            node = item.get("node")
+            if node is not None and any(
+                ancestor.tag in ("fig", "table-wrap") for ancestor in node.iterancestors()
+            ):
+                continue
+            counter[(ArticleCountType.TYPE_GRAPHIC, lang)] += 1
+
+def count_visual_and_formula_items(xmltree):
+    """
+    Conta fig, table-wrap, fórmulas e graphics por idioma do article/sub-article.
+    """
+    counter = Counter()
+    count_items_by_parent_lang(
+        items=ArticleFigs(xmltree).get_all_figs,
+        count_type=ArticleCountType.TYPE_FIG,
+        counter=counter,
+    )
+    count_items_by_parent_lang(
+        items=ArticleTableWrappers(xmltree).get_all_table_wrappers,
+        count_type=ArticleCountType.TYPE_TABLE_WRAP,
+        counter=counter,
+    )
+    formulas = ArticleFormulas(xmltree)
+    count_items_by_parent_lang(
+        items=formulas.disp_formula_items,
+        count_type=ArticleCountType.TYPE_DISP_FORMULA,
+        counter=counter,
+    )
+    count_items_by_parent_lang(
+        items=formulas.inline_formula_items,
+        count_type=ArticleCountType.TYPE_INLINE_FORMULA,
+        counter=counter,
+    )
+    count_graphics_and_inline_graphics(xmltree, counter)
+
+    return counter
+
+
+def create_or_update_article_counts(xmltree, article, user, errors):
+    """
+    Substitui os totais visuais/fórmulas do artigo a partir do xmltree.
+    """
+    counter = count_visual_and_formula_items(xmltree)
+    kept_ids = []
+    for (count_type_code, lang), count in counter.items():
+        language = get_or_create_language(lang, user, errors) if lang else None
+        if lang and not language:
+            add_error(
+                errors,
+                "create_or_update_article_counts",
+                ValueError(f"Missing language for {count_type_code}"),
+                count_type=count_type_code,
+                lang=lang,
+            )
+        count_type, _ = ArticleCountType.objects.get_or_create(code=count_type_code)
+        article_count, _ = ArticleCount.objects.update_or_create(
+            article=article,
+            count_type=count_type,
+            language=language,
+            defaults={"count": count},
+        )
+        kept_ids.append(article_count.pk)
+    article.counts.exclude(pk__in=kept_ids).delete()

article/tasks.py

@@ -966,3 +966,30 @@ def task_process_article_pipeline(
             },
         )
         raise
+
+
+@celery_app.task(bind=True)
+def task_create_article_visual_counts(self, user_id=None, username=None, article_id=None):
+    """
+    Preenche os totais visuais dos artigos já carregados que têm XML.
+    """
+    user = _get_user(self.request, username=username, user_id=user_id)
+    articles = Article.objects.filter(pp_xml__isnull=False)
+    if article_id:
+        articles = articles.filter(pk=article_id)
+    for article in articles.iterator():
+        logging.info(
+            f"Starting visual counts for article id={article.pk} ({article})"
+        )
+        try:
+            article.create_or_update_article_visual_counts(user)
+        except Exception as e:
+            exc_type, exc_value, exc_traceback = sys.exc_info()
+            UnexpectedEvent.create(
+                action="article.tasks.task_create_article_visual_counts",
+                item=str(article),
+                exception=e,
+                exc_traceback=exc_traceback,
+                detail={"article_id": article.id},
+            )
+