Adiciona relatórios COUNTER 5.1 e consolida a migração histórica

.dockerignore

@@ -10,5 +10,6 @@
 .travis.yml
 .continue
 .envs/.local/.django
+core/media
 temp
 venv

VERSION

@@ -1 +1 @@
-2.5.2
+2.6.0

config/sentry.py

@@ -0,0 +1,28 @@
+from urllib.parse import parse_qsl, urlsplit, urlunsplit
+
+
+def scrub_api_key(event, _hint):
+    request = event.get("request") or {}
+    query_string = request.get("query_string") or ""
+    url = request.get("url") or ""
+    url_query = urlsplit(url).query if url else ""
+
+    if not _contains_api_key(query_string) and not _contains_api_key(url_query):
+        return event
+
+    request["query_string"] = ""
+    if url:
+        parts = urlsplit(url)
+        request["url"] = urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+    return event
+
+
+def _contains_api_key(query_string):
+    if not isinstance(query_string, str):
+        return False
+
+    return any(
+        name.lower() == "api_key"
+        for name, _value in parse_qsl(query_string, keep_blank_values=True)
+    )

config/settings/base.py

@@ -1,6 +1,7 @@
 """
 Base settings to build other settings files upon.
 """
+
 from datetime import timedelta
 from pathlib import Path
 
@@ -93,10 +94,8 @@
     "core.search_site",
     "wagtail.contrib.forms",
     "wagtail.contrib.redirects",
-    "wagtail_modeladmin",
     "wagtail.contrib.simple_translation",
     "wagtail_localize",
-    "wagtail_localize.modeladmin",
     "wagtail.embeds",
     "wagtail.sites",
     "wagtail.users",
@@ -139,6 +138,7 @@
     "wagtailcaptcha",
     "wagtailmenus",
     "rest_framework",
+    "drf_spectacular",
 ]
 
 LOCAL_APPS = [
@@ -151,6 +151,7 @@
     "log_manager",
     "log_manager_config",
     "metrics",
+    "counter_api",
     "reports",
     "resources",
     "source",
@@ -457,11 +458,38 @@
 REST_FRAMEWORK = {
     "DEFAULT_PAGINATION_CLASS": "rest_framework.pagination.PageNumberPagination",
     "PAGE_SIZE": env.int("DRF_PAGE_SIZE", default=10),
+    "DEFAULT_SCHEMA_CLASS": "drf_spectacular.openapi.AutoSchema",
     "DEFAULT_AUTHENTICATION_CLASSES": (
         "rest_framework_simplejwt.authentication.JWTAuthentication",
     ),
 }
 
+SPECTACULAR_SETTINGS = {
+    "TITLE": "SciELO COUNTER API",
+    "DESCRIPTION": (
+        "COUNTER Project Release 5.1 API with SciELO extensions. "
+        "Report availability follows completed months in OpenSearch."
+    ),
+    "VERSION": "5.1",
+    "SERVE_INCLUDE_SCHEMA": False,
+    "SERVE_AUTHENTICATION": [],
+    "SERVE_PERMISSIONS": ["rest_framework.permissions.AllowAny"],
+    "SWAGGER_UI_SETTINGS": {
+        "deepLinking": True,
+        "displayOperationId": True,
+        "persistAuthorization": False,
+    },
+    "APPEND_COMPONENTS": {
+        "securitySchemes": {
+            "CounterAPIKey": {
+                "type": "apiKey",
+                "in": "query",
+                "name": "api_key",
+            }
+        }
+    },
+}
+
 # JWT
 # ------------------------------------------------------------------------------
 SIMPLE_JWT = {
@@ -503,10 +531,22 @@
 COUNTER_ROBOTS_URL = env(
     "COUNTER_ROBOTS_URL",
     default=(
-        "https://raw.githubusercontent.com/atmire/COUNTER-Robots/master/"
-        "COUNTER_Robots_list.json"
+        "https://raw.githubusercontent.com/Project-Counter/counter-bots/"
+        "counter-bots/counter_bots.json"
     ),
 )
+COUNTER_REGISTRY_URL = env("COUNTER_REGISTRY_URL", default="")
+COUNTER_CUSTOMER_ID = env("COUNTER_CUSTOMER_ID", default="0000000000000000")
+COUNTER_INSTITUTION_NAME = env("COUNTER_INSTITUTION_NAME", default="The World")
+COUNTER_CREATED_BY = env("COUNTER_CREATED_BY", default="SciELO")
+COUNTER_QUERY_BUDGET_SECONDS = env.int(
+    "COUNTER_QUERY_BUDGET_SECONDS",
+    default=25,
+)
+COUNTER_MAX_REPORT_ENTRIES = env.int(
+    "COUNTER_MAX_REPORT_ENTRIES",
+    default=100000,
+)
 MMDB_URL_TEMPLATE = env(
     "MMDB_URL_TEMPLATE",
     default=(

config/settings/production.py

@@ -6,6 +6,8 @@
 from sentry_sdk.integrations.logging import LoggingIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
 
+from config.sentry import scrub_api_key
+
 from .base import *  # noqa
 from .base import env
 
@@ -86,7 +88,9 @@
 EMAIL_PORT = env.int("DJANGO_EMAIL_PORT", default=25)
 
 # https://docs.djangoproject.com/en/dev/ref/settings/#email-host-user
-EMAIL_HOST_USER = env.str("DJANGO_EMAIL_HOST_USER", default="suporte.aplicacao@scielo.org")
+EMAIL_HOST_USER = env.str(
+    "DJANGO_EMAIL_HOST_USER", default="suporte.aplicacao@scielo.org"
+)
 
 # https://docs.djangoproject.com/en/dev/ref/settings/#std-setting-EMAIL_HOST_PASSWORD
 EMAIL_HOST_PASSWORD = env.str("DJANGO_EMAIL_HOST_PASSWORD", default="")
@@ -197,6 +201,8 @@
     integrations=integrations,
     environment=env("SENTRY_ENVIRONMENT", default="production"),
     traces_sample_rate=env.float("SENTRY_TRACES_SAMPLE_RATE", default=0.0),
+    before_send=scrub_api_key,
+    before_send_transaction=scrub_api_key,
 )
 
 # Redis

config/urls.py

@@ -2,6 +2,7 @@
 from django.conf.urls.i18n import i18n_patterns
 from django.conf.urls.static import static
 from django.contrib import admin
+from django.contrib.staticfiles.urls import staticfiles_urlpatterns
 from django.urls import include, path, re_path
 from django.views import defaults as default_views
 from wagtail import urls as wagtail_urls
@@ -10,6 +11,7 @@
 from wagtailautocomplete.urls.admin import urlpatterns as autocomplete_admin_urls
 
 urlpatterns = [
+    path("r51/", include("counter_api.urls")),
     path("admin/autocomplete/", include(autocomplete_admin_urls)),
     path(settings.DJANGO_ADMIN_URL, admin.site.urls),
     # Wagtail Admin
@@ -30,8 +32,6 @@
 
 if settings.DEBUG:
     # Wagtail settings: Serve static and media files from development server
-    from django.contrib.staticfiles.urls import staticfiles_urlpatterns
-
     urlpatterns += staticfiles_urlpatterns()
     urlpatterns += static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
     # This allows the error pages to be debugged during development, just visit
@@ -55,6 +55,4 @@
         path("500/", default_views.server_error),
     ]
     if "debug_toolbar" in settings.INSTALLED_APPS:
-        import debug_toolbar
-
-        urlpatterns = [path("__debug__/", include(debug_toolbar.urls))] + urlpatterns
+        urlpatterns = [path("__debug__/", include("debug_toolbar.urls"))] + urlpatterns

counter_api/access.py

@@ -0,0 +1,25 @@
+from rest_framework.renderers import JSONRenderer
+from rest_framework.response import Response
+from rest_framework.views import APIView
+
+from counter_api.exceptions import CounterAPIError
+from counter_api.models import APIKey
+from counter_api.negotiation import CounterContentNegotiation
+
+
+class CounterAPIView(APIView):
+    authentication_classes = []
+    permission_classes = []
+    renderer_classes = [JSONRenderer]
+    content_negotiation_class = CounterContentNegotiation
+    http_method_names = ["get"]
+
+
+def get_authentication_error(request):
+    key = APIKey.authenticate(request.query_params.get("api_key"))
+    if key:
+        return None
+
+    error = CounterAPIError(2020, "APIKey Invalid", 401)
+
+    return Response(error.as_dict(), status=error.status_code)

counter_api/admin.py

@@ -0,0 +1,21 @@
+from django.contrib import admin
+
+from counter_api.models import APIKey
+
+
+@admin.register(APIKey)
+class APIKeyAdmin(admin.ModelAdmin):
+    list_display = (
+        "name",
+        "active",
+        "expires_at",
+        "last_used_at",
+        "created_at",
+    )
+    list_filter = ("active",)
+    search_fields = ("name",)
+    fields = ("name", "active", "expires_at", "last_used_at", "created_at")
+    readonly_fields = ("name", "last_used_at", "created_at")
+
+    def has_add_permission(self, request):
+        return False

counter_api/api_urls.py

@@ -0,0 +1,11 @@
+from django.urls import path
+
+from counter_api import views
+
+urlpatterns = [
+    path("status", views.StatusView.as_view(), name="status"),
+    path("members", views.MembersView.as_view(), name="members"),
+    path("platforms", views.PlatformsView.as_view(), name="platforms"),
+    path("reports", views.ReportsView.as_view(), name="reports"),
+    path("reports/<str:report_id>", views.ReportView.as_view(), name="report"),
+]

counter_api/apps.py

@@ -0,0 +1,6 @@
+from django.apps import AppConfig
+
+
+class CounterAPIConfig(AppConfig):
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "counter_api"

counter_api/availability.py

@@ -0,0 +1,79 @@
+from datetime import date
+from time import monotonic
+
+from django.conf import settings
+from opensearchpy import NotFoundError
+
+from counter_api.dates import iter_months
+from counter_api.search import search
+from metrics.opensearch.client import OpenSearchUsageClient
+from metrics.opensearch.names import generate_month_status_index_name
+
+PAGE_SIZE = 1000
+
+
+def available_months(platform):
+    index_name = generate_month_status_index_name(settings.OPENSEARCH_INDEX_NAME)
+    client = OpenSearchUsageClient().client
+    deadline = monotonic() + settings.COUNTER_QUERY_BUDGET_SECONDS
+    current_month = date.today().strftime("%Y-%m")
+    months = []
+    after = None
+
+    while True:
+        body = {
+            "size": 0,
+            "track_total_hits": False,
+            "query": {
+                "bool": {
+                    "filter": [
+                        {"term": {"collection": platform.acron3}},
+                        {"term": {"complete": True}},
+                        {"range": {"month": {"lt": current_month}}},
+                    ]
+                }
+            },
+            "aggs": {
+                "months": {
+                    "composite": {
+                        "size": PAGE_SIZE,
+                        "sources": [{"month": {"terms": {"field": "month"}}}],
+                    }
+                }
+            },
+        }
+        if after:
+            body["aggs"]["months"]["composite"]["after"] = after
+
+        try:
+            response = search(client, index_name, body, deadline)
+        except NotFoundError:
+            return []
+
+        page = response["aggregations"]["months"]
+        months.extend(
+            date.fromisoformat(f"{bucket['key']['month']}-01")
+            for bucket in page["buckets"]
+        )
+        after = page.get("after_key")
+        if not after:
+            return months
+
+
+def split_period(platform, begin, end):
+    requested = list(iter_months(begin, end))
+    complete = set(available_months(platform))
+
+    if not complete:
+        return [], requested, []
+
+    first_available = min(complete)
+    available = [month for month in requested if month in complete]
+    unavailable = [month for month in requested if month < first_available]
+    pending = [
+        month
+        for month in requested
+        if month not in complete and month not in unavailable
+    ]
+
+    return available, pending, unavailable