Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -85,12 +85,19 @@ jobs:
cd backend
python3.11 -m venv venv

- name: Install system libraries for the PDF report (WeasyPrint)
run: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends libpango-1.0-0 libpangoft2-1.0-0 libharfbuzz-subset0 fonts-dejavu-core

- name: Install packages
run: |
cd backend
source venv/bin/activate
pip install --upgrade pip
find . -name 'requirements.txt' -exec pip install -r {} \;
# test-only PDF inspection tools; versions come from requirements-dev.txt
grep -E '^(pypdf|PyMuPDF)' requirements-dev.txt | xargs pip install

- name: Install ifcopenshell package (temp)
run: |
Expand Down Expand Up @@ -125,3 +132,9 @@ jobs:
python3 -m pytest apps/ifc_validation/checks/statistics/tests -v
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_report_view --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_step_lines --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_ui --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_report_html --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_report_pdf --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_report_values --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation_bff.tests.tests_model_summary --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3
MEDIA_ROOT=./apps/ifc_validation/fixtures python3 manage.py test apps.ifc_validation.tests.tests_file_retention_task --settings apps.ifc_validation.test_settings --debug-mode --verbosity 3
213 changes: 213 additions & 0 deletions .github/workflows/playwright-htmx.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,213 @@
name: Playwright HTMX Tests
on:
workflow_dispatch:
push:
branches: development
paths-ignore:
- 'README.md'
- '.github/workflows/ci.yml'
- '.github/workflows/ci_cd.yml'
pull_request:
branches: development

permissions:
contents: read
actions: read
checks: write
pull-requests: write

jobs:
playwright:
name: 'HTMX (${{ matrix.mount }})'
strategy:
fail-fast: false
matrix:
include:
- frontend_mode: htmx
mount: /
- frontend_mode: react
mount: /v2/
runs-on: ubuntu-latest
container:
image: mcr.microsoft.com/playwright:v1.60.0-noble
env:
# Exercise the HTMX UI at / and /v2/; neither job starts React.
FRONTEND_MODE: ${{ matrix.frontend_mode }}
DJANGO_DB: postgresql
POSTGRES_DB: postgres
POSTGRES_HOST: postgres
POSTGRES_NAME: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PORT: 5432
MEDIA_ROOT: /tmp/files_storage
CELERY_BROKER_URL: redis://redis:6379/0
CELERY_RESULT_BACKEND: django-db
DJANGO_ALLOWED_HOSTS: "localhost 127.0.0.1"
DJANGO_TRUSTED_ORIGINS: "http://localhost:3000 http://127.0.0.1:3000 http://localhost:8000 http://127.0.0.1:8000"
POSTGRES_USER: postgres
VIRTUAL_ENV: backend/.dev/venv
HTMX_BASE_URL: http://localhost:8000${{ matrix.mount }}
HTMX_SERVER_COMMAND: .dev/venv/bin/python manage.py runserver 8000 --noreload
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

services:
postgres:
image: postgres:16.8-alpine
env:
POSTGRES_DB: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
options: >-
--health-cmd pg_isready
--health-interval 10s
--health-timeout 5s
--health-retries 5
--network-alias postgres
ports:
- 5432:5432
redis:
image: redis:8.4-alpine
options: >-
--health-cmd "redis-cli ping"
--health-interval 10s
--health-timeout 5s
--health-retries 5
ports:
- 6379:6379
volumes:
- redis_data:/data

steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive

- name: Prepare MEDIA_ROOT
run: |
mkdir -p /tmp/files_storage
chmod -R 777 /tmp/files_storage

- name: Install system dependencies
run: |
apt-get update
apt-get install -y make wget unzip libpq-dev libpango-1.0-0 libpangoft2-1.0-0

- name: Setup Node.js
uses: actions/setup-node@v4
with:
node-version: lts/*

- name: Setup Python
uses: actions/setup-python@v5
with:
python-version: 3.11

- name: Create venv
run: |
cd backend
python3.11 -m venv .dev/venv --clear
echo "venv/bin folder content:"
ls -l .dev/venv/bin

- name: Install backend dependencies
run: |
cd backend
. .dev/venv/bin/activate
pip install --upgrade pip
find . -name 'requirements.txt' -exec pip install -r {} \;

- name: Download and install ifcopenshell
run: |
wget -O /tmp/ifcopenshell_python.zip "https://s3.amazonaws.com/ifcopenshell-builds/ifcopenshell-python-311-v0.8.6-8b5b400-linux64.zip"
mkdir -p .dev/venv/lib/python3.11/site-packages
unzip -o -d .dev/venv/lib/python3.11/site-packages /tmp/ifcopenshell_python.zip
rm /tmp/ifcopenshell_python.zip
make install-ifcopenshell-mvd IFCOPENSHELL_SITE_PACKAGES=.dev/venv/lib/python3.11/site-packages
# Verify installation
ls -la .dev/venv/lib/python3.11/site-packages/
echo "Checking for ifcopenshell installation:"
find .dev/venv/lib/python3.11/site-packages -name "*ifcopenshell*" || echo "No ifcopenshell files found"
working-directory: ./backend

- name: Cache e2e dependencies
uses: actions/cache@v4
with:
path: e2e/node_modules
key: ${{ runner.os }}-e2e-${{ hashFiles('e2e/package-lock.json') }}
restore-keys: |
${{ runner.os }}-e2e-

- name: Install e2e dependencies (replicating make e2e-test)
run: npm install
working-directory: ./e2e

- name: Setup Django database, migrations, and superusers
run: |
cd backend
.dev/venv/bin/python manage.py migrate
.dev/venv/bin/python manage.py collectstatic --noinput
DJANGO_SUPERUSER_USERNAME=root DJANGO_SUPERUSER_PASSWORD=root DJANGO_SUPERUSER_EMAIL=root@localhost .dev/venv/bin/python manage.py createsuperuser --noinput
DJANGO_SUPERUSER_USERNAME=SYSTEM DJANGO_SUPERUSER_PASSWORD=system DJANGO_SUPERUSER_EMAIL=system@localhost .dev/venv/bin/python manage.py createsuperuser --noinput || echo "SYSTEM user already exists, skipping"
.dev/venv/bin/python manage.py seed_dummy_requests
env:
DJANGO_DB: postgresql
POSTGRES_HOST: postgres
POSTGRES_NAME: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PORT: 5432
VIRTUAL_ENV: backend/.dev/venv

- name: Check admin routes in both frontend modes
working-directory: ./backend
run: |
for mode in react htmx; do
FRONTEND_MODE=$mode .dev/venv/bin/python manage.py test core.tests.test_urls
done

- name: Start Celery worker
run: |
cd backend
nohup .dev/venv/bin/python -m celery --app=core worker \
--loglevel=INFO --concurrency 2 --hostname=worker@%n \
> /tmp/celery.log 2>&1 &
sleep 3
env:
DJANGO_DB: postgresql
POSTGRES_HOST: postgres
POSTGRES_NAME: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PORT: 5432
VIRTUAL_ENV: backend/.dev/venv

- name: Run HTMX end-to-end tests
run: npm run test:htmx
working-directory: ./e2e
env:
DJANGO_DB: postgresql
POSTGRES_HOST: postgres
POSTGRES_NAME: postgres
POSTGRES_USER: postgres
POSTGRES_PASSWORD: postgres
POSTGRES_PORT: 5432

- name: Run standalone HTMX browser fixtures
working-directory: ./e2e
run: |
export PATH="$GITHUB_WORKSPACE/backend/.dev/venv/bin:$PATH"
export HTMX_BASE_URL="http://localhost:8765${{ matrix.mount }}"
npm run test:htmx-fixtures

- name: Upload test results
uses: actions/upload-artifact@v4
if: always()
with:
name: playwright-htmx-${{ matrix.frontend_mode }}
path: |
e2e/test-results/htmx/
e2e/test-results/htmx-fixtures/
e2e/playwright-report/htmx/
/tmp/celery.log
retention-days: 7
9 changes: 9 additions & 0 deletions .github/workflows/playwright.yml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ jobs:
container:
image: mcr.microsoft.com/playwright:v1.60.0-noble
env:
# This suite exercises the React UI; HTMX has its own report-parity suite.
FRONTEND_MODE: react
DJANGO_DB: postgresql
POSTGRES_DB: postgres
POSTGRES_HOST: postgres
Expand Down Expand Up @@ -159,6 +161,13 @@ jobs:
POSTGRES_PORT: 5432
VIRTUAL_ENV: backend/.dev/venv

- name: Check admin routes in both frontend modes
working-directory: ./backend
run: |
for mode in react htmx; do
FRONTEND_MODE=$mode .dev/venv/bin/python manage.py test core.tests.test_urls
done

- name: Start Celery worker
run: |
cd backend
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ stop:
REGISTRY ?= localhost:5000
WORKERS ?= 2
ENV_FILE ?= .env
SWARM_VARS = REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP TEXTFILE_DIR NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION WORKER_REPLICAS BACKEND_MEMORY_LIMIT SCHEDULER_MEMORY_LIMIT AV_WORKER_MEMORY_LIMIT
SWARM_VARS = FRONTEND_MODE REGISTRY CERTBOT_DOMAIN CERTBOT_EMAIL NFS_SERVER_IP TEXTFILE_DIR NFS_BASE_PATH WORKER_CPU_LIMIT WORKER_MEMORY_LIMIT WORKER_CPU_RESERVATION WORKER_MEMORY_RESERVATION WORKER_REPLICAS BACKEND_MEMORY_LIMIT SCHEDULER_MEMORY_LIMIT AV_WORKER_MEMORY_LIMIT
SWARM_ENV = ENV_FILE="$(ENV_FILE)" $(foreach v,$(SWARM_VARS),$(v)="$(shell grep '^$(v)=' $(ENV_FILE) | head -1 | cut -d= -f2-)")

start-swarm:
Expand Down
7 changes: 6 additions & 1 deletion backend/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ stop-worker:
-$(PYTHON) -m celery -A core control shutdown \
--destination=worker@$(shell hostname) || true

test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task test-utils test-file-retention-task test-status-combine test-management-commands test-report-views
test: test-models test-header-validation-task test-syntax-task test-syntax-header-validation-task test-schema-task test-magic-and-av-task test-utils test-file-retention-task test-status-combine test-management-commands test-report-views test-ui-and-reports

test-models:
MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps/ifc_validation_models --settings apps.ifc_validation_models.test_settings --debug-mode --verbosity 3
Expand Down Expand Up @@ -113,6 +113,11 @@ test-management-commands:
test-report-views:
MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation_bff.tests.tests_report_view --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3

# HTMX UI, HTML report and PDF report. The PDF tests need the test-only packages:
# pip install $$(grep -E '^(pypdf|PyMuPDF)' requirements-dev.txt)
test-ui-and-reports:
MEDIA_ROOT=./apps/ifc_validation/fixtures $(PYTHON) manage.py test apps.ifc_validation_bff.tests.tests_ui apps.ifc_validation_bff.tests.tests_report_html apps.ifc_validation_bff.tests.tests_report_pdf apps.ifc_validation_bff.tests.tests_report_values apps.ifc_validation_bff.tests.tests_model_summary apps.ifc_validation_bff.tests.tests_step_lines --settings apps.ifc_validation_bff.tests.test_settings --debug-mode --verbosity 3

test-utils:
$(PYTHON) manage.py test core.tests.test_utils --debug-mode --verbosity 3

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -183,4 +183,4 @@ def handle(self, *args, **options):
else:
logger.info(
f"Completed {mode_str} of {processed}, skipped {skipped}, total considered {total}. Freed up {savings_str}."
)
)
25 changes: 25 additions & 0 deletions backend/apps/ifc_validation/tasks/report_tasks.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
"""Post-validation exports. A rendering failure never changes validation results."""
from celery import shared_task
from celery.exceptions import SoftTimeLimitExceeded

from apps.ifc_validation_models.decorators import requires_django_user_context

from .logger import logger


# acks_late=False: with the global acks_late + reject_on_worker_lost, a render whose worker
# is killed (e.g. out of memory) would be redelivered without end.
@shared_task(bind=True, max_retries=2, soft_time_limit=120, time_limit=150, acks_late=False)
@requires_django_user_context
def generate_validation_report(self, request_id):
from apps.ifc_validation_bff.report.pdf import generate_pdf

try:
path = generate_pdf(request_id)
return str(path) if path else None
except SoftTimeLimitExceeded:
# A report that does not render within the limit will not render on a retry either.
logger.warning(f"PDF for validation request {request_id} exceeded the time limit; not retrying.")
return None
except Exception as error:
raise self.retry(exc=error, countdown=30 * (self.request.retries + 1))
7 changes: 7 additions & 0 deletions backend/apps/ifc_validation/tasks/task_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
from .logger import logger
from .email_tasks import *
from .file_retention_tasks import *
from .report_tasks import generate_validation_report


def terminate_subprocesses():
Expand Down Expand Up @@ -149,6 +150,10 @@ def on_workflow_completed(self, result, **kwargs):
# queue sending email
send_completion_email_task.delay(id=id, file_name=request.file_name)

# The immutable export is a post-validation artifact, outside the check registry
# and its progress/blocker semantics. PDF errors are retried independently.
generate_validation_report.delay(request_id=id)


@shared_task(bind=True)
@log_execution
Expand All @@ -174,6 +179,8 @@ def on_workflow_failed(self, *args, **kwargs):
reason += f" Error: {exc}"
request.mark_as_failed(reason)

generate_validation_report.delay(request_id=id)

# queue sending email
send_failure_email_task.delay(id=id, file_name=request.file_name)
send_failure_admin_email_task.delay(id=id, file_name=request.file_name)
Expand Down
Loading
Loading