CI: run the tests and a smoke test on every PR

.github/scripts/make_rid_fixture.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+# A couple of lines and stops for import_rid, so the smoke test has data to show.
+set -euo pipefail
+
+dir="$1"
+mkdir -p "$dir"
+
+cat > "$dir/openebs_lines.csv" <<'EOF'
+bison_id,publiccode,name
+HTM:1,1,Scheveningen Noorderstrand - Delft Tanthof
+HTM:9,9,Scheveningen Noorderstrand - Den Haag Vrederust
+EOF
+
+cat > "$dir/openebs_stops.csv" <<'EOF'
+operator_id,name,latitude,longitude,timingpointcode,quaycoderef
+HTM:3100,Den Haag Centraal,52.080887,4.324971,31001000,
+HTM:3200,Den Haag Hollands Spoor,52.069637,4.322389,31002000,
+EOF

.github/scripts/smoke.py

@@ -0,0 +1,80 @@
+#!/usr/bin/env python3
+"""Log in to a running server and check the main pages come back.
+
+Usage: smoke.py http://127.0.0.1:8000
+"""
+import http.cookiejar
+import os
+import re
+import sys
+import urllib.parse
+import urllib.request
+
+BASE = sys.argv[1].rstrip('/')
+USERNAME = os.environ.get('SMOKE_USER', 'smoke')
+PASSWORD = os.environ.get('SMOKE_PASSWORD', 'smoke-password')
+
+PAGES = [
+    ('/bericht', None),
+    ('/bericht/nieuw', None),
+    ('/kaart', 'autocomplete_holder'),  # The stop search box, see #234
+    ('/haltes.geojson', 'FeatureCollection'),
+    ('/stop/search.json?q=Den+Haag', 'Den Haag'),
+    ('/scenario', None),
+    ('/ritaanpassing', None),
+    ('/admin/', None),
+]
+
+opener = urllib.request.build_opener(
+    urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
+failures = []
+
+
+def fetch(path, data=None):
+    url = BASE + path
+    request = urllib.request.Request(url, data=data, headers={'Referer': url})
+    with opener.open(request) as response:
+        return response.getcode(), response.read().decode('utf-8', 'replace')
+
+
+def check(path, contains=None):
+    try:
+        status, body = fetch(path)
+    except Exception as error:  # urllib raises on 4xx/5xx
+        failures.append('%s -> %s' % (path, error))
+        print('FAIL %s (%s)' % (path, error))
+        return
+    if status != 200:
+        failures.append('%s -> %s' % (path, status))
+    elif contains and contains not in body:
+        failures.append('%s -> 200 but missing %r' % (path, contains))
+    else:
+        print('ok   %s (%s)' % (path, status))
+        return
+    print('FAIL %s' % failures[-1])
+
+
+def login():
+    _, body = fetch('/inloggen/')
+    match = re.search(r'name="csrfmiddlewaretoken" value="([^"]+)"', body)
+    if not match:
+        sys.exit('no csrf token on the login page')
+    data = urllib.parse.urlencode({
+        'csrfmiddlewaretoken': match.group(1),
+        'username': USERNAME,
+        'password': PASSWORD,
+    }).encode()
+    _, body = fetch('/inloggen/', data=data)
+    if 'Uitloggen' not in body:
+        sys.exit('login as %s failed' % USERNAME)
+    print('ok   logged in as %s' % USERNAME)
+
+
+check('/inloggen/')
+login()
+for page, contains in PAGES:
+    check(page, contains)
+
+if failures:
+    sys.exit('\n%d check(s) failed:\n  %s' % (len(failures), '\n  '.join(failures)))
+print('\nall checks passed')

.github/scripts/write_local_settings.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# settings.py requires local_settings to exist - point it at the CI database.
+set -euo pipefail
+
+cat > openebs2/local_settings.py <<'EOF'
+DEBUG = True
+
+ALLOWED_HOSTS = ['localhost', '127.0.0.1']
+
+SOCIAL_LOGIN_ENABLED = False  # No SSO provider in CI
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.contrib.gis.db.backends.postgis',
+        'NAME': 'openebs2',
+        'USER': 'openebs',
+        'PASSWORD': 'openebs',
+        'HOST': '127.0.0.1',
+        'PORT': '5432',
+    }
+}
+EOF

.github/workflows/ci.yml

@@ -0,0 +1,119 @@
+name: CI
+
+on:
+  pull_request:
+  push:
+    branches: [master]
+
+env:
+  PYTHON_VERSION: '3.13'
+
+jobs:
+  test:
+    name: Tests
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgis/postgis:17-3.5
+        env:
+          POSTGRES_USER: openebs
+          POSTGRES_PASSWORD: openebs
+          POSTGRES_DB: openebs2
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+
+      - name: Install GeoDjango system libraries
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libgeos-dev
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Write local_settings
+        run: .github/scripts/write_local_settings.sh
+
+      - name: Run tests
+        run: python manage.py test
+
+  smoke:
+    name: Smoke test
+    runs-on: ubuntu-latest
+
+    services:
+      postgres:
+        image: postgis/postgis:17-3.5
+        env:
+          POSTGRES_USER: openebs
+          POSTGRES_PASSWORD: openebs
+          POSTGRES_DB: openebs2
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_VERSION }}
+          cache: pip
+
+      - name: Install GeoDjango system libraries
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y binutils libproj-dev gdal-bin libgdal-dev libgeos-dev
+
+      - name: Install dependencies
+        run: pip install -r requirements.txt
+
+      - name: Write local_settings
+        run: .github/scripts/write_local_settings.sh
+
+      - name: Migrate
+        run: python manage.py migrate --noinput
+
+      - name: Import stop and line data
+        run: |
+          .github/scripts/make_rid_fixture.sh "${RUNNER_TEMP}/rid"
+          python manage.py import_rid "${RUNNER_TEMP}/rid"
+
+      - name: Create the smoke test user
+        run: |
+          python manage.py createsuperuser --noinput --username smoke --email smoke@example.com
+        env:
+          DJANGO_SUPERUSER_PASSWORD: smoke-password
+
+      - name: Start the server
+        run: |
+          python manage.py runserver 8000 --noreload > "${RUNNER_TEMP}/server.log" 2>&1 &
+          for i in $(seq 30); do
+            curl -sf -o /dev/null http://127.0.0.1:8000/inloggen/ && exit 0
+            sleep 1
+          done
+          echo "server did not come up"; cat "${RUNNER_TEMP}/server.log"; exit 1
+
+      - name: Check the pages load
+        run: python .github/scripts/smoke.py http://127.0.0.1:8000
+
+      - name: Server log
+        if: always()
+        run: cat "${RUNNER_TEMP}/server.log"

ferry/management/commands/sendkv6.py

@@ -46,7 +46,7 @@ def handle_ferry(self, ferry):
             try:
                 msg = FerryKv6Messages.objects.get(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber,
                                                    status=FerryKv6Messages.Status.READY, cancelled=False)
-                if msg.delay > 0 and journey.departuretime + msg.delay > depart_target:
+                if msg.delay is not None and msg.delay > 0 and journey.departuretime + msg.delay > depart_target:
                     continue
 
                 msg.status = FerryKv6Messages.Status.DEPARTED
@@ -66,7 +66,7 @@ def handle_ferry(self, ferry):
             try:
                 msg = FerryKv6Messages.objects.get(operatingday=date, ferry=ferry, journeynumber=journey.journeynumber,
                                                    status=FerryKv6Messages.Status.DEPARTED, cancelled=False)
-                if msg.delay > 0 and journey.departuretime + msg.delay > arrival_target:
+                if msg.delay is not None and msg.delay > 0 and journey.departuretime + msg.delay > arrival_target:
                     continue
 
                 msg.status = FerryKv6Messages.Status.ARRIVED

kv1/tests/__init__.py

@@ -1,9 +0,0 @@
-import pkgutil
-import unittest
-
-for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
-    module = loader.find_module(module_name).load_module(module_name)
-    for name in dir(module):
-        obj = getattr(module, name)
-        if isinstance(obj, type) and issubclass(obj, unittest.case.TestCase):
-            exec ('%s = obj' % obj.__name__)

kv1/tests/test_import_rid.py

@@ -1,4 +1,3 @@
-from unittest.test.test_case import Test
 
 from django.core.management import call_command
 from django.test import TestCase
@@ -45,12 +44,12 @@ def testStopsSimple(self):
 
         # Data
         lines = [
-            ['operator_id','publiccode','name'],
+            ['bison_id','publiccode','name'],
             ['VTN:1049','62','Gulpen - Vaals'],
         ]
         stops = [
-            ['operator_id','name','longitude','latitude','timingpointcode',],
-            ['VTN:15023014','Busstation Perron C', '1', '2','15023014']  # Load an updated name
+            ['operator_id','name','longitude','latitude','timingpointcode','quaycoderef'],
+            ['VTN:15023014','Busstation Perron C', '1', '2','15023014','NL:Q:15023014']  # Load an updated name
         ]
         self.createTestFile('openebs_lines.csv', lines)
         self.createTestFile('openebs_stops.csv', stops)

openebs/management/commands/verify_messages.py

@@ -96,10 +96,17 @@ def process_message(self, row, deleted):
             self.add_stop_for_message(msg, row)
 
         else:
-            msg, created = Kv15Stopmessage.objects.get_or_create(dataownercode=row['DataOwnerCode'],
-                                                                 kv8messagecodedate=row['MessageCodeDate'],
-                                                                 kv8messagecodenumber=row['MessageCodeNumber'],
-                                                                 defaults={'user': self.get_user()})
+            # Updating a message keeps the superseded version around, and it carries the same
+            # KV8 identity, so match the most recent one instead of tripping over the duplicates
+            msg = Kv15Stopmessage.objects.filter(dataownercode=row['DataOwnerCode'],
+                                                 kv8messagecodedate=row['MessageCodeDate'],
+                                                 kv8messagecodenumber=row['MessageCodeNumber']).order_by('id').last()
+            created = msg is None
+            if created:
+                msg = Kv15Stopmessage.objects.create(dataownercode=row['DataOwnerCode'],
+                                                     kv8messagecodedate=row['MessageCodeDate'],
+                                                     kv8messagecodenumber=row['MessageCodeNumber'],
+                                                     user=self.get_user())
 
             if not created:
                 self.log.info("Message confirmed deleted: %s (Stop/TPC %s)" % (msg, row['TimingPointCode']))

openebs/models.py

@@ -342,7 +342,7 @@ def plan_messages(self, user, start, end):
 
     def delete_all(self):
         msgs = []
-        for inst in Kv15ScenarioInstance.objects.filter(scenario=self, message__messageendtime__gt=now):
+        for inst in Kv15ScenarioInstance.objects.filter(scenario=self, message__messageendtime__gt=now()):
             inst.message.delete()
             msgs.append(inst.message.to_xml_delete())
 

openebs/templates/xml/kv17journey.xml

@@ -47,7 +47,7 @@
     {% endif %}
 </KV17MUTATEJOURNEY>
 {% endif %}
-{% if object.journey_details_mutation_message.count > 0 or object.shorten_details.count > 0 and not object.is_recovered %}
+{% if object.journey_details_mutation_message.count > 0 or object.shorten_details.count > 0 %}{% if not object.is_recovered %}
 <KV17MUTATEJOURNEYSTOP>
     <timestamp>{{ object.created|date:"c" }}</timestamp>
 {% for shorten in object.shorten_details.all %}
@@ -75,4 +75,4 @@
     </MUTATIONMESSAGE>
 {% endfor %}
 </KV17MUTATEJOURNEYSTOP>
-{% endif %}
+{% endif %}{% endif %}

openebs/tests/__init__.py

@@ -1,9 +0,0 @@
-import pkgutil
-import unittest
-
-for loader, module_name, is_pkg in pkgutil.walk_packages(__path__):
-    module = loader.find_module(module_name).load_module(module_name)
-    for name in dir(module):
-        obj = getattr(module, name)
-        if isinstance(obj, type) and issubclass(obj, unittest.case.TestCase):
-            exec ('%s = obj' % obj.__name__)

openebs/tests/output/kv17_cancel.xml

@@ -8,6 +8,9 @@
     </KV17JOURNEY>
     <KV17MUTATEJOURNEY>
         <timestamp>...</timestamp>
-        <CANCEL></CANCEL>
+        <CANCEL>
+            <showcancelledtrip>true</showcancelledtrip>
+            <autorecover>false</autorecover>
+        </CANCEL>
     </KV17MUTATEJOURNEY>
 </DOSSIER>

openebs/tests/output/kv17_mutationmessage.xml

@@ -14,6 +14,7 @@
             <reasontype>3</reasontype>
             <subreasontype>7</subreasontype>
             <reasoncontent>Boot is vol</reasoncontent>
+            <showcancelledtrip>true</showcancelledtrip>
         </MUTATIONMESSAGE>
     </KV17MUTATEJOURNEYSTOP>
 </DOSSIER>

openebs/tests/output/kv17_mutationmessage_cancel.xml

@@ -8,7 +8,10 @@
     </KV17JOURNEY>
     <KV17MUTATEJOURNEY>
         <timestamp>...</timestamp>
-        <CANCEL></CANCEL>
+        <CANCEL>
+            <showcancelledtrip>true</showcancelledtrip>
+            <autorecover>false</autorecover>
+        </CANCEL>
     </KV17MUTATEJOURNEY>
     <KV17MUTATEJOURNEYSTOP>
         <timestamp>...</timestamp>
@@ -18,6 +21,7 @@
             <reasontype>3</reasontype>
             <subreasontype>7</subreasontype>
             <reasoncontent>Boot is vol en vaart niet</reasoncontent>
+            <showcancelledtrip>true</showcancelledtrip>
         </MUTATIONMESSAGE>
     </KV17MUTATEJOURNEYSTOP>
 </DOSSIER>