johannes/transport-accessibility
SHA256
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Got rid of bridge and fixed GTFS compliance

Browse Source 
- Ported the bridge that was using a custom GTFS class and Pandas
  Dataframes to a native Django solution for and interface between db
  and csv (see api/io.py)
- Fixed some issues regarding the compliance of the exported csv files
  with the GTFS reference. I.e. now allowing times 24:00:00 <= t >=
  24:59:59
This commit is contained in:
Johannes Randerath
2024-06-24 14:21:53 +02:00
parent 1494dba808
commit f314bfb396
13 changed files with 955 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
transport_accessibility/pt_map/bridge.py
View File

@@ -234,6 +234,7 @@ def db_to_gtfs(q: list[django.db.models.query.QuerySet], folder_path: str = ""):
object containing the queried data
"""
dfs = {reversed_file_mapping[m.model.__name__]: (pd.DataFrame(list(m.values())) if m else pd.DataFrame()) for m in q}
dfs = {key: dfs[key].astype({col: pd.Timestamp for col in dfs[key].columns if isinstance(getattr(getattr(pt_map.models, {v:k for k,v in reversed_file_mapping.items()}[key]), col), django.db.models.DateField)}) for key in dfs.keys()}
g = pt_map.gtfs.GTFS(folder_path, dfs)
g.validate()
return g

65
transport_accessibility/pt_map/class_names.py
View File

@@ -139,6 +139,40 @@ pt_map.models.Shape: "shapes",
]
file_names = {
pt_map.models.Agency: "agency.txt",
pt_map.models.Stop: "stops.txt",
pt_map.models.Route: "routes.txt",
pt_map.models.Trip: "trips.txt",
pt_map.models.StopTime: "stop_times.txt",
pt_map.models.Calendar: "calendar.txt",
pt_map.models.CalendarDate: "calendar_dates.txt",
pt_map.models.FareAttribute: "fare_attributes.txt",
pt_map.models.FareRule: "fare_rules.txt",
pt_map.models.Timeframe: "timeframes.txt",
pt_map.models.FareMedium: "fare_media.txt",
pt_map.models.FareProduct: "fare_products.txt",
pt_map.models.FareLegRule: "fare_leg_rules.txt",
pt_map.models.FareTransferRule: "fare_transfer_rules.txt",
pt_map.models.Area: "areas.txt",
pt_map.models.StopArea: "stop_areas.txt",
pt_map.models.Network: "networks.txt",
pt_map.models.RouteNetwork: "route_networks.txt",
pt_map.models.Shape: "shapes.txt",
pt_map.models.Frequency: "frequencies.txt",
pt_map.models.Transfer: "transfers.txt",
pt_map.models.Pathway: "pathways.txt",
pt_map.models.Level: "levels.txt",
pt_map.models.LocationGroup: "location_groups.txt",
pt_map.models.LocationGroupStop: "location_group_stops.txt",
pt_map.models.LocationsGeojson: "locations.geojson",
pt_map.models.BookingRule: "booking_rules.txt",
pt_map.models.Translation: "translations.txt",
pt_map.models.FeedInfo: "feed_info.txt",
pt_map.models.Attribution: "attributions.txt",
}
reversed_file_mapping = {
"Agency": "agency",
"Stop": "stops",
@@ -276,4 +310,35 @@ foreign_keys = [
(pt_map.models.FareTransferRule, [(pt_map.models.FeedInfo, 'feed_info_id'),(pt_map.models.FareProduct, 'fare_product_id'), ]),
]
mtm = {
'shape_id': pt_map.models.Shape,
'from_timeframe_group_id': pt_map.models.Timeframe,
'to_timeframe_group_id': pt_map.models.Timeframe,
}
fks = {
'feed_info_id_id': pt_map.models.FeedInfo,
'parent_station': pt_map.models.Stop,
'level_id': pt_map.models.Level,
'agency_id': pt_map.models.Agency,
'route_id': pt_map.models.Route,
'trip_id': pt_map.models.Trip,
'stop_id': pt_map.models.Stop,
'location_group_id': pt_map.models.LocationGroup,
'location_id': pt_map.models.LocationsGeojson,
'fare_id': pt_map.models.FareAttribute,
'from_stop_id': pt_map.models.Stop,
'to_stop_id': pt_map.models.Stop,
'from_route_id': pt_map.models.Route,
'to_route_id': pt_map.models.Route,
'from_trip_id': pt_map.models.Trip,
'to_trip_id': pt_map.models.Trip,
'network_id': pt_map.models.Network,
'area_id': pt_map.models.Area,
'from_area_id': pt_map.models.Area,
'to_area_id': pt_map.models.Area,
'fare_product_id': pt_map.models.FareProduct,
}
fk_dict = {fk[0]: fk[1] for fk in foreign_keys}

3
transport_accessibility/pt_map/gtfs.py
View File

@@ -405,12 +405,13 @@ class GTFS:
os.mkdir(path)
for name in self.get_files():
df = getattr(self, name).data
df = df.astype({col: 'int8' for col in df.columns if df[col].dtype == 'bool'})
fpath = f"{path}/{name}.txt"
if name == 'locations_geojson':
fpath = f"{path}/{name}.geojson"
df.to_json(fpath)
else:
df.to_csv(fpath, index=False)
df.to_csv(fpath, date_format='%Y%m%d', index=False)
def validate(self):
"""

22
transport_accessibility/pt_map/migrations/0004_remove_trip_shape_id_trip_shape_id.py Normal file
View File

@@ -0,0 +1,22 @@
# Generated by Django 5.0.6 on 2024-06-22 09:38
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pt_map', '0003_faretransferrule_feed_info_id'),
]
operations = [
migrations.RemoveField(
model_name='trip',
name='shape_id',
),
migrations.AddField(
model_name='trip',
name='shape_id',
field=models.ManyToManyField(blank=True, related_name='shape', to='pt_map.shape'),
),
]

18
transport_accessibility/pt_map/migrations/0005_alter_trip_shape_id.py Normal file
View File

@@ -0,0 +1,18 @@
# Generated by Django 5.0.6 on 2024-06-22 09:40
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pt_map', '0004_remove_trip_shape_id_trip_shape_id'),
]
operations = [
migrations.AlterField(
model_name='trip',
name='shape_id',
field=models.ManyToManyField(blank=True, related_name='trips', to='pt_map.shape'),
),
]

53
transport_accessibility/pt_map/migrations/0006_alter_bookingrule_end_time_and_more.py Normal file
View File

@@ -0,0 +1,53 @@
# Generated by Django 5.0.6 on 2024-06-23 20:37
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pt_map', '0005_alter_trip_shape_id'),
]
operations = [
migrations.AlterField(
model_name='bookingrule',
name='end_time',
field=models.CharField(blank=True, max_length=7, null=True),
),
migrations.AlterField(
model_name='bookingrule',
name='start_time',
field=models.CharField(blank=True, max_length=7, null=True),
),
migrations.AlterField(
model_name='frequency',
name='end_time',
field=models.CharField(max_length=7),
),
migrations.AlterField(
model_name='frequency',
name='start_time',
field=models.CharField(max_length=7),
),
migrations.AlterField(
model_name='stoptime',
name='arrival_time',
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AlterField(
model_name='stoptime',
name='departure_time',
field=models.CharField(blank=True, max_length=7, null=True),
),
migrations.AlterField(
model_name='timeframe',
name='end_time',
field=models.CharField(max_length=7),
),
migrations.AlterField(
model_name='timeframe',
name='start_time',
field=models.CharField(max_length=7),
),
]

48
transport_accessibility/pt_map/migrations/0007_alter_bookingrule_end_time_and_more.py Normal file
View File

@@ -0,0 +1,48 @@
# Generated by Django 5.0.6 on 2024-06-23 21:26
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pt_map', '0006_alter_bookingrule_end_time_and_more'),
]
operations = [
migrations.AlterField(
model_name='bookingrule',
name='end_time',
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AlterField(
model_name='bookingrule',
name='start_time',
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AlterField(
model_name='frequency',
name='end_time',
field=models.CharField(max_length=255),
),
migrations.AlterField(
model_name='frequency',
name='start_time',
field=models.CharField(max_length=255),
),
migrations.AlterField(
model_name='stoptime',
name='departure_time',
field=models.CharField(blank=True, max_length=255, null=True),
),
migrations.AlterField(
model_name='timeframe',
name='end_time',
field=models.CharField(max_length=255),
),
migrations.AlterField(
model_name='timeframe',
name='start_time',
field=models.CharField(max_length=255),
),
]

23
transport_accessibility/pt_map/migrations/0008_alter_timeframe_service_id_alter_trip_service_id.py Normal file
View File

@@ -0,0 +1,23 @@
# Generated by Django 5.0.6 on 2024-06-23 23:00
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('pt_map', '0007_alter_bookingrule_end_time_and_more'),
]
operations = [
migrations.AlterField(
model_name='timeframe',
name='service_id',
field=models.CharField(max_length=255),
),
migrations.AlterField(
model_name='trip',
name='service_id',
field=models.CharField(max_length=255),
),
]

600
transport_accessibility/pt_map/model_test_fields.py Normal file
View File

@@ -0,0 +1,600 @@
"""
Constants defining requirements of model fields.
To be used to decide wether a test should run or fail and which tests to run to test for self-preserved integrity.
"""
from pt_map.models import *
field_requirements = \
[
{
"model": Agency,
"fields": [
{
"name": "agency_id",
"type": "pk",
"required": "true",
},
{
"name": "agency_name",
"type": "str",
"required": "true",
},
{
"name": "agency_url",
"type": "url",
"required": "true",
},
{
"name": "agency_timezone",
"type": "timezone",
"required": "true",
},
{
"name": "agency_lang",
"type": "langcode",
"required": "false",
},
{
"name": "agency_phone",
"type": "telephone",
"required": "false",
},
{
"name": "agency_email",
"type": "email",
"required": "false",
},
],
},
{
"model": "Stop",
"fields": [
{
"name": "stop_id",
"type": "pk",
"required": "true",
},
{
"name": "stop_code",
"type": "str",
"required": "false",
},
{
"name": "stop_name",
"type": "str",
"required": "true",
},
{
"name": "stop_desc",
"type": "str",
"required": "false",
},
{
"name": "stop_lat",
"type": "float",
"required": "if",
"required_if": ["location_type", [0,1,2]],
},
{
"name": "stop_lon",
"type": "float",
"required": "if",
"required_if": ["location_type", [0,1,2]],
},
{
"name": "zone_id",
"type": "str",
"required": "false",
},
{
"name": "stop_url",
"type": "url",
"required": "false",
},
{
"name": "location_type",
"type": "int",
"required": "true",
},
{
"name": "parent_station",
"type": "fk",
"required": "if",
"required_if": ["location_type", [2,3,4]],
"forbidden_if": ["location_type", [1]],
},
{
"name": "stop_timezone",
"type": "timezone",
"required": "false",
},
{
"name": "wheelchair_boarding",
"type": "int",
"required": "false",
},
],
},
{
"model": "Stop",
"fields": [
{
"name": "stop_id",
"type": "pk",
"required": "true",
},
{
"name": "stop_code",
"type": "str",
"required": "false",
},
{
"name": "stop_name",
"type": "str",
"required": "true",
},
{
"name": "stop_desc",
"type": "str",
"required": "false",
},
{
"name": "stop_lat",
"type": "float",
"required": "true",
},
{
"name": "stop_lon",
"type": "float",
"required": "true",
},
{
"name": "zone_id",
"type": "str",
"required": "false",
},
{
"name": "stop_url",
"type": "url",
"required": "false",
},
{
"name": "location_type",
"type": "int",
"required": "true",
},
{
"name": "parent_station",
"type": "str",
"required": "false",
},
{
"name": "stop_timezone",
"type": "timezone",
"required": "false",
},
{
"name": "wheelchair_boarding",
"type": "int",
"required": "false",
},
],
},
{
"model": "Route",
"fields": [
{
"name": "route_id",
"type": "pk",
"required": "true",
},
{
"name": "route_short_name",
"type": "str",
"required": "false",
},
{
"name": "route_long_name",
"type": "str",
"required": "true",
},
{
"name": "route_desc",
"type": "str",
"required": "false",
},
{
"name": "route_type",
"type": "int",
"required": "true",
},
{
"name": "route_url",
"type": "url",
"required": "false",
},
{
"name": "route_color",
"type": "str",
"required": "false",
},
{
"name": "route_text_color",
"type": "str",
"required": "false",
}
],
},
{
"model": "Trip",
"fields": [
{
"name": "trip_id",
"type": "pk",
"required": "true",
},
{
"name": "route_id",
"type": "fk",
"required": "true",
},
{
"name": "service_id",
"type": "fk",
"required": "true",
},
{
"name": "trip_headsign",
"type": "str",
"required": "true",
},
{
"name": "trip_short_name",
"type": "str",
"required": "false",
},
{
"name": "direction_id",
"type": "int",
"required": "true",
},
{
"name": "block_id",
"type": "str",
"required": "false",
},
{
"name": "shape_id",
"type": "fk",
"required": "true",
},
],
},
{
"model": "StopTime",
"fields": [
{
"name": "trip_id",
"type": "fk",
"required": "true",
},
{
"name": "arrival_time",
"type": "time",
"required": "true",
},
{
"name": "departure_time",
"type": "time",
"required": "true",
},
{
"name": "stop_id",
"type": "fk",
"required": "true",
},
{
"name": "stop_sequence",
"type": "int",
"required": "true",
},
{
"name": "pickup_type",
"type": "int",
"required": "true",
},
{
"name": "drop_off_type",
"type": "int",
"required": "true",
},
{
"name": "shape_dist_traveled",
"type": "float",
"required": "false",
},
{
"name": "timepoint",
"type": "int",
"required": "false",
},
],
},
{
"model": "Calendar",
"fields": [
{
"name": "service_id",
"type": "pk",
"required": "true",
},
{
"name": "monday",
"type": "int",
"required": "true",
},
{
"name": "tuesday",
"type": "int",
"required": "true",
},
{
"name": "wednesday",
"type": "int",
"required": "true",
},
{
"name": "thursday",
"type": "int",
"required": "true",
},
{
"name": "friday",
"type": "int",
"required": "true",
},
{
"name": "saturday",
"type": "int",
"required": "true",
},
{
"name": "sunday",
"type": "int",
"required": "true",
},
{
"name": "start_date",
"type": "date",
"required": "true",
},
{
"name": "end_date",
"type": "date",
"required": "true",
},
],
},
{
"model": "CalendarDates",
"fields": [
{
"name": "service_id",
"type": "fk",
"required": "true",
},
{
"name": "date",
"type": "date",
"required": "true",
},
{
"name": "exception_type",
"type": "int",
"required": "true",
},
],
},
{
"model": "FareAttributes",
"fields": [
{
"name": "fare_id",
"type": "pk",
"required": "true",
},
{
"name": "price",
"type": "float",
"required": "true",
},
{
"name": "currency_type",
"type": "str",
"required": "true",
},
{
"name": "payment_method",
"type": "int",
"required": "true",
},
{
"name": "transfers",
"type": "int",
"required": "true",
},
{
"name": "transfer_duration",
"type": "int",
"required": "true",
},
],
},
{
"model": "FareRules",
"fields": [
{
"name": "fare_id",
"type": "fk",
"required": "true",
},
{
"name": "route_id",
"type": "fk",
"required": "true",
},
{
"name": "origin_id",
"type": "fk",
"required": "true",
},
{
"name": "destination_id",
"type": "fk",
"required": "true",
},
{
"name": "contains_id",
"type": "fk",
"required": "false",
},
],
},
{
"model": "FareZones",
"fields": [
{
"name": "fare_zone_id",
"type": "pk",
"required": "true",
},
{
"name": "zone_id",
"type": "str",
"required": "true",
},
],
},
{
"model": "Shape",
"fields": [
{
"name": "shape_id",
"type": "pk",
"required": "true",
},
{
"name": "shape_pt_lat",
"type": "float",
"required": "true",
},
{
"name": "shape_pt_lon",
"type": "float",
"required": "true",
},
{
"name": "shape_pt_sequence",
"type": "int",
"required": "true",
},
{
"name": "shape_dist_traveled",
"type": "float",
"required": "true",
},
],
},
{
"model": "Frequencies",
"fields": [
{
"name": "trip_id",
"type": "fk",
"required": "true",
},
{
"name": "start_time",
"type": "time",
"required": "true",
},
{
"name": "end_time",
"type": "time",
"required": "true",
},
{
"name": "headway_secs",
"type": "int",
"required": "true",
},
{
"name": "exact_times",
"type": "int",
"required": "true",
},
],
},
{
"model": "Transfers",
"fields": [
{
"name": "from_stop_id",
"type": "fk",
"required": "true",
},
{
"name": "to_stop_id",
"type": "fk",
"required": "true",
},
{
"name": "transfer_type",
"type": "int",
"required": "true",
},
{
"name": "min_transfer_time",
"type": "int",
"required": "true",
},
],
},
{
"model": "FeedInfo",
"fields": [
{
"name": "feed_publisher_name",
"type": "str",
"required": "true",
},
{
"name": "feed_publisher_url",
"type": "url",
"required": "true",
},
{
"name": "feed_lang",
"type": "langcode",
"required": "true",
},
{
"name": "feed_start_date",
"type": "date",
"required": "true",
},
{
"name": "feed_end_date",
"type": "date",
"required": "false",
},
{
"name": "feed_version",
"type": "str",
"required": "true",
},
],
},
]

22
transport_accessibility/pt_map/models.py
View File

@@ -139,12 +139,12 @@ class Trip(models.Model):
"""
trip_id = models.CharField(max_length=255, primary_key=True)
route_id = models.ForeignKey(Route, on_delete=models.CASCADE)
service_id = models.IntegerField()
service_id = models.CharField(max_length=255)
trip_headsign = models.CharField(max_length=255, blank=True, null=True)
trip_short_name = models.CharField(max_length=255, blank=True, null=True)
direction_id = models.IntegerField(blank=True, null=True)
block_id = models.CharField(max_length=255, blank=True, null=True)
shape_id = models.ForeignKey(Shape, on_delete=models.CASCADE, blank=True)
shape_id = models.ManyToManyField(Shape, related_name='trips', blank=True)
wheelchair_accessible = models.IntegerField(blank=True, null=True)
bikes_allowed = models.IntegerField(blank=True, null=True)
@@ -191,8 +191,8 @@ class StopTime(models.Model):
"""
stop_time_id = models.BigAutoField(primary_key=True)
trip_id = models.ForeignKey(Trip, on_delete=models.CASCADE)
arrival_time = models.TimeField(blank=True, null=True)
departure_time = models.TimeField(blank=True, null=True)
arrival_time = models.CharField(max_length=255, blank=True, null=True)
departure_time = models.CharField(max_length=255, blank=True, null=True)
stop_id = models.ForeignKey(Stop, on_delete=models.CASCADE)
location_group_id = models.ForeignKey(LocationGroup, on_delete=models.SET_NULL, blank=True, null=True)
location_id = models.ForeignKey(LocationsGeojson, on_delete=models.SET_NULL, blank=True, null=True)
@@ -238,8 +238,8 @@ class Frequency(models.Model):
"""
frequency_id = models.BigAutoField(primary_key=True)
trip_id = models.ForeignKey(Trip, on_delete=models.CASCADE)
start_time = models.TimeField()
end_time = models.TimeField()
start_time = models.CharField(max_length=255)
end_time = models.CharField(max_length=255)
headway_secs = models.IntegerField()
exact_times = models.IntegerField(blank=True, null=True)
feed_info_id = models.ForeignKey(FeedInfo, on_delete=models.CASCADE)
@@ -286,8 +286,8 @@ class BookingRule(models.Model):
"""
booking_rule_id = models.CharField(max_length=255, primary_key=True)
trip_id = models.ForeignKey(Trip, on_delete=models.CASCADE)
start_time = models.TimeField(blank=True, null=True)
end_time = models.TimeField(blank=True, null=True)
start_time = models.CharField(max_length=255, blank=True, null=True)
end_time = models.CharField(max_length=255, blank=True, null=True)
booking_type = models.CharField(max_length=255)
rule_criteria = models.TextField(blank=True, null=True)
booking_rule_instructions = models.TextField(blank=True, null=True)
@@ -390,11 +390,11 @@ class Timeframe(models.Model):
Represents timeframe.txt from the GTFS Reference.
"""
timeframe_group_id = models.CharField(max_length=255,primary_key=True)
service_id = models.IntegerField()
service_id = models.CharField(max_length=255)
start_date = models.DateField()
end_date = models.DateField()
start_time = models.TimeField()
end_time = models.TimeField()
start_time = models.CharField(max_length=255)
end_time = models.CharField(max_length=255)
feed_info_id = models.ForeignKey(FeedInfo, on_delete=models.CASCADE)
class FareLegRule(models.Model):

23
transport_accessibility/pt_map/tests.py
View File

@@ -1,3 +1,26 @@
from django.test import TestCase
from pt_map.models import Agency, FeedInfo, Level, Stop, Route, Shape, Calendar, CalendarDate, LocationGroup, LocationGroupStop, LocationsGeojson, Trip, StopArea, StopTime, FareAttribute, FareLegRule, FareMedium, FareProduct, FareTransferRule, FareRule, Frequency, Transfer, Pathway, BookingRule, Translation, Attribution, Network, RouteNetwork, Area, Timeframe
from datetime import date, datetime
# Create your tests here.
class FeedInfoTestCase(TestCase):
def setUp(self):
FeedInfo.objects.create(feed_publisher_name="All fields", feed_publisher_url='example.com', feed_lang='pl', default_lang='pl', feed_start_date=datetime(2024, 1, 1), feed_end_date=datetime.now(), feed_version='1.0', feed_contact_email='me@example.com', feed_contact_url='example.com')
def test_ok(self):
FeedInfo.objects.get(feed_publisher_name="All fields")
FeedInfo.objects.get(feed_publisher_url='example.com')
FeedInfo.objects.get(feed_lang='pl')
FeedInfo.objects.get(default_lang='pl')
FeedInfo.objects.get(feed_start_date=datetime(2024, 1, 1))
FeedInfo.objects.get(feed_end_date=datetime.now())
FeedInfo.objects.get(feed_version='1.0')
FeedInfo.objects.get(feed_contact_email='me@example.com')
FeedInfo.objects.get(feed_contact_email='me@example.com')
class AgencyTestCase(TestCase):
def setUp(self):
FeedInfo.objects.create(feed_publisher_name="All fields", feed_publisher_url='example.com', feed_lang='pl', default_lang='pl', feed_start_date=datetime(2024, 1, 1), feed_end_date=datetime.now(), feed_version='1.0', feed_contact_email='me@example.com', feed_contact_url='example.com')
Agency.objects.create(agency_name='test', agency_url='example.com', agency_timezone='Europe/Berlin', agency_lang='pl', agency_phone='0123456574', agency_fare_url='example.com', agency_email='me@example.com', feed_info_id=FeedInfo.objects.get(feed_publisher_name="All fields"))
def test_ok(self):
print(Agency.objects.get(agency_name='test').feed_info_id.feed_id)