From 7be364650e170910b1b84174926f70f755fb2590 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Jul 2022 21:06:08 +0000 Subject: [PATCH 01/38] chore(deps): bump aws-java-sdk-s3 from 1.11.18 to 1.12.261 Bumps [aws-java-sdk-s3](https://github.com/aws/aws-sdk-java) from 1.11.18 to 1.12.261. - [Release notes](https://github.com/aws/aws-sdk-java/releases) - [Changelog](https://github.com/aws/aws-sdk-java/blob/master/CHANGELOG.md) - [Commits](https://github.com/aws/aws-sdk-java/compare/1.11.18...1.12.261) --- updated-dependencies: - dependency-name: com.amazonaws:aws-java-sdk-s3 dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa11a3762..c3d09335d 100644 --- a/pom.xml +++ b/pom.xml @@ -296,7 +296,7 @@ com.amazonaws aws-java-sdk-s3 - 1.11.18 + 1.12.261 From f803a1bd5e57969127bca1e8c4e95f612b718a09 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 8 Sep 2022 01:17:59 +0000 Subject: [PATCH 02/38] chore(deps): bump postgresql from 42.2.25 to 42.4.1 Bumps [postgresql](https://github.com/pgjdbc/pgjdbc) from 42.2.25 to 42.4.1. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.2.25...REL42.4.1) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa11a3762..a94ac9b44 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.postgresql postgresql - 42.2.25 + 42.4.1 org.apache.commons From 84c58a3e9b1e867207e09f4b2d4744cfb341daf2 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 15 Dec 2022 15:51:48 -0500 Subject: [PATCH 03/38] feat(JdbcTableWriter): support stop headsigns --- src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index ef46be59a..aec1926e8 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -635,6 +635,7 @@ private String updateChildTable( "pattern_id", "timepoint", "drop_off_type", + "stop_headsign", "pickup_type", "continuous_pickup", "continuous_drop_off", From 38c86c128ca9d054c21d7424dda9b4b263961ef8 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 16 Dec 2022 16:08:01 -0500 Subject: [PATCH 04/38] refactor(Pattern Classes): support stop headsigns --- src/main/java/com/conveyal/gtfs/loader/Table.java | 1 + src/main/java/com/conveyal/gtfs/model/PatternStop.java | 1 + 2 files changed, 2 insertions(+) diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index ed8e7fcba..705691f06 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -299,6 +299,7 @@ public Table (String name, Class entityClass, Requirement requ // FIXME: Do we need an index on stop_id? new StringField("stop_id", REQUIRED).isReferenceTo(STOPS), // Editor-specific fields + new StringField("stop_headsign", EDITOR), new IntegerField("default_travel_time", EDITOR,0, Integer.MAX_VALUE), new IntegerField("default_dwell_time", EDITOR, 0, Integer.MAX_VALUE), new IntegerField("drop_off_type", EDITOR, 2), diff --git a/src/main/java/com/conveyal/gtfs/model/PatternStop.java b/src/main/java/com/conveyal/gtfs/model/PatternStop.java index 62f343c66..6696e7652 100644 --- a/src/main/java/com/conveyal/gtfs/model/PatternStop.java +++ b/src/main/java/com/conveyal/gtfs/model/PatternStop.java @@ -23,6 +23,7 @@ public class PatternStop extends Entity { public int pickup_type; public int drop_off_type; public int timepoint; + public String stop_headsign; public int continuous_pickup = INT_MISSING; public int continuous_drop_off = INT_MISSING; From 570b023d7f2b85a08a8bcaddde036fdf4bc4c502 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 20 Dec 2022 16:17:37 -0500 Subject: [PATCH 05/38] refactor(GraphQL endpoint): support stop headsigns in pattern stops query --- src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java index acd077d3d..fe5d901ce 100644 --- a/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java +++ b/src/main/java/com/conveyal/gtfs/graphql/GraphQLGtfsSchema.java @@ -435,6 +435,7 @@ public class GraphQLGtfsSchema { .field(MapFetcher.field("shape_dist_traveled", GraphQLFloat)) .field(MapFetcher.field("drop_off_type", GraphQLInt)) .field(MapFetcher.field("pickup_type", GraphQLInt)) + .field(MapFetcher.field("stop_headsign")) .field(MapFetcher.field("continuous_drop_off", GraphQLInt)) .field(MapFetcher.field("continuous_pickup", GraphQLInt)) .field(MapFetcher.field("stop_sequence", GraphQLInt)) From 71bab164783dd5d9250e264474d401a265d78116 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 20 Dec 2022 16:41:40 -0500 Subject: [PATCH 06/38] refactor(Unit Tests): add stop_headsigns into unit tests --- src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java b/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java index f61b68cb5..e089041d4 100644 --- a/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java +++ b/src/test/java/com/conveyal/gtfs/dto/PatternStopDTO.java @@ -9,6 +9,7 @@ public class PatternStopDTO { public Double shape_dist_traveled; public Integer drop_off_type; public Integer pickup_type; + public String stop_headsign; public Integer stop_sequence; public Integer timepoint; public Integer continuous_pickup; From 31725d8167d3ebd9f260ec8b217d661068ba6f81 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 21 Dec 2022 11:24:17 -0500 Subject: [PATCH 07/38] refactor(Unit Tests): adjust PatternFinderValidator to include stop_headsigns --- .../java/com/conveyal/gtfs/TripPatternKey.java | 2 ++ .../gtfs/validator/PatternFinderValidator.java | 17 +++++++++-------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/TripPatternKey.java b/src/main/java/com/conveyal/gtfs/TripPatternKey.java index fd546a69d..f74c3eeea 100644 --- a/src/main/java/com/conveyal/gtfs/TripPatternKey.java +++ b/src/main/java/com/conveyal/gtfs/TripPatternKey.java @@ -26,6 +26,7 @@ public class TripPatternKey { public TIntList arrivalTimes = new TIntArrayList(); public TIntList departureTimes = new TIntArrayList(); public TIntList timepoints = new TIntArrayList(); + public List stopHeadsigns = new ArrayList<>(); public TIntList continuous_pickup = new TIntArrayList(); public TIntList continuous_drop_off = new TIntArrayList(); public TDoubleList shapeDistances = new TDoubleArrayList(); @@ -42,6 +43,7 @@ public void addStopTime (StopTime st) { arrivalTimes.add(st.arrival_time); departureTimes.add(st.departure_time); timepoints.add(st.timepoint); + stopHeadsigns.add(st.stop_headsign); shapeDistances.add(st.shape_dist_traveled); continuous_pickup.add(st.continuous_pickup); continuous_drop_off.add(st.continuous_drop_off); diff --git a/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java b/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java index f24ac1c44..96c7956f2 100644 --- a/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java +++ b/src/main/java/com/conveyal/gtfs/validator/PatternFinderValidator.java @@ -152,14 +152,15 @@ public void complete(ValidationResult validationResult) { // Stop sequence is zero-based. setIntParameter(insertPatternStopStatement, 2, i); insertPatternStopStatement.setString(3, stopId); - setIntParameter(insertPatternStopStatement,4, travelTime); - setIntParameter(insertPatternStopStatement,5, dwellTime); - setIntParameter(insertPatternStopStatement,6, key.dropoffTypes.get(i)); - setIntParameter(insertPatternStopStatement,7, key.pickupTypes.get(i)); - setDoubleParameter(insertPatternStopStatement, 8, key.shapeDistances.get(i)); - setIntParameter(insertPatternStopStatement,9, key.timepoints.get(i)); - setIntParameter(insertPatternStopStatement,10, key.continuous_pickup.get(i)); - setIntParameter(insertPatternStopStatement,11, key.continuous_drop_off.get(i)); + insertPatternStopStatement.setString(4, key.stopHeadsigns.get(i)); + setIntParameter(insertPatternStopStatement,5, travelTime); + setIntParameter(insertPatternStopStatement,6, dwellTime); + setIntParameter(insertPatternStopStatement,7, key.dropoffTypes.get(i)); + setIntParameter(insertPatternStopStatement,8, key.pickupTypes.get(i)); + setDoubleParameter(insertPatternStopStatement, 9, key.shapeDistances.get(i)); + setIntParameter(insertPatternStopStatement,10, key.timepoints.get(i)); + setIntParameter(insertPatternStopStatement,11, key.continuous_pickup.get(i)); + setIntParameter(insertPatternStopStatement,12, key.continuous_drop_off.get(i)); patternStopTracker.addBatch(); } // Finally, update all trips on this pattern to reference this pattern's ID. From d905f96bc8d83f59030a7ef57af7d4834d447de5 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 6 Jan 2023 16:03:13 -0500 Subject: [PATCH 08/38] refactor(Unit Tests): address PR feedback to improve tests and several missed methods --- .../java/com/conveyal/gtfs/loader/EntityPopulator.java | 1 + .../java/com/conveyal/gtfs/loader/JdbcTableWriter.java | 1 + src/test/java/com/conveyal/gtfs/GTFSTest.java | 1 + src/test/resources/fake-agency/stop_times.txt | 8 ++++---- src/test/resources/graphql/feedPatterns.txt | 1 + .../gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json | 4 ++++ .../gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json | 8 ++++---- 7 files changed, 16 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java b/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java index 7105c6c71..36d40a427 100644 --- a/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java +++ b/src/main/java/com/conveyal/gtfs/loader/EntityPopulator.java @@ -60,6 +60,7 @@ public interface EntityPopulator { patternStop.default_travel_time = getIntIfPresent(result, "default_travel_time", columnForName); patternStop.pattern_id = getStringIfPresent(result, "pattern_id", columnForName); patternStop.drop_off_type = getIntIfPresent(result, "drop_off_type", columnForName); + patternStop.stop_headsign = getStringIfPresent(result, "stop_headsign", columnForName); patternStop.pickup_type = getIntIfPresent(result, "pickup_type", columnForName); patternStop.stop_sequence = getIntIfPresent(result, "stop_sequence", columnForName); patternStop.timepoint = getIntIfPresent(result, "timepoint", columnForName); diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index aec1926e8..a565151a7 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -1150,6 +1150,7 @@ private void insertBlankStopTimes( stopTime.stop_id = patternStop.stop_id; stopTime.drop_off_type = patternStop.drop_off_type; stopTime.pickup_type = patternStop.pickup_type; + stopTime.stop_headsign = patternStop.stop_headsign; stopTime.timepoint = patternStop.timepoint; stopTime.shape_dist_traveled = patternStop.shape_dist_traveled; stopTime.continuous_drop_off = patternStop.continuous_drop_off; diff --git a/src/test/java/com/conveyal/gtfs/GTFSTest.java b/src/test/java/com/conveyal/gtfs/GTFSTest.java index 44553ebfb..19b0413d8 100644 --- a/src/test/java/com/conveyal/gtfs/GTFSTest.java +++ b/src/test/java/com/conveyal/gtfs/GTFSTest.java @@ -1291,6 +1291,7 @@ private void assertThatPersistenceExpectationRecordWasFound( new RecordExpectation("stop_id", "4u6g"), new RecordExpectation("stop_sequence", 1), new RecordExpectation("pickup_type", 0), + new RecordExpectation("stop_headsign", "Test stop headsign"), new RecordExpectation("drop_off_type", 0), new RecordExpectation("shape_dist_traveled", 0.0, 0.01) } diff --git a/src/test/resources/fake-agency/stop_times.txt b/src/test/resources/fake-agency/stop_times.txt index 22534abf2..5d8793689 100755 --- a/src/test/resources/fake-agency/stop_times.txt +++ b/src/test/resources/fake-agency/stop_times.txt @@ -1,5 +1,5 @@ trip_id,arrival_time,departure_time,stop_id,stop_sequence,stop_headsign,pickup_type,drop_off_type,shape_dist_traveled,timepoint -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,,0,0,0.0000000, -a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,,0,0,341.4491961, -frequency-trip,08:00:00,08:00:00,4u6g,1,,0,0,0.0000000, -frequency-trip,08:29:00,08:29:00,1234,2,,0,0,341.4491961, +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:00:00,07:00:00,4u6g,1,Test stop headsign,0,0,0.0000000, +a30277f8-e50a-4a85-9141-b1e0da9d429d,07:01:00,07:01:00,johv,2,Test stop headsign 2,0,0,341.4491961, +frequency-trip,08:00:00,08:00:00,4u6g,1,Test stop headsign frequency trip,0,0,0.0000000, +frequency-trip,08:29:00,08:29:00,1234,2,Test stop headsign frequency trip 2,0,0,341.4491961, diff --git a/src/test/resources/graphql/feedPatterns.txt b/src/test/resources/graphql/feedPatterns.txt index a1afb856e..a144092ed 100644 --- a/src/test/resources/graphql/feedPatterns.txt +++ b/src/test/resources/graphql/feedPatterns.txt @@ -13,6 +13,7 @@ query ($namespace: String) { id pattern_id pickup_type + stop_headsign shape_dist_traveled stop { stop_id diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json index 16501ea29..3732092fe 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchPatterns-0.json @@ -14,6 +14,7 @@ "id" : 1, "pattern_id" : "1", "pickup_type" : 0, + "stop_headsign" : "Test stop headsign", "shape_dist_traveled" : 0.0, "stop" : [ { "stop_id" : "4u6g" @@ -28,6 +29,7 @@ "id" : 2, "pattern_id" : "1", "pickup_type" : 0, + "stop_headsign" : "Test stop headsign 2", "shape_dist_traveled" : 341.4491961, "stop" : [ { "stop_id" : "johv" @@ -113,6 +115,7 @@ "id" : 3, "pattern_id" : "2", "pickup_type" : 0, + "stop_headsign" : "Test stop headsign frequency trip", "shape_dist_traveled" : 0.0, "stop" : [ { "stop_id" : "4u6g" @@ -127,6 +130,7 @@ "id" : 4, "pattern_id" : "2", "pickup_type" : 0, + "stop_headsign" : "Test stop headsign frequency trip 2", "shape_dist_traveled" : 341.4491961, "stop" : [ { "stop_id" : "1234" diff --git a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json index cfb8c37d2..370176de9 100644 --- a/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json +++ b/src/test/resources/snapshots/com/conveyal/gtfs/graphql/GTFSGraphQLTest/canFetchStopTimes-0.json @@ -8,7 +8,7 @@ "drop_off_type" : 0, "pickup_type" : 0, "shape_dist_traveled" : 0.0, - "stop_headsign" : null, + "stop_headsign" : "Test stop headsign", "stop_id" : "4u6g", "stop_sequence" : 1, "timepoint" : null, @@ -19,7 +19,7 @@ "drop_off_type" : 0, "pickup_type" : 0, "shape_dist_traveled" : 341.4491961, - "stop_headsign" : null, + "stop_headsign" : "Test stop headsign 2", "stop_id" : "johv", "stop_sequence" : 2, "timepoint" : null, @@ -30,7 +30,7 @@ "drop_off_type" : 0, "pickup_type" : 0, "shape_dist_traveled" : 0.0, - "stop_headsign" : null, + "stop_headsign" : "Test stop headsign frequency trip", "stop_id" : "4u6g", "stop_sequence" : 1, "timepoint" : null, @@ -41,7 +41,7 @@ "drop_off_type" : 0, "pickup_type" : 0, "shape_dist_traveled" : 341.4491961, - "stop_headsign" : null, + "stop_headsign" : "Test stop headsign frequency trip", "stop_id" : "1234", "stop_sequence" : 2, "timepoint" : null, From d768544b2a8f513c0a483c2279bfa06b4675b195 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 6 Jan 2023 16:28:11 -0500 Subject: [PATCH 09/38] refactor(maven workflow): try upgrading node --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9cdc9e195..340f44fe7 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -30,10 +30,10 @@ jobs: with: java-version: 1.8 # Install node 14+ for running maven-semantic-release. - - name: Use Node.js 14.x + - name: Use Node.js 16.X uses: actions/setup-node@v1 with: - node-version: 14.x + node-version: 16 - name: Install maven-semantic-release # FIXME: Enable cache for node packages (add package.json?) run: | From 23a51095c0a86af9c244d4f8684ac8757e59a1a0 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 6 Jan 2023 16:30:35 -0500 Subject: [PATCH 10/38] refactor(maven workflow): try upgrading node again (to 18) --- .github/workflows/maven.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 340f44fe7..41f0ffaa4 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -30,10 +30,10 @@ jobs: with: java-version: 1.8 # Install node 14+ for running maven-semantic-release. - - name: Use Node.js 16.X + - name: Use Node.js 18.X uses: actions/setup-node@v1 with: - node-version: 16 + node-version: 18 - name: Install maven-semantic-release # FIXME: Enable cache for node packages (add package.json?) run: | From 9b192dba574bd223635910b732cc7864ef4c5c07 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 9 Jan 2023 09:09:58 -0500 Subject: [PATCH 11/38] refactor(maven workflow): test node version --- .github/workflows/maven.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 41f0ffaa4..2a7ed526c 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -37,6 +37,7 @@ jobs: - name: Install maven-semantic-release # FIXME: Enable cache for node packages (add package.json?) run: | + node --version yarn global add @conveyal/maven-semantic-release semantic-release # Add yarn path to GITHUB_PATH so that global package is executable. echo "$(yarn global bin)" >> $GITHUB_PATH @@ -66,4 +67,5 @@ jobs: env: GH_TOKEN: ${{ secrets.GH_TOKEN }} run: | - semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy + node --version + semantic-release --prepare @conveyal/maven-semantic-release --publish @semantic-release/github,@conveyal/maven-semantic-release --verify-conditions @semantic-release/github,@conveyal/maven-semantic-release --verify-release @conveyal/maven-semantic-release --use-conveyal-workflow --dev-branch=dev --skip-maven-deploy From 640be9da724c0520a2893b1a93f37154049ba3d1 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 9 Jan 2023 09:13:56 -0500 Subject: [PATCH 12/38] refactor(maven workflow): setup node immediately before run semantic release --- .github/workflows/maven.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 2a7ed526c..16e39476e 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -63,6 +63,11 @@ jobs: # See https://github.com/conveyal/gtfs-lib/issues/193. # # The git commands get the commit hash of the HEAD commit and the commit just before HEAD. + # Install node 14+ for running maven-semantic-release. + - name: Use Node.js 18.X + uses: actions/setup-node@v1 + with: + node-version: 18 - name: Run maven-semantic-release env: GH_TOKEN: ${{ secrets.GH_TOKEN }} From 6d5b3e923bfff16bd7458b39f321d12109297b13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 25 Jan 2023 16:21:01 +0000 Subject: [PATCH 13/38] chore(deps): bump commons-text from 1.6 to 1.10.0 Bumps commons-text from 1.6 to 1.10.0. --- updated-dependencies: - dependency-name: org.apache.commons:commons-text dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index fa11a3762..86c2ceb74 100644 --- a/pom.xml +++ b/pom.xml @@ -337,7 +337,7 @@ org.apache.commons commons-text - 1.6 + 1.10.0 From b557f59417b97b62482e1b5bfa2e62d31ad6803f Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 17 Apr 2023 21:54:22 -0400 Subject: [PATCH 14/38] feat(table writer): add deletion of cds for exception --- .../conveyal/gtfs/loader/JdbcTableWriter.java | 72 +++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index a565151a7..16a74c41d 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -18,6 +18,8 @@ import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; import org.apache.commons.dbutils.DbUtils; +import org.apache.commons.lang3.ArrayUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1427,6 +1429,11 @@ private static Set getReferencingTables(Table table) { // IMPORTANT: Skip the table for the entity we're modifying or if loop table does not have field. if (table.name.equals(gtfsTable.name)) continue; for (Field field : gtfsTable.fields) { + if (table.name.equals("schedule_exceptions") && gtfsTable.name.equals("calendar_dates")) { + // HACK: schedule exceptions is really a reference to calendars and calendar_dates. We need to update both. + // However, schedule_exceptions does not require a reference to calendar_dates right now. + referencingTables.add(gtfsTable); + } if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { // If any of the table's fields are foreign references to the specified table, add to the return set. referencingTables.add(gtfsTable); @@ -1497,6 +1504,71 @@ private void updateReferencingTables( for (Table referencingTable : referencingTables) { // Update/delete foreign references that have match the key value. String refTableName = String.join(".", namespace, referencingTable.name); + // Custom logic for calendar dates because it's not captured right now + if (table.name.equals("schedule_exceptions") && referencingTable.name.equals("calendar_dates")){ + // The keys are all calendars in the added and removed service columns + // TODO: generalise this + String addedService = getValueForId(id, "added_service", namespace, table, connection); + String[] addedServiceList = StringUtils.strip(addedService, "{}").split("[,]", 0); + String removedService = getValueForId(id, "removed_service", namespace, table, connection); + String[] removedServiceList = StringUtils.strip(removedService, "{}").split("[,]", 0); + String dates = getValueForId(id, "dates", namespace, table, connection); + dates = StringUtils.strip(dates, "{}"); + + String[] allServiceToDelete = ArrayUtils.addAll(addedServiceList, removedServiceList); + + // Schedule exceptions map to many different calendar dates (one for each date / service ID combination) + for (String date : dates.split("[,]", 0)) { + for (String serviceId : allServiceToDelete) { + LOG.info("service", serviceId); + // TODO: better way to do this with existing method? + // Prepare the SQL statement: + String sql; + PreparedStatement updateStatement; + boolean isArrayField = keyField.getSqlType().equals(JDBCType.ARRAY); + switch (sqlMethod) { + case DELETE: + sql = String.format( + "delete from %s where service_id = ? and date = ?", + refTableName + ); + + updateStatement = connection.prepareStatement(sql); + updateStatement.setString(1, serviceId); + updateStatement.setString(2, date); + LOG.info(updateStatement.toString()); + int result = updateStatement.executeUpdate(); + + if (result > 0) { + // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override + // deleteEntityHook). + if (sqlMethod.equals(SqlMethod.DELETE)) { + // Check for restrictions on delete. + if (table.isCascadeDeleteRestricted()) { + // The entity must not have any referencing entities in order to delete it. + connection.rollback(); + String message = String.format( + "Cannot delete %s %s=%s. %d %s reference this %s.", + entityClass.getSimpleName(), + keyField.name, + keyValue, + result, + referencingTable.name, + entityClass.getSimpleName() + ); + LOG.warn(message); + throw new SQLException(message); + } + } + LOG.info("{} reference(s) in {} {}D!", result, refTableName, sqlMethod); + } else { + LOG.info("No references in {} found!", refTableName); + } + } + } + } + } + for (Field field : referencingTable.editorFields()) { if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { // Get statement to update or delete entities that reference the key value. From 9abc2827f19ddcc9c9bede104f409c721fbf170a Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 19 Apr 2023 22:38:15 -0400 Subject: [PATCH 15/38] refactor(table writer): clean up calendar dates deletion --- .../conveyal/gtfs/loader/JdbcTableWriter.java | 172 ++++++++---------- 1 file changed, 80 insertions(+), 92 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index 16a74c41d..c925f7d8e 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -1432,6 +1432,9 @@ private static Set
getReferencingTables(Table table) { if (table.name.equals("schedule_exceptions") && gtfsTable.name.equals("calendar_dates")) { // HACK: schedule exceptions is really a reference to calendars and calendar_dates. We need to update both. // However, schedule_exceptions does not require a reference to calendar_dates right now. + // This is OK for an update event because there is code that avoids updating referenceTables + // with a null parentTable. However, this makes calendar_dates a referenceTable for all situations, + // which could have unexpected behaviour. referencingTables.add(gtfsTable); } if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { @@ -1459,6 +1462,49 @@ private static String getValueForId(int id, String fieldName, String namespace, return value; } + /** + * Parse a list field in the schedule exceptions table (dates, added_service, or removed_service) into a list of string. + * List fields take the form {ServiceId1, ServiceId2, ServiceId3, ...} + */ + private String[] parseExceptionListField(int id, String namespace, Table table, String type) throws SQLException { + String parsedString; + if (type.equals("service")) { + String addedServiceIds = getValueForId(id, "added_service", namespace, table, connection); + String removedServiceIds = getValueForId(id, "removed_service", namespace, table, connection); + parsedString = addedServiceIds + removedServiceIds; + } else { + parsedString = getValueForId(id, type, namespace, table, connection); + } + return StringUtils.strip(parsedString, "{}").split("[,]", 0); + } + + /** + * Delete all entries in calendar dates associated with a schedule exception. + */ + private Integer deleteCalendarDatesForException( + int id, + String namespace, + Table table + ) throws SQLException { + String[] serviceIds = parseExceptionListField(id, namespace, table, "service"); + String[] dates = parseExceptionListField(id, namespace, table, "dates"); + int resultCount = 0; + + // Schedule exceptions map to many different calendar dates (one for each date / service ID combination) + for (String date : dates) { + for (String serviceId : serviceIds) { + String sql =String.format("delete from calendar_dates where service_id = ? and date = ?"); + PreparedStatement updateStatement = connection.prepareStatement(sql); + updateStatement.setString(1, serviceId); + updateStatement.setString(2, date); + LOG.info(updateStatement.toString()); + resultCount += updateStatement.executeUpdate(); + } + } + + return resultCount; + } + /** * Updates any foreign references that exist should a GTFS key field (e.g., stop_id or route_id) be updated via an * HTTP request for a given integer ID. First, all GTFS tables are filtered to find referencing tables. Then records @@ -1504,101 +1550,43 @@ private void updateReferencingTables( for (Table referencingTable : referencingTables) { // Update/delete foreign references that have match the key value. String refTableName = String.join(".", namespace, referencingTable.name); - // Custom logic for calendar dates because it's not captured right now - if (table.name.equals("schedule_exceptions") && referencingTable.name.equals("calendar_dates")){ - // The keys are all calendars in the added and removed service columns - // TODO: generalise this - String addedService = getValueForId(id, "added_service", namespace, table, connection); - String[] addedServiceList = StringUtils.strip(addedService, "{}").split("[,]", 0); - String removedService = getValueForId(id, "removed_service", namespace, table, connection); - String[] removedServiceList = StringUtils.strip(removedService, "{}").split("[,]", 0); - String dates = getValueForId(id, "dates", namespace, table, connection); - dates = StringUtils.strip(dates, "{}"); - - String[] allServiceToDelete = ArrayUtils.addAll(addedServiceList, removedServiceList); - - // Schedule exceptions map to many different calendar dates (one for each date / service ID combination) - for (String date : dates.split("[,]", 0)) { - for (String serviceId : allServiceToDelete) { - LOG.info("service", serviceId); - // TODO: better way to do this with existing method? - // Prepare the SQL statement: - String sql; - PreparedStatement updateStatement; - boolean isArrayField = keyField.getSqlType().equals(JDBCType.ARRAY); - switch (sqlMethod) { - case DELETE: - sql = String.format( - "delete from %s where service_id = ? and date = ?", - refTableName - ); - - updateStatement = connection.prepareStatement(sql); - updateStatement.setString(1, serviceId); - updateStatement.setString(2, date); - LOG.info(updateStatement.toString()); - int result = updateStatement.executeUpdate(); - - if (result > 0) { - // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override - // deleteEntityHook). - if (sqlMethod.equals(SqlMethod.DELETE)) { - // Check for restrictions on delete. - if (table.isCascadeDeleteRestricted()) { - // The entity must not have any referencing entities in order to delete it. - connection.rollback(); - String message = String.format( - "Cannot delete %s %s=%s. %d %s reference this %s.", - entityClass.getSimpleName(), - keyField.name, - keyValue, - result, - referencingTable.name, - entityClass.getSimpleName() - ); - LOG.warn(message); - throw new SQLException(message); - } - } - LOG.info("{} reference(s) in {} {}D!", result, refTableName, sqlMethod); - } else { - LOG.info("No references in {} found!", refTableName); + int result; + // Custom logic for calendar dates because schedule_exceptions does not reference calendar dates right now. + if (table.name.equals("schedule_exceptions") && referencingTable.name.equals("calendar_dates")) { + result = deleteCalendarDatesForException(id, namespace, table); + LOG.info("Deleted {} entries in calendar dates associated with schedule exception {}", result, id); + } else { // General deletion + for (Field field : referencingTable.editorFields()) { + if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { + // Get statement to update or delete entities that reference the key value. + PreparedStatement updateStatement = getUpdateReferencesStatement(sqlMethod, refTableName, field, keyValue, newKeyValue); + LOG.info(updateStatement.toString()); + result = updateStatement.executeUpdate(); + if (result > 0) { + // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override + // deleteEntityHook). + if (sqlMethod.equals(SqlMethod.DELETE)) { + // Check for restrictions on delete. + if (table.isCascadeDeleteRestricted()) { + // The entity must not have any referencing entities in order to delete it. + connection.rollback(); + String message = String.format( + "Cannot delete %s %s=%s. %d %s reference this %s.", + entityClass.getSimpleName(), + keyField.name, + keyValue, + result, + referencingTable.name, + entityClass.getSimpleName() + ); + LOG.warn(message); + throw new SQLException(message); } - } - } - } - } - - for (Field field : referencingTable.editorFields()) { - if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { - // Get statement to update or delete entities that reference the key value. - PreparedStatement updateStatement = getUpdateReferencesStatement(sqlMethod, refTableName, field, keyValue, newKeyValue); - LOG.info(updateStatement.toString()); - int result = updateStatement.executeUpdate(); - if (result > 0) { - // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override - // deleteEntityHook). - if (sqlMethod.equals(SqlMethod.DELETE)) { - // Check for restrictions on delete. - if (table.isCascadeDeleteRestricted()) { - // The entity must not have any referencing entities in order to delete it. - connection.rollback(); - String message = String.format( - "Cannot delete %s %s=%s. %d %s reference this %s.", - entityClass.getSimpleName(), - keyField.name, - keyValue, - result, - referencingTable.name, - entityClass.getSimpleName() - ); - LOG.warn(message); - throw new SQLException(message); } + LOG.info("{} reference(s) in {} {}D!", result, refTableName, sqlMethod); + } else { + LOG.info("No references in {} found!", refTableName); } - LOG.info("{} reference(s) in {} {}D!", result, refTableName, sqlMethod); - } else { - LOG.info("No references in {} found!", refTableName); } } } From 4807999cbde9bbd4396a8130770f716864a4455f Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 20 Apr 2023 15:38:52 -0400 Subject: [PATCH 16/38] refactor(table writer): clean up calendar dates deletion and add unit tests --- .../conveyal/gtfs/loader/JdbcTableWriter.java | 15 ++-- .../conveyal/gtfs/dto/CalendarDateDTO.java | 8 ++ .../gtfs/loader/JDBCTableWriterTest.java | 81 +++++++++++++++++++ 3 files changed, 98 insertions(+), 6 deletions(-) create mode 100644 src/test/java/com/conveyal/gtfs/dto/CalendarDateDTO.java diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index c925f7d8e..aa7238efc 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -1314,6 +1314,9 @@ private void ensureReferentialIntegrity( Integer id ) throws SQLException { final boolean isCreating = id == null; + // HACK: when ensuring referential integrity for a table like calendar_dates, our key is multiple columns. + // TODO: adapt this method to check a key of several columns (e.g. service_id, date combination) + if (!table.hasUniqueKeyField) return; String keyField = table.getKeyFieldName(); String tableName = String.join(".", namespace, table.name); JsonNode val = jsonObject.get(keyField); @@ -1471,11 +1474,11 @@ private String[] parseExceptionListField(int id, String namespace, Table table, if (type.equals("service")) { String addedServiceIds = getValueForId(id, "added_service", namespace, table, connection); String removedServiceIds = getValueForId(id, "removed_service", namespace, table, connection); - parsedString = addedServiceIds + removedServiceIds; + parsedString = addedServiceIds + "," + removedServiceIds; } else { parsedString = getValueForId(id, type, namespace, table, connection); } - return StringUtils.strip(parsedString, "{}").split("[,]", 0); + return parsedString.replaceAll("[{}]", "").split("[,]", 0); } /** @@ -1484,7 +1487,8 @@ private String[] parseExceptionListField(int id, String namespace, Table table, private Integer deleteCalendarDatesForException( int id, String namespace, - Table table + Table table, + String refTableName ) throws SQLException { String[] serviceIds = parseExceptionListField(id, namespace, table, "service"); String[] dates = parseExceptionListField(id, namespace, table, "dates"); @@ -1493,7 +1497,7 @@ private Integer deleteCalendarDatesForException( // Schedule exceptions map to many different calendar dates (one for each date / service ID combination) for (String date : dates) { for (String serviceId : serviceIds) { - String sql =String.format("delete from calendar_dates where service_id = ? and date = ?"); + String sql =String.format("delete from %s where service_id = ? and date = ?", refTableName); PreparedStatement updateStatement = connection.prepareStatement(sql); updateStatement.setString(1, serviceId); updateStatement.setString(2, date); @@ -1501,7 +1505,6 @@ private Integer deleteCalendarDatesForException( resultCount += updateStatement.executeUpdate(); } } - return resultCount; } @@ -1553,7 +1556,7 @@ private void updateReferencingTables( int result; // Custom logic for calendar dates because schedule_exceptions does not reference calendar dates right now. if (table.name.equals("schedule_exceptions") && referencingTable.name.equals("calendar_dates")) { - result = deleteCalendarDatesForException(id, namespace, table); + result = deleteCalendarDatesForException(id, namespace, table, refTableName); LOG.info("Deleted {} entries in calendar dates associated with schedule exception {}", result, id); } else { // General deletion for (Field field : referencingTable.editorFields()) { diff --git a/src/test/java/com/conveyal/gtfs/dto/CalendarDateDTO.java b/src/test/java/com/conveyal/gtfs/dto/CalendarDateDTO.java new file mode 100644 index 000000000..254a6e00a --- /dev/null +++ b/src/test/java/com/conveyal/gtfs/dto/CalendarDateDTO.java @@ -0,0 +1,8 @@ +package com.conveyal.gtfs.dto; + +public class CalendarDateDTO { + public Integer id; + public String service_id; + public String date; + public Integer exception_type; +} diff --git a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java index a340c39bf..d91fb4743 100644 --- a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java +++ b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java @@ -2,6 +2,7 @@ import com.conveyal.gtfs.TestUtils; import com.conveyal.gtfs.dto.CalendarDTO; +import com.conveyal.gtfs.dto.CalendarDateDTO; import com.conveyal.gtfs.dto.FareDTO; import com.conveyal.gtfs.dto.FareRuleDTO; import com.conveyal.gtfs.dto.FeedInfoDTO; @@ -421,6 +422,60 @@ public void canCreateUpdateAndDeleteScheduleExceptions() throws IOException, SQL assertThatSqlQueryYieldsZeroRows(getColumnsForId(scheduleException.id, scheduleExceptionTable)); } + /** + * Ensure that {@link ScheduleException}s which are loaded from an existing GTFS can be removed properly, + * including created entries in calendar_dates. + */ + @Test + public void canCreateAndDeleteCalendarDates() throws IOException, SQLException, InvalidNamespaceException { + String firstServiceId = "REMOVED"; + String secondServiceId = "ADDED"; + String[] allServiceIds = new String[]{firstServiceId, secondServiceId}; + String[] holidayDates = new String[]{"20190812", "20190813", "20190814"}; + + final Table scheduleExceptionTable = Table.SCHEDULE_EXCEPTIONS; + final Table calendarDatesTable = Table.CALENDAR_DATES; + final Class scheduleExceptionDTOClass = ScheduleExceptionDTO.class; + + // Create new schedule exception which involves 2 service IDs and multiple dates + ScheduleExceptionDTO exceptionInput = new ScheduleExceptionDTO(); + exceptionInput.name = "Incredible multi day holiday"; + exceptionInput.exemplar = 9; // Add, swap, or remove type + exceptionInput.removed_service = new String[]{firstServiceId}; + exceptionInput.added_service = new String[]{secondServiceId}; + exceptionInput.dates = holidayDates; + + // Save the schedule exception + // TODO: share this with other schedule exception method? + TableWriter createTableWriter = createTestTableWriter(scheduleExceptionTable); + String scheduleExceptionOutput = createTableWriter.create(mapper.writeValueAsString(exceptionInput), true); + ScheduleExceptionDTO scheduleException = mapper.readValue(scheduleExceptionOutput, scheduleExceptionDTOClass); + + // Create a calendar_dates entry for each date of the schedule exception + for (String date: holidayDates) { + createAndStoreCalendarDate(firstServiceId, date, 2); // firstServiceId is removed + createAndStoreCalendarDate(secondServiceId, date, 1); // secondServiceId is added + } + + // Delete a schedule exception + JdbcTableWriter deleteTableWriter = createTestTableWriter(scheduleExceptionTable); + int deleteOutput = deleteTableWriter.delete(scheduleException.id, true); + LOG.info("deleted {} records from {}", deleteOutput, scheduleExceptionTable.name); + + // Verify that the entries in calendar_dates are removed after deleting the schedule exception. + for (String date : holidayDates) { + for (String serviceId : allServiceIds){ + String sql = String.format("select * from %s.%s where service_id = '%s' and date = '%s'", + testNamespace, + calendarDatesTable.name, + serviceId, + date + ); + assertThatSqlQueryYieldsZeroRows(sql); + } + } + } + /** * This test verifies that stop_times#shape_dist_traveled and other linked fields are updated when a pattern * is updated. @@ -1065,4 +1120,30 @@ private static CalendarDTO createWeekdayCalendar(String serviceId, String startD LOG.info(output); return mapper.readValue(output, CalendarDTO.class); } + + /** + * Create and store a simple calendar date that modifies service on one day. + */ + private static CalendarDateDTO createAndStoreCalendarDate(String serviceId, String date, int exceptionType) throws IOException, SQLException, InvalidNamespaceException { + JdbcTableWriter createCalendarDateWriter = new JdbcTableWriter(Table.CALENDAR_DATES, testDataSource, testNamespace); + + CalendarDateDTO calendarDate = createCalendarDate(serviceId, date, exceptionType); + String output = createCalendarDateWriter.create(mapper.writeValueAsString(calendarDate), true); + + LOG.info("create {} output:", Table.CALENDAR_DATES.name); + LOG.info(output); + + return mapper.readValue(output, CalendarDateDTO.class); + } + + /** + * Create a calendar date. + */ + private static CalendarDateDTO createCalendarDate(String serviceId, String date, Integer exceptionType) { + CalendarDateDTO calenderDate = new CalendarDateDTO(); + calenderDate.date = date; + calenderDate.service_id = serviceId; + calenderDate.exception_type = exceptionType; + return calenderDate; + } } From f035d18dc1721f03bfea4a1d5435184ab22f01e5 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 21 Apr 2023 09:45:26 -0400 Subject: [PATCH 17/38] refactor(table writer): respond to PR comments. --- .../java/com/conveyal/gtfs/loader/JdbcTableWriter.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index aa7238efc..85e0601dd 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -18,8 +18,6 @@ import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; import org.apache.commons.dbutils.DbUtils; -import org.apache.commons.lang3.ArrayUtils; -import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1466,7 +1464,7 @@ private static String getValueForId(int id, String fieldName, String namespace, } /** - * Parse a list field in the schedule exceptions table (dates, added_service, or removed_service) into a list of string. + * Parse a list field in the schedule exceptions table (dates, added_service, or removed_service) into an array of strings. * List fields take the form {ServiceId1, ServiceId2, ServiceId3, ...} */ private String[] parseExceptionListField(int id, String namespace, Table table, String type) throws SQLException { @@ -1554,11 +1552,12 @@ private void updateReferencingTables( // Update/delete foreign references that have match the key value. String refTableName = String.join(".", namespace, referencingTable.name); int result; - // Custom logic for calendar dates because schedule_exceptions does not reference calendar dates right now. if (table.name.equals("schedule_exceptions") && referencingTable.name.equals("calendar_dates")) { + // Custom logic for calendar dates because schedule_exceptions does not reference calendar dates right now. result = deleteCalendarDatesForException(id, namespace, table, refTableName); LOG.info("Deleted {} entries in calendar dates associated with schedule exception {}", result, id); - } else { // General deletion + } else { + // General deletion for (Field field : referencingTable.editorFields()) { if (field.isForeignReference() && field.referenceTable.name.equals(table.name)) { // Get statement to update or delete entities that reference the key value. From 577ae945553f3c5f6f2a480ae4a0588b5c40e116 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 24 Apr 2023 11:06:20 -0400 Subject: [PATCH 18/38] refactor(table writer): respond to PR comments r2. --- .../java/com/conveyal/gtfs/loader/JdbcTableWriter.java | 4 ++-- .../com/conveyal/gtfs/loader/JDBCTableWriterTest.java | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index 85e0601dd..2d3c9acab 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -1491,12 +1491,12 @@ private Integer deleteCalendarDatesForException( String[] serviceIds = parseExceptionListField(id, namespace, table, "service"); String[] dates = parseExceptionListField(id, namespace, table, "dates"); int resultCount = 0; + String sql = String.format("delete from %s where service_id = ? and date = ?", refTableName); + PreparedStatement updateStatement = connection.prepareStatement(sql); // Schedule exceptions map to many different calendar dates (one for each date / service ID combination) for (String date : dates) { for (String serviceId : serviceIds) { - String sql =String.format("delete from %s where service_id = ? and date = ?", refTableName); - PreparedStatement updateStatement = connection.prepareStatement(sql); updateStatement.setString(1, serviceId); updateStatement.setString(2, date); LOG.info(updateStatement.toString()); diff --git a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java index d91fb4743..62f2a8d57 100644 --- a/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java +++ b/src/test/java/com/conveyal/gtfs/loader/JDBCTableWriterTest.java @@ -430,8 +430,8 @@ public void canCreateUpdateAndDeleteScheduleExceptions() throws IOException, SQL public void canCreateAndDeleteCalendarDates() throws IOException, SQLException, InvalidNamespaceException { String firstServiceId = "REMOVED"; String secondServiceId = "ADDED"; - String[] allServiceIds = new String[]{firstServiceId, secondServiceId}; - String[] holidayDates = new String[]{"20190812", "20190813", "20190814"}; + String[] allServiceIds = new String[] {firstServiceId, secondServiceId}; + String[] holidayDates = new String[] {"20190812", "20190813", "20190814"}; final Table scheduleExceptionTable = Table.SCHEDULE_EXCEPTIONS; final Table calendarDatesTable = Table.CALENDAR_DATES; @@ -441,8 +441,8 @@ public void canCreateAndDeleteCalendarDates() throws IOException, SQLException, ScheduleExceptionDTO exceptionInput = new ScheduleExceptionDTO(); exceptionInput.name = "Incredible multi day holiday"; exceptionInput.exemplar = 9; // Add, swap, or remove type - exceptionInput.removed_service = new String[]{firstServiceId}; - exceptionInput.added_service = new String[]{secondServiceId}; + exceptionInput.removed_service = new String[] {firstServiceId}; + exceptionInput.added_service = new String[] {secondServiceId}; exceptionInput.dates = holidayDates; // Save the schedule exception @@ -1124,7 +1124,7 @@ private static CalendarDTO createWeekdayCalendar(String serviceId, String startD /** * Create and store a simple calendar date that modifies service on one day. */ - private static CalendarDateDTO createAndStoreCalendarDate(String serviceId, String date, int exceptionType) throws IOException, SQLException, InvalidNamespaceException { + private static CalendarDateDTO createAndStoreCalendarDate(String serviceId, String date, int exceptionType) throws IOException, SQLException, InvalidNamespaceException { JdbcTableWriter createCalendarDateWriter = new JdbcTableWriter(Table.CALENDAR_DATES, testDataSource, testNamespace); CalendarDateDTO calendarDate = createCalendarDate(serviceId, date, exceptionType); From 4de8567173065fa2eabb08ce267673483ce70da0 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 27 Apr 2023 17:18:42 -0400 Subject: [PATCH 19/38] feat(table writer): return pattern and route information for stop deletion --- .../conveyal/gtfs/loader/JdbcTableWriter.java | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index 2d3c9acab..b759a96c8 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -18,6 +18,7 @@ import gnu.trove.set.TIntSet; import gnu.trove.set.hash.TIntHashSet; import org.apache.commons.dbutils.DbUtils; +import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -1568,10 +1569,41 @@ private void updateReferencingTables( // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override // deleteEntityHook). if (sqlMethod.equals(SqlMethod.DELETE)) { + ArrayList patternAndRouteIds = new ArrayList(); // Check for restrictions on delete. if (table.isCascadeDeleteRestricted()) { // The entity must not have any referencing entities in order to delete it. connection.rollback(); + // Check if the class is a stop + if (entityClass.getSimpleName().equals("Stop")) { + // if it is a stop, then we can initiate some function that looks up the pattern stops associated with the stop + String patternStopLookup = String.format( + "select distinct ps.pattern_id, r.id " + + "from %s.pattern_stops ps " + + "inner join " + + "%s.patterns p " + + "on p.pattern_id = ps.pattern_id " + + "inner join " + + "%s.routes r " + + "on p.route_id = r.route_id " + + "where %s = '%s'", + namespace, + namespace, + namespace, + keyField.name, + keyValue + ); + PreparedStatement patternStopSelectStatement = connection.prepareStatement(patternStopLookup); + if (patternStopSelectStatement.execute()) { + ResultSet resultSet = patternStopSelectStatement.getResultSet(); + while (resultSet.next()) { + patternAndRouteIds.add( + "{" + resultSet.getString(1) + "-" + resultSet.getString(2) + "}" + ); + } + } + } + // once we know the pattern stops associated, then we can add some links to the error message.. String message = String.format( "Cannot delete %s %s=%s. %d %s reference this %s.", entityClass.getSimpleName(), @@ -1581,6 +1613,10 @@ private void updateReferencingTables( referencingTable.name, entityClass.getSimpleName() ); + if (patternAndRouteIds.size() > 0) { + String patternIdsString = StringUtils.join(patternAndRouteIds, ","); + message += "\n" + "Referenced pattern IDs: [" + patternIdsString + "]"; + } LOG.warn(message); throw new SQLException(message); } From 30754ba741ee414fc1778837160d91e12694eddf Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 28 Apr 2023 11:23:20 -0400 Subject: [PATCH 20/38] refactor(table writer): respond to PR comments --- .../conveyal/gtfs/loader/JdbcTableWriter.java | 43 ++++++++++--------- 1 file changed, 22 insertions(+), 21 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index b759a96c8..a860e36e5 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -1569,41 +1569,38 @@ private void updateReferencingTables( // FIXME: is this where a delete hook should go? (E.g., CalendarController subclass would override // deleteEntityHook). if (sqlMethod.equals(SqlMethod.DELETE)) { - ArrayList patternAndRouteIds = new ArrayList(); + ArrayList patternAndRouteIds = new ArrayList<>(); // Check for restrictions on delete. if (table.isCascadeDeleteRestricted()) { // The entity must not have any referencing entities in order to delete it. connection.rollback(); - // Check if the class is a stop if (entityClass.getSimpleName().equals("Stop")) { - // if it is a stop, then we can initiate some function that looks up the pattern stops associated with the stop String patternStopLookup = String.format( - "select distinct ps.pattern_id, r.id " + - "from %s.pattern_stops ps " + - "inner join " + - "%s.patterns p " + - "on p.pattern_id = ps.pattern_id " + - "inner join " + - "%s.routes r " + - "on p.route_id = r.route_id " + - "where %s = '%s'", - namespace, - namespace, - namespace, - keyField.name, - keyValue + "select distinct ps.pattern_id, r.id " + + "from %s.pattern_stops ps " + + "inner join " + + "%s.patterns p " + + "on p.pattern_id = ps.pattern_id " + + "inner join " + + "%s.routes r " + + "on p.route_id = r.route_id " + + "where %s = '%s'", + namespace, + namespace, + namespace, + keyField.name, + keyValue ); PreparedStatement patternStopSelectStatement = connection.prepareStatement(patternStopLookup); if (patternStopSelectStatement.execute()) { ResultSet resultSet = patternStopSelectStatement.getResultSet(); while (resultSet.next()) { patternAndRouteIds.add( - "{" + resultSet.getString(1) + "-" + resultSet.getString(2) + "}" + "{" + resultSet.getString(1) + "-" + resultSet.getString(2) + "}" ); } } } - // once we know the pattern stops associated, then we can add some links to the error message.. String message = String.format( "Cannot delete %s %s=%s. %d %s reference this %s.", entityClass.getSimpleName(), @@ -1614,8 +1611,12 @@ private void updateReferencingTables( entityClass.getSimpleName() ); if (patternAndRouteIds.size() > 0) { - String patternIdsString = StringUtils.join(patternAndRouteIds, ","); - message += "\n" + "Referenced pattern IDs: [" + patternIdsString + "]"; + // Append referenced patterns data to the end of the error. + message = String.format( + "%s\nReferenced patterns: [%s]", + message, + StringUtils.join(patternAndRouteIds, ",") + ); } LOG.warn(message); throw new SQLException(message); From 41a6503b2bd70f4f00d3368be34fbd74f6f4dcbb Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 10 May 2023 15:24:19 -0400 Subject: [PATCH 21/38] feat(Table): add primary key names to Table class. --- .../java/com/conveyal/gtfs/loader/Table.java | 58 ++++++++++++++----- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index 705691f06..c4bc39eb1 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -100,6 +100,9 @@ public class Table { * */ private boolean compoundKey; + /** Key(s) that uniquely identify an entry in the respective table.*/ + private String[] primaryKeyNames; + public Table (String name, Class entityClass, Requirement required, Field... fields) { // TODO: verify table name is OK for use in constructing dynamic SQL queries this.name = name; @@ -123,7 +126,7 @@ public Table (String name, Class entityClass, Requirement requ new URLField("agency_branding_url", OPTIONAL), new URLField("agency_fare_url", OPTIONAL), new StringField("agency_email", OPTIONAL) // FIXME new field type for emails? - ).restrictDelete().addPrimaryKey(); + ).restrictDelete().addPrimaryKey().addPrimaryKeyNames("agency_id"); // The GTFS spec says this table is required, but in practice it is not required if calendar_dates is present. public static final Table CALENDAR = new Table("calendar", Calendar.class, OPTIONAL, @@ -139,7 +142,7 @@ public Table (String name, Class entityClass, Requirement requ new DateField("end_date", REQUIRED), // Editor-specific field new StringField("description", EDITOR) - ).restrictDelete().addPrimaryKey(); + ).restrictDelete().addPrimaryKey().addPrimaryKeyNames("service_id"); public static final Table SCHEDULE_EXCEPTIONS = new Table("schedule_exceptions", ScheduleException.class, EDITOR, new StringField("name", REQUIRED), // FIXME: This makes name the key field... @@ -155,7 +158,7 @@ public Table (String name, Class entityClass, Requirement requ new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), new DateField("date", REQUIRED), new IntegerField("exception_type", REQUIRED, 1, 2) - ).keyFieldIsNotUnique(); + ).keyFieldIsNotUnique().addPrimaryKeyNames("service_id", "date"); public static final Table FARE_ATTRIBUTES = new Table("fare_attributes", FareAttribute.class, OPTIONAL, new StringField("fare_id", REQUIRED), @@ -169,7 +172,7 @@ public Table (String name, Class entityClass, Requirement requ new ReferenceFieldShouldBeProvidedCheck("agency_id") ), new IntegerField("transfer_duration", OPTIONAL) - ).addPrimaryKey(); + ).addPrimaryKey().addPrimaryKeyNames("fare_id"); // FIXME: Should we add some constraint on number of rows that this table has? Perhaps this is a GTFS editor specific // feature. @@ -218,7 +221,7 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("status", EDITOR, 2), new ShortField("continuous_pickup", OPTIONAL,3), new ShortField("continuous_drop_off", OPTIONAL,3) - ).addPrimaryKey(); + ).addPrimaryKey().addPrimaryKeyNames("route_id"); public static final Table SHAPES = new Table("shapes", ShapePoint.class, OPTIONAL, new StringField("shape_id", REQUIRED), @@ -231,7 +234,7 @@ public Table (String name, Class entityClass, Requirement requ // 1 - user-designated anchor point (handle with which the user can manipulate shape) // 2 - stop-projected point (dictates the value of shape_dist_traveled for a pattern stop) new ShortField("point_type", EDITOR, 2) - ); + ).addPrimaryKeyNames("shape_id", "shape_pt_sequence"); public static final Table PATTERNS = new Table("patterns", Pattern.class, OPTIONAL, new StringField("pattern_id", REQUIRED), @@ -242,7 +245,7 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("direction_id", EDITOR, 1), new ShortField("use_frequency", EDITOR, 1), new StringField("shape_id", EDITOR).isReferenceTo(SHAPES) - ).addPrimaryKey(); + ).addPrimaryKey().addPrimaryKeyNames("pattern_id"); public static final Table STOPS = new Table("stops", Stop.class, REQUIRED, new StringField("stop_id", REQUIRED), @@ -271,7 +274,8 @@ public Table (String name, Class entityClass, Requirement requ new StringField("platform_code", OPTIONAL) ) .restrictDelete() - .addPrimaryKey(); + .addPrimaryKey() + .addPrimaryKeyNames("stop_id"); // GTFS reference: https://developers.google.com/transit/gtfs/reference#fare_rulestxt public static final Table FARE_RULES = new Table("fare_rules", FareRule.class, OPTIONAL, @@ -291,7 +295,8 @@ public Table (String name, Class entityClass, Requirement requ ) ) .withParentTable(FARE_ATTRIBUTES) - .addPrimaryKey().keyFieldIsNotUnique(); + .addPrimaryKey().keyFieldIsNotUnique() + .addPrimaryKeyNames("fare_id", "route_id", "origin_id", "destination_id", "contains_id"); public static final Table PATTERN_STOP = new Table("pattern_stops", PatternStop.class, OPTIONAL, new StringField("pattern_id", REQUIRED).isReferenceTo(PATTERNS), @@ -308,7 +313,7 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("timepoint", EDITOR, 1), new ShortField("continuous_pickup", OPTIONAL,3), new ShortField("continuous_drop_off", OPTIONAL,3) - ).withParentTable(PATTERNS); + ).withParentTable(PATTERNS).addPrimaryKeyNames("pattern_id", "stop_sequence"); public static final Table TRANSFERS = new Table("transfers", Transfer.class, OPTIONAL, // FIXME: Do we need an index on from_ and to_stop_id @@ -318,7 +323,7 @@ public Table (String name, Class entityClass, Requirement requ new StringField("min_transfer_time", OPTIONAL)) .addPrimaryKey() .keyFieldIsNotUnique() - .hasCompoundKey(); + .addPrimaryKeyNames("from_stop_id", "to_stop_id"); public static final Table TRIPS = new Table("trips", Trip.class, REQUIRED, new StringField("trip_id", REQUIRED), @@ -335,7 +340,7 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("bikes_allowed", OPTIONAL, 2), // Editor-specific fields below. new StringField("pattern_id", EDITOR).isReferenceTo(PATTERNS) - ).addPrimaryKey(); + ).addPrimaryKey().addPrimaryKeyNames("trip_id"); // Must come after TRIPS and STOPS table to which it has references public static final Table STOP_TIMES = new Table("stop_times", StopTime.class, REQUIRED, @@ -355,7 +360,7 @@ public Table (String name, Class entityClass, Requirement requ new DoubleField("shape_dist_traveled", OPTIONAL, 0, Double.POSITIVE_INFINITY, -1), new ShortField("timepoint", OPTIONAL, 1), new IntegerField("fare_units_traveled", EXTENSION) // OpenOV NL extension - ).withParentTable(TRIPS); + ).withParentTable(TRIPS).addPrimaryKeyNames("trip_id", "stop_sequence"); // Must come after TRIPS table to which it has a reference public static final Table FREQUENCIES = new Table("frequencies", Frequency.class, OPTIONAL, @@ -368,7 +373,8 @@ public Table (String name, Class entityClass, Requirement requ new IntegerField("headway_secs", REQUIRED, 20, 60*60*6), new IntegerField("exact_times", OPTIONAL, 1)) .withParentTable(TRIPS) - .keyFieldIsNotUnique(); + .keyFieldIsNotUnique() + .addPrimaryKeyNames("trip_id", "start_time"); // GTFS reference: https://developers.google.com/transit/gtfs/reference#attributionstxt public static final Table TRANSLATIONS = new Table("translations", Translation.class, OPTIONAL, @@ -388,7 +394,8 @@ public Table (String name, Class entityClass, Requirement requ // If the record_id is empty the field_value is required. new FieldIsEmptyCheck("record_id") )) - .keyFieldIsNotUnique(); + .keyFieldIsNotUnique() + .addPrimaryKeyNames("table_name", "field_name", "language", "record_id", "record_sub_id", "field_value"); public static final Table ATTRIBUTIONS = new Table("attributions", Attribution.class, OPTIONAL, new StringField("attribution_id", OPTIONAL), @@ -401,7 +408,8 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("is_authority", OPTIONAL, 1), new URLField("attribution_url", OPTIONAL), new StringField("attribution_email", OPTIONAL), - new StringField("attribution_phone", OPTIONAL)); + new StringField("attribution_phone", OPTIONAL)) + .addPrimaryKeyNames("attribution_id"); /** List of tables in order needed for checking referential integrity during load stage. */ public static final Table[] tablesInOrder = { @@ -461,6 +469,24 @@ public Table addPrimaryKey () { return this; } + /** + * Fluent method that records the field names that are primary keys for a table. + */ + public Table addPrimaryKeyNames(String ...keys) { + this.primaryKeyNames = keys; + return this; + } + + /** + * Get field names that are primary keys for a table. + */ + public List getPrimaryKeyNames () { + List keys = new ArrayList<>(); + for (String key : primaryKeyNames) keys.add(key); + return keys; + } + + /** * Registers the table with a parent table. When updates are made to the parent table, updates to child entities * nested in the JSON string will be made. For example, pattern stops and shape points use this method to point to From 031872599d41fbafd026dec175d5292fbe7b2301 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Mon, 15 May 2023 10:41:34 -0400 Subject: [PATCH 22/38] refactor(Table): respond to PR comments --- src/main/java/com/conveyal/gtfs/loader/Table.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index c4bc39eb1..b5c70680b 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -323,6 +323,7 @@ public Table (String name, Class entityClass, Requirement requ new StringField("min_transfer_time", OPTIONAL)) .addPrimaryKey() .keyFieldIsNotUnique() + .hasCompoundKey() .addPrimaryKeyNames("from_stop_id", "to_stop_id"); public static final Table TRIPS = new Table("trips", Trip.class, REQUIRED, @@ -478,12 +479,11 @@ public Table addPrimaryKeyNames(String ...keys) { } /** - * Get field names that are primary keys for a table. + * Get field names that are primary keys for a table. This is used by the PreserveCustomFields transformation + * in datatools-server for determining keys for CSV matching. */ public List getPrimaryKeyNames () { - List keys = new ArrayList<>(); - for (String key : primaryKeyNames) keys.add(key); - return keys; + return Arrays.asList(primaryKeyNames); } From d7623e80bfbdd31c651959e29d1918b517a6c7cd Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 16 May 2023 10:18:54 -0400 Subject: [PATCH 23/38] refactor(Table): make formatting consistent --- .../java/com/conveyal/gtfs/loader/Table.java | 63 +++++++++++-------- 1 file changed, 36 insertions(+), 27 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index b5c70680b..abded3214 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -126,7 +126,9 @@ public Table (String name, Class entityClass, Requirement requ new URLField("agency_branding_url", OPTIONAL), new URLField("agency_fare_url", OPTIONAL), new StringField("agency_email", OPTIONAL) // FIXME new field type for emails? - ).restrictDelete().addPrimaryKey().addPrimaryKeyNames("agency_id"); + ).restrictDelete() + .addPrimaryKey() + .addPrimaryKeyNames("agency_id"); // The GTFS spec says this table is required, but in practice it is not required if calendar_dates is present. public static final Table CALENDAR = new Table("calendar", Calendar.class, OPTIONAL, @@ -142,7 +144,9 @@ public Table (String name, Class entityClass, Requirement requ new DateField("end_date", REQUIRED), // Editor-specific field new StringField("description", EDITOR) - ).restrictDelete().addPrimaryKey().addPrimaryKeyNames("service_id"); + ).restrictDelete() + .addPrimaryKey() + .addPrimaryKeyNames("service_id"); public static final Table SCHEDULE_EXCEPTIONS = new Table("schedule_exceptions", ScheduleException.class, EDITOR, new StringField("name", REQUIRED), // FIXME: This makes name the key field... @@ -158,7 +162,8 @@ public Table (String name, Class entityClass, Requirement requ new StringField("service_id", REQUIRED).isReferenceTo(CALENDAR), new DateField("date", REQUIRED), new IntegerField("exception_type", REQUIRED, 1, 2) - ).keyFieldIsNotUnique().addPrimaryKeyNames("service_id", "date"); + ).keyFieldIsNotUnique() + .addPrimaryKeyNames("service_id", "date"); public static final Table FARE_ATTRIBUTES = new Table("fare_attributes", FareAttribute.class, OPTIONAL, new StringField("fare_id", REQUIRED), @@ -172,7 +177,8 @@ public Table (String name, Class entityClass, Requirement requ new ReferenceFieldShouldBeProvidedCheck("agency_id") ), new IntegerField("transfer_duration", OPTIONAL) - ).addPrimaryKey().addPrimaryKeyNames("fare_id"); + ).addPrimaryKey() + .addPrimaryKeyNames("fare_id"); // FIXME: Should we add some constraint on number of rows that this table has? Perhaps this is a GTFS editor specific // feature. @@ -221,7 +227,8 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("status", EDITOR, 2), new ShortField("continuous_pickup", OPTIONAL,3), new ShortField("continuous_drop_off", OPTIONAL,3) - ).addPrimaryKey().addPrimaryKeyNames("route_id"); + ).addPrimaryKey() + .addPrimaryKeyNames("route_id"); public static final Table SHAPES = new Table("shapes", ShapePoint.class, OPTIONAL, new StringField("shape_id", REQUIRED), @@ -245,7 +252,8 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("direction_id", EDITOR, 1), new ShortField("use_frequency", EDITOR, 1), new StringField("shape_id", EDITOR).isReferenceTo(SHAPES) - ).addPrimaryKey().addPrimaryKeyNames("pattern_id"); + ).addPrimaryKey() + .addPrimaryKeyNames("pattern_id"); public static final Table STOPS = new Table("stops", Stop.class, REQUIRED, new StringField("stop_id", REQUIRED), @@ -272,8 +280,7 @@ public Table (String name, Class entityClass, Requirement requ new StringField("stop_timezone", OPTIONAL), new ShortField("wheelchair_boarding", OPTIONAL, 2), new StringField("platform_code", OPTIONAL) - ) - .restrictDelete() + ).restrictDelete() .addPrimaryKey() .addPrimaryKeyNames("stop_id"); @@ -293,8 +300,7 @@ public Table (String name, Class entityClass, Requirement requ // If the contains_id is defined, its value must exist as a zone_id in stops.txt. new ForeignRefExistsCheck("zone_id", "fare_rules") ) - ) - .withParentTable(FARE_ATTRIBUTES) + ).withParentTable(FARE_ATTRIBUTES) .addPrimaryKey().keyFieldIsNotUnique() .addPrimaryKeyNames("fare_id", "route_id", "origin_id", "destination_id", "contains_id"); @@ -313,18 +319,19 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("timepoint", EDITOR, 1), new ShortField("continuous_pickup", OPTIONAL,3), new ShortField("continuous_drop_off", OPTIONAL,3) - ).withParentTable(PATTERNS).addPrimaryKeyNames("pattern_id", "stop_sequence"); + ).withParentTable(PATTERNS) + .addPrimaryKeyNames("pattern_id", "stop_sequence"); public static final Table TRANSFERS = new Table("transfers", Transfer.class, OPTIONAL, // FIXME: Do we need an index on from_ and to_stop_id new StringField("from_stop_id", REQUIRED).isReferenceTo(STOPS), new StringField("to_stop_id", REQUIRED).isReferenceTo(STOPS), new ShortField("transfer_type", REQUIRED, 3), - new StringField("min_transfer_time", OPTIONAL)) - .addPrimaryKey() - .keyFieldIsNotUnique() - .hasCompoundKey() - .addPrimaryKeyNames("from_stop_id", "to_stop_id"); + new StringField("min_transfer_time", OPTIONAL) + ).addPrimaryKey() + .keyFieldIsNotUnique() + .hasCompoundKey() + .addPrimaryKeyNames("from_stop_id", "to_stop_id"); public static final Table TRIPS = new Table("trips", Trip.class, REQUIRED, new StringField("trip_id", REQUIRED), @@ -341,7 +348,8 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("bikes_allowed", OPTIONAL, 2), // Editor-specific fields below. new StringField("pattern_id", EDITOR).isReferenceTo(PATTERNS) - ).addPrimaryKey().addPrimaryKeyNames("trip_id"); + ).addPrimaryKey() + .addPrimaryKeyNames("trip_id"); // Must come after TRIPS and STOPS table to which it has references public static final Table STOP_TIMES = new Table("stop_times", StopTime.class, REQUIRED, @@ -361,7 +369,8 @@ public Table (String name, Class entityClass, Requirement requ new DoubleField("shape_dist_traveled", OPTIONAL, 0, Double.POSITIVE_INFINITY, -1), new ShortField("timepoint", OPTIONAL, 1), new IntegerField("fare_units_traveled", EXTENSION) // OpenOV NL extension - ).withParentTable(TRIPS).addPrimaryKeyNames("trip_id", "stop_sequence"); + ).withParentTable(TRIPS) + .addPrimaryKeyNames("trip_id", "stop_sequence"); // Must come after TRIPS table to which it has a reference public static final Table FREQUENCIES = new Table("frequencies", Frequency.class, OPTIONAL, @@ -372,10 +381,10 @@ public Table (String name, Class entityClass, Requirement requ // (e.g., a ferry running exact times at a 4 hour headway), but will catch cases where milliseconds were // exported accidentally. new IntegerField("headway_secs", REQUIRED, 20, 60*60*6), - new IntegerField("exact_times", OPTIONAL, 1)) - .withParentTable(TRIPS) - .keyFieldIsNotUnique() - .addPrimaryKeyNames("trip_id", "start_time"); + new IntegerField("exact_times", OPTIONAL, 1) + ).withParentTable(TRIPS) + .keyFieldIsNotUnique() + .addPrimaryKeyNames("trip_id", "start_time"); // GTFS reference: https://developers.google.com/transit/gtfs/reference#attributionstxt public static final Table TRANSLATIONS = new Table("translations", Translation.class, OPTIONAL, @@ -394,9 +403,9 @@ public Table (String name, Class entityClass, Requirement requ new StringField("field_value", OPTIONAL).requireConditions( // If the record_id is empty the field_value is required. new FieldIsEmptyCheck("record_id") - )) - .keyFieldIsNotUnique() - .addPrimaryKeyNames("table_name", "field_name", "language", "record_id", "record_sub_id", "field_value"); + ) + ).keyFieldIsNotUnique() + .addPrimaryKeyNames("table_name", "field_name", "language", "record_id", "record_sub_id", "field_value"); public static final Table ATTRIBUTIONS = new Table("attributions", Attribution.class, OPTIONAL, new StringField("attribution_id", OPTIONAL), @@ -409,8 +418,8 @@ public Table (String name, Class entityClass, Requirement requ new ShortField("is_authority", OPTIONAL, 1), new URLField("attribution_url", OPTIONAL), new StringField("attribution_email", OPTIONAL), - new StringField("attribution_phone", OPTIONAL)) - .addPrimaryKeyNames("attribution_id"); + new StringField("attribution_phone", OPTIONAL) + ).addPrimaryKeyNames("attribution_id"); /** List of tables in order needed for checking referential integrity during load stage. */ public static final Table[] tablesInOrder = { From 8bfbcc0fe3a4739c5af8d54ddc527fe35153550a Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 16 May 2023 12:12:26 -0400 Subject: [PATCH 24/38] refactor(Table): remove reference to datatools-server --- src/main/java/com/conveyal/gtfs/loader/Table.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/Table.java b/src/main/java/com/conveyal/gtfs/loader/Table.java index abded3214..b3344c5c5 100644 --- a/src/main/java/com/conveyal/gtfs/loader/Table.java +++ b/src/main/java/com/conveyal/gtfs/loader/Table.java @@ -488,8 +488,7 @@ public Table addPrimaryKeyNames(String ...keys) { } /** - * Get field names that are primary keys for a table. This is used by the PreserveCustomFields transformation - * in datatools-server for determining keys for CSV matching. + * Get field names that are primary keys for a table. */ public List getPrimaryKeyNames () { return Arrays.asList(primaryKeyNames); From 673353d32533366dae35e076666eba0a1ee65665 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 16 May 2023 16:45:28 -0400 Subject: [PATCH 25/38] feat(readme): add gitter to readme --- README.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/README.md b/README.md index 8dc117d4b..fb4ee3b9b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ # gtfs-lib [![Build Status](https://travis-ci.org/conveyal/gtfs-lib.svg?branch=master)](https://travis-ci.org/conveyal/gtfs-lib) +[![Join the chat at https://matrix.to/#/#gtfs-lib:gitter.im](https://badges.gitter.im/repo.png)](https://matrix.to/#/#gtfs-lib:gitter.im) + A library for loading and saving GTFS feeds of arbitrary size with disk-backed storage. @@ -14,6 +16,10 @@ The main design goals are: A gtfs-lib GTFSFeed object should faithfully represent the contents of a single GTFS feed file. At least during the initial load, no heuristics are applied to clean up the data. Basic syntax is verified, and any problems encountered are logged in detail. At this stage, fields or entites may be missing, and the data may be nonsensical. Then in an optional post-load validation phase, semantic checks are performed and problems are optionally repaired. +## Getting in touch + +We have a Gitter [space](https://matrix.to/#/#transit-data-tools:gitter.im) for the full TRANSIT-Data-Tools project where you can post questions and comments. This includes a room dedicated to GTFS-lib discussions. + ## Usage gtfs-lib can be used as a Java library or run via the command line. If using this library with PostgreSQL for persistence, you must use at least version 9.6 of PostgreSQL. From 7fd58e6a3550ea006f4b2b9cdb794f64e70a3954 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 17 May 2023 09:48:15 -0400 Subject: [PATCH 26/38] fix(stop delete): Fix pattern ID uses in stop deletion link --- src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index a860e36e5..ffd669ed1 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -1576,7 +1576,7 @@ private void updateReferencingTables( connection.rollback(); if (entityClass.getSimpleName().equals("Stop")) { String patternStopLookup = String.format( - "select distinct ps.pattern_id, r.id " + + "select distinct p.id, r.id " + "from %s.pattern_stops ps " + "inner join " + "%s.patterns p " + From fbd40a35faf1af74ac6363049941dc664ff76a7f Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Tue, 4 Jul 2023 17:02:46 -0400 Subject: [PATCH 27/38] fix(exceptions): allow commas in service IDs --- src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java | 5 ++++- src/main/java/com/conveyal/gtfs/loader/StringListField.java | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java index ffd669ed1..a967f0f96 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcTableWriter.java @@ -427,7 +427,10 @@ private void setStatementParameters( // Array field type expects comma-delimited values. List values = new ArrayList<>(); for (JsonNode node : value) { - values.add(node.asText()); + String nodeText = node.asText(); + // Surround text value in quotes to preserve any internal commas + if (field instanceof StringListField) nodeText = "\"" + nodeText + "\""; + values.add(nodeText); } field.setParameter(preparedStatement, index, String.join(",", values)); } else { diff --git a/src/main/java/com/conveyal/gtfs/loader/StringListField.java b/src/main/java/com/conveyal/gtfs/loader/StringListField.java index 6319afe67..63474339c 100644 --- a/src/main/java/com/conveyal/gtfs/loader/StringListField.java +++ b/src/main/java/com/conveyal/gtfs/loader/StringListField.java @@ -7,6 +7,7 @@ import java.sql.JDBCType; import java.sql.PreparedStatement; import java.sql.SQLType; +import java.util.Arrays; import java.util.Collections; import java.util.Set; @@ -34,7 +35,10 @@ public ValidateFieldResult validateAndConvert(String original) { public Set setParameter(PreparedStatement preparedStatement, int oneBasedIndex, String string) { // FIXME try { - Array array = preparedStatement.getConnection().createArrayOf("text", string.split(",")); + String[] stringList = string.split("(?<=\"),"); + // Clean the string list of any escaped quotations which are required to preserve any internal commas + stringList = Arrays.stream(stringList).map(s -> s.replace("\"", "")).toArray(String[]::new); + Array array = preparedStatement.getConnection().createArrayOf("text", stringList); preparedStatement.setArray(oneBasedIndex, array); return Collections.EMPTY_SET; } catch (Exception e) { From bc1e1b7051acc7a8c9c149c2453bdf38bb94a2f9 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 5 Jul 2023 10:16:42 -0400 Subject: [PATCH 28/38] refactor(exceptions): address PR comments --- src/main/java/com/conveyal/gtfs/loader/StringListField.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/StringListField.java b/src/main/java/com/conveyal/gtfs/loader/StringListField.java index 63474339c..8d386a645 100644 --- a/src/main/java/com/conveyal/gtfs/loader/StringListField.java +++ b/src/main/java/com/conveyal/gtfs/loader/StringListField.java @@ -33,10 +33,10 @@ public ValidateFieldResult validateAndConvert(String original) { @Override public Set setParameter(PreparedStatement preparedStatement, int oneBasedIndex, String string) { - // FIXME try { + // Only split on commas following an escaped quotation mark, as this indicates a new item in the list. String[] stringList = string.split("(?<=\"),"); - // Clean the string list of any escaped quotations which are required to preserve any internal commas + // Clean the string list of any escaped quotations which are required to preserve any internal commas. stringList = Arrays.stream(stringList).map(s -> s.replace("\"", "")).toArray(String[]::new); Array array = preparedStatement.getConnection().createArrayOf("text", stringList); preparedStatement.setArray(oneBasedIndex, array); From fcdc1cc9057071b37983c7f8d8b7edb6499c7db0 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Thu, 6 Jul 2023 15:30:45 -0400 Subject: [PATCH 29/38] fix(exceptions): allow exception overlap in backend --- .../java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java index 61f6bb8cf..647c497aa 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java @@ -30,6 +30,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import java.util.zip.ZipEntry; import java.util.zip.ZipOutputStream; @@ -135,7 +136,12 @@ public FeedLoadResult exportTables() { for (Calendar cal : calendars) { Service service = new Service(cal.service_id); service.calendar = cal; - for (ScheduleException ex : exceptions) { + List calendarExceptions = exceptions.stream().filter(ex -> + ex.addedService.contains(cal.service_id) || + ex.removedService.contains(cal.service_id) || + ex.customSchedule.contains(cal.service_id) + ).collect(Collectors.toList()); + for (ScheduleException ex : calendarExceptions) { if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.SWAP) && (!ex.addedService.contains(cal.service_id) && !ex.removedService.contains(cal.service_id))) { // Skip swap exception if cal is not referenced by added or removed service. From 677ba68d91cf7e9f7609373bd6b3d031e90ed797 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 7 Jul 2023 09:07:16 -0400 Subject: [PATCH 30/38] refactor(exceptions): update to avoid merge conflict --- src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java index 647c497aa..63821baa3 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java @@ -159,7 +159,7 @@ public FeedLoadResult exportTables() { calendarDate.date = date; calendarDate.service_id = cal.service_id; calendarDate.exception_type = ex.serviceRunsOn(cal) ? 1 : 2; - LOG.info("Adding exception {} (type={}) for calendar {} on date {}", ex.name, calendarDate.exception_type, cal.service_id, date.toString()); + LOG.info("Adding exception {} (type={}) for calendar {} on date {}", ex.name, calendarDate.exception_type, cal.service_id, date); if (service.calendar_dates.containsKey(date)) throw new IllegalArgumentException("Duplicate schedule exceptions on " + date.toString()); From e17463e59d2c55efebd5b106287cb0e70bf83412 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Fri, 7 Jul 2023 13:54:46 -0400 Subject: [PATCH 31/38] refactor(exceptions): use list methods for cleaning quotations --- .../java/com/conveyal/gtfs/loader/StringListField.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/StringListField.java b/src/main/java/com/conveyal/gtfs/loader/StringListField.java index 8d386a645..c971e1a1f 100644 --- a/src/main/java/com/conveyal/gtfs/loader/StringListField.java +++ b/src/main/java/com/conveyal/gtfs/loader/StringListField.java @@ -9,6 +9,7 @@ import java.sql.SQLType; import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.Set; /** @@ -35,10 +36,10 @@ public ValidateFieldResult validateAndConvert(String original) { public Set setParameter(PreparedStatement preparedStatement, int oneBasedIndex, String string) { try { // Only split on commas following an escaped quotation mark, as this indicates a new item in the list. - String[] stringList = string.split("(?<=\"),"); + List stringList = Arrays.asList(string.split("(?<=\"),")); // Clean the string list of any escaped quotations which are required to preserve any internal commas. - stringList = Arrays.stream(stringList).map(s -> s.replace("\"", "")).toArray(String[]::new); - Array array = preparedStatement.getConnection().createArrayOf("text", stringList); + stringList.replaceAll(s -> s.replace("\"", "")); + Array array = preparedStatement.getConnection().createArrayOf("text", stringList.toArray(new String[0])); preparedStatement.setArray(oneBasedIndex, array); return Collections.EMPTY_SET; } catch (Exception e) { From e51456e60c8761f646a7bcb71e600676c49606aa Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 12 Jul 2023 08:35:43 -0400 Subject: [PATCH 32/38] refactor(exceptions): inline exceptions filtering for For loop --- .../gtfs/loader/JdbcGtfsExporter.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java index 63821baa3..cd3b25aef 100644 --- a/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java +++ b/src/main/java/com/conveyal/gtfs/loader/JdbcGtfsExporter.java @@ -68,6 +68,17 @@ public JdbcGtfsExporter(String feedId, String outFile, DataSource dataSource, bo this.fromEditor = fromEditor; } + /** + * Utility method to check if an exception uses a specific service. + */ + public Boolean exceptionInvolvesService(ScheduleException ex, String serviceId) { + return ( + ex.addedService.contains(serviceId) || + ex.removedService.contains(serviceId) || + ex.customSchedule.contains(serviceId) + ); + } + /** * Export primary entity tables as well as Pattern and PatternStops tables. * @@ -136,12 +147,10 @@ public FeedLoadResult exportTables() { for (Calendar cal : calendars) { Service service = new Service(cal.service_id); service.calendar = cal; - List calendarExceptions = exceptions.stream().filter(ex -> - ex.addedService.contains(cal.service_id) || - ex.removedService.contains(cal.service_id) || - ex.customSchedule.contains(cal.service_id) - ).collect(Collectors.toList()); - for (ScheduleException ex : calendarExceptions) { + for (ScheduleException ex : exceptions.stream() + .filter(ex -> exceptionInvolvesService(ex, cal.service_id)) + .collect(Collectors.toList()) + ) { if (ex.exemplar.equals(ScheduleException.ExemplarServiceDescriptor.SWAP) && (!ex.addedService.contains(cal.service_id) && !ex.removedService.contains(cal.service_id))) { // Skip swap exception if cal is not referenced by added or removed service. From 3be63a7b7ae75ed2a1040e9aa2d0e774294af89c Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Tue, 25 Jul 2023 19:34:23 +0200 Subject: [PATCH 33/38] fix: new shared stops error types --- src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java index 477c034dc..adb83f674 100644 --- a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java +++ b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java @@ -85,6 +85,11 @@ public enum NewGTFSErrorType { // MTC-specific errors. FIELD_VALUE_TOO_LONG(Priority.MEDIUM, "Field value has too many characters."), + // Shared Stops-specifc errors. + MULTIPLE_SHARED_STOPS_GROUPS(Priority.HIGH, "GTFS entity belongs to more than one shared-stop group."), + SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS(Priority.HIGH, "Shared-stop group has multiple primary stops."), + SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST(Priority.MEDIUM, "Shared-stop group entity does not exist."), + // Unknown errors. OTHER(Priority.LOW, "Other errors."); From 9903b3b63ad26d135b5a55cf17e595557db037bb Mon Sep 17 00:00:00 2001 From: miles-grant-ibigroup Date: Fri, 28 Jul 2023 13:42:30 +0200 Subject: [PATCH 34/38] refactor: imporve error messages --- src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java index adb83f674..80154ae83 100644 --- a/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java +++ b/src/main/java/com/conveyal/gtfs/error/NewGTFSErrorType.java @@ -86,9 +86,9 @@ public enum NewGTFSErrorType { FIELD_VALUE_TOO_LONG(Priority.MEDIUM, "Field value has too many characters."), // Shared Stops-specifc errors. - MULTIPLE_SHARED_STOPS_GROUPS(Priority.HIGH, "GTFS entity belongs to more than one shared-stop group."), - SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS(Priority.HIGH, "Shared-stop group has multiple primary stops."), - SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST(Priority.MEDIUM, "Shared-stop group entity does not exist."), + MULTIPLE_SHARED_STOPS_GROUPS(Priority.HIGH, "A GTFS stop belongs to more than one shared-stop group."), + SHARED_STOP_GROUP_MUTLIPLE_PRIMARY_STOPS(Priority.HIGH, "A Shared-stop group has multiple primary stops."), + SHARED_STOP_GROUP_ENTITY_DOES_NOT_EXIST(Priority.MEDIUM, "The stop referenced by a shared-stop does not exist."), // Unknown errors. OTHER(Priority.LOW, "Other errors."); From 25b37a8d97b82b4115e2d8d71a1ec9edc8e0bfa3 Mon Sep 17 00:00:00 2001 From: "philip.cline" Date: Wed, 9 Aug 2023 10:42:52 -0400 Subject: [PATCH 35/38] refactor(readme): change badge to main link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fb4ee3b9b..a7435da35 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # gtfs-lib [![Build Status](https://travis-ci.org/conveyal/gtfs-lib.svg?branch=master)](https://travis-ci.org/conveyal/gtfs-lib) -[![Join the chat at https://matrix.to/#/#gtfs-lib:gitter.im](https://badges.gitter.im/repo.png)](https://matrix.to/#/#gtfs-lib:gitter.im) +[![Join the chat at https://matrix.to/#/#transit-data-tools:gitter.im](https://badges.gitter.im/repo.png)](https://matrix.to/#/#gtfs-lib:gitter.im) A library for loading and saving GTFS feeds of arbitrary size with disk-backed storage. From ed6fe94cbceadf0fe3677915fec79e553638cb6a Mon Sep 17 00:00:00 2001 From: Philip Cline <63798641+philip-cline@users.noreply.github.com> Date: Thu, 10 Aug 2023 10:03:21 -0400 Subject: [PATCH 36/38] refactor(README): update link Co-authored-by: binh-dam-ibigroup <56846598+binh-dam-ibigroup@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index a7435da35..08531d516 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ A gtfs-lib GTFSFeed object should faithfully represent the contents of a single ## Getting in touch -We have a Gitter [space](https://matrix.to/#/#transit-data-tools:gitter.im) for the full TRANSIT-Data-Tools project where you can post questions and comments. This includes a room dedicated to GTFS-lib discussions. +We have a [Gitter space](https://matrix.to/#/#transit-data-tools:gitter.im) for the full TRANSIT-Data-Tools project where you can post questions and comments. This includes a room dedicated to GTFS-lib discussions. ## Usage From 42fc7e1fe29352131e4284d19680ba1f70599942 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 09:16:50 +0000 Subject: [PATCH 37/38] chore(deps): bump com.google.guava:guava from 30.0-jre to 32.0.0-jre Bumps [com.google.guava:guava](https://github.com/google/guava) from 30.0-jre to 32.0.0-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c3d09335d..8cfbc17b7 100644 --- a/pom.xml +++ b/pom.xml @@ -241,7 +241,7 @@ com.google.guava guava - 30.0-jre + 32.0.0-jre org.locationtech.jts From c60b3f7430d2b3a52b3bf9a0a0d366672d9cc0a8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 21 Aug 2023 09:24:45 +0000 Subject: [PATCH 38/38] chore(deps): bump org.postgresql:postgresql from 42.4.1 to 42.4.3 Bumps [org.postgresql:postgresql](https://github.com/pgjdbc/pgjdbc) from 42.4.1 to 42.4.3. - [Release notes](https://github.com/pgjdbc/pgjdbc/releases) - [Changelog](https://github.com/pgjdbc/pgjdbc/blob/master/CHANGELOG.md) - [Commits](https://github.com/pgjdbc/pgjdbc/compare/REL42.4.1...REL42.4.3) --- updated-dependencies: - dependency-name: org.postgresql:postgresql dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 66be192b2..889de28ba 100644 --- a/pom.xml +++ b/pom.xml @@ -314,7 +314,7 @@ org.postgresql postgresql - 42.4.1 + 42.4.3 org.apache.commons