diff --git a/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java b/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java index efd02b294..99b128e2d 100644 --- a/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java +++ b/src/main/java/com/conveyal/datatools/manager/DataSanitizer.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Set; import java.util.stream.Collectors; +import java.util.stream.Stream; import static com.conveyal.datatools.manager.DataManager.GTFS_DATA_SOURCE; import static com.conveyal.datatools.manager.DataManager.initializeApplication; @@ -170,7 +171,7 @@ public static int deleteObsoleteFeedVersions( int keepCount = 0; int deleteCount = 0; - + for (FeedVersion feedVersion : feedVersions) { if (keepCount < numberOfVersionsToKeep) { keepCount++; @@ -231,7 +232,7 @@ public static List feedVersionAudit() { * Group orphaned schemas and optionally delete. */ public static void sanitizeDBSchemas(boolean delete) { - Set orphanedSchemas = getOrphanedDBSchemas(getFieldFromDocument("namespace", "FeedVersion")); + Set orphanedSchemas = getOrphanedDBSchemas(getActiveDBSchemas()); if (orphanedSchemas.isEmpty()) { System.out.println("No orphaned DB schemas found!"); @@ -247,6 +248,18 @@ public static void sanitizeDBSchemas(boolean delete) { } } + /** + * Get all active namespaces/schemas from feed sources and feed versions. + */ + public static Set getActiveDBSchemas() { + return Stream + .concat( + getFieldFromDocument("namespace", "FeedVersion").stream(), + getFieldFromDocument("editorNamespace", "FeedSource").stream() + ) + .collect(Collectors.toSet()); + } + /** * Delete orphaned feed versions. */ @@ -283,7 +296,7 @@ private static List getOrphanedFeedVersions() { } /** - * Get all qualifying schemas that are not associated with a feed version. + * Get all qualifying schemas that are not associated with a feed version or a GTFS Editor for a feed source. */ public static Set getOrphanedDBSchemas(Set associatedSchemas) { String whereClause = associatedSchemas.isEmpty() ? "" : String.format(" WHERE nspname NOT IN (%s)", associatedSchemas diff --git a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java index 9648d0982..b17a84c67 100644 --- a/src/test/java/com/conveyal/datatools/DataSanitizerTest.java +++ b/src/test/java/com/conveyal/datatools/DataSanitizerTest.java @@ -1,6 +1,5 @@ package com.conveyal.datatools; -import com.conveyal.datatools.common.utils.aws.CheckedAWSException; import com.conveyal.datatools.manager.DataManager; import com.conveyal.datatools.manager.DataSanitizer; import com.conveyal.datatools.manager.auth.Auth0Connection; @@ -10,14 +9,12 @@ import com.conveyal.datatools.manager.models.Project; import com.conveyal.datatools.manager.persistence.Persistence; import com.conveyal.gtfs.GTFS; -import com.conveyal.gtfs.util.InvalidNamespaceException; import com.mongodb.client.model.Sorts; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import java.io.IOException; -import java.sql.SQLException; import java.util.Collection; import java.util.List; import java.util.HashSet; @@ -54,6 +51,7 @@ static void setUp() throws IOException { FeedSource feedSource = new FeedSource(appendDate("Test Feed"), project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(feedSource); feedSourceParent = new FeedSource(appendDate("Test Feed 2"), project.id, MANUALLY_UPLOADED); + feedSourceParent.editorNamespace = "active-editor-namespace"; Persistence.feedSources.create(feedSourceParent); feedSourceWithObsoleteFeedVersion = new FeedSource(appendDate("Test Feed 3"), project.id, MANUALLY_UPLOADED); Persistence.feedSources.create(feedSourceWithObsoleteFeedVersion); @@ -84,7 +82,7 @@ static void setUp() throws IOException { } @AfterAll - static void tearDown() throws SQLException, InvalidNamespaceException, CheckedAWSException { + static void tearDown() { Auth0Connection.setAuthDisabled(false); ProcessSingleFeedJob.VALIDATE_MOBILITY_DATA = true; project.delete(); @@ -95,7 +93,7 @@ static void tearDown() throws SQLException, InvalidNamespaceException, CheckedAW try { GTFS.delete(orphanedDBSchema, DataManager.GTFS_DATA_SOURCE); } catch (Exception e) { - // Do nothing. Schema removed in unit test. + // Do nothing. Schema removed in unit test. } } @@ -120,6 +118,13 @@ void canIdentifyOrphanedDBSchemas() { assertTrue(orphanedSchemas.contains(orphanedDBSchema)); } + @Test + void canIdentifyActiveDBSchemas() { + // Other schemas will exist. For this test, just make sure the list contains the feed source editor namespace. + Set activeDBSchemas = DataSanitizer.getActiveDBSchemas(); + assertTrue(activeDBSchemas.contains(feedSourceParent.editorNamespace)); + } + @Test void canRemoveOrphanedDBSchema() { assertEquals(1, DataSanitizer.deleteOrphanedDBSchemas(Set.of(orphanedDBSchema)));