H-3572, H-4105: Track Node migrations run, skip already-run migrations

README.md

@@ -177,19 +177,10 @@ If you need to run the browser plugin locally, see the `README.md` in the `apps/
 
 ##### Resetting the local database
 
-If you need to reset the local database, to clear out test data or because it has become corrupted during development, you have two options:
+If you need to reset the local database, to clear out test data or because it has become corrupted during development:
 
-1. The slow option – rebuild in Docker
-
-   1. In the Docker UI (or via CLI at your preference), stop and delete the `hash-external-services` container
-   2. In 'Volumes', search 'hash-external-services' and delete the volumes shown
-   3. Run `yarn external-services up --wait` to rebuild the services
-
-2. The fast option – reset the database via the Graph API
-
-   1. Run the Graph API in test mode by running `yarn dev:graph:test-server`
-   2. Run `yarn graph:reset-database` to reset the database
-   3. **If you need to use the frontend**, you will also need to delete the rows in the `identities` table in the `dev_kratos` database, or signin will not work. You can do so via any Postgres UI or CLI. The db connection and user details are in `.env`
+1. Run `yarn external-services down -v` (this will take the Docker services down and drop the volumes)
+2. Run `yarn external-services up --wait` to start everything again
 
 ##### External services test mode
 

apps/hash-api/src/graph/ensure-system-graph-is-initialized/migrate-ontology-types.ts

@@ -2,7 +2,21 @@ import { readdir } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";
 
+import type { ProvidedEntityEditionProvenance } from "@blockprotocol/type-system";
+import { componentsFromVersionedUrl } from "@blockprotocol/type-system";
+import { getHashInstance } from "@local/hash-backend-utils/hash-instance";
 import type { Logger } from "@local/hash-backend-utils/logger";
+import { queryDataTypes } from "@local/hash-graph-sdk/data-type";
+import { queryEntityTypes } from "@local/hash-graph-sdk/entity-type";
+import { queryPropertyTypes } from "@local/hash-graph-sdk/property-type";
+import { currentTimeInstantTemporalAxes } from "@local/hash-isomorphic-utils/graph-queries";
+import {
+  systemDataTypes,
+  systemEntityTypes,
+  systemLinkEntityTypes,
+  systemPropertyTypes,
+} from "@local/hash-isomorphic-utils/ontology-type-ids";
+import type { MigrationsCompletedPropertyValueWithMetadata } from "@local/hash-isomorphic-utils/system-types/hashinstance";
 
 import { isProdEnv } from "../../lib/env-config";
 import type { ImpureGraphContext } from "../context-types";
@@ -39,6 +53,144 @@ export const migrateOntologyTypes = async (params: {
     dataTypeVersions: {},
   };
 
+  /**
+   * `migrationState` is used as a cache for "current" ontology type versions while applying
+   * migrations. Historically this cache was only populated by the migrations that create/update
+   * ontology types. When migrations are skipped on an existing instance, the cache would be empty
+   * and later migrations that rely on `getCurrentHashSystemEntityTypeId` (etc.) would fail.
+   *
+   * To make migrations idempotent across fresh installs and existing deployments, we hydrate the
+   * cache from the graph before running any migration functions.
+   */
+  const hydrateMigrationStateFromGraph = async () => {
+    const entityTypeBaseUrls = [
+      ...Object.values(systemEntityTypes).map(({ entityTypeBaseUrl }) =>
+        entityTypeBaseUrl,
+      ),
+      ...Object.values(systemLinkEntityTypes).map(({ linkEntityTypeBaseUrl }) =>
+        linkEntityTypeBaseUrl,
+      ),
+    ];
+
+    const propertyTypeBaseUrls = Object.values(systemPropertyTypes).map(
+      ({ propertyTypeBaseUrl }) => propertyTypeBaseUrl,
+    );
+
+    const dataTypeBaseUrls = Object.values(systemDataTypes).map(
+      ({ dataTypeBaseUrl }) => dataTypeBaseUrl,
+    );
+
+    await Promise.all([
+      ...entityTypeBaseUrls.map(async (baseUrl) => {
+        if (migrationState.entityTypeVersions[baseUrl]) {
+          return;
+        }
+
+        const { entityTypes } = await queryEntityTypes(
+          params.context.graphApi,
+          authentication,
+          {
+            filter: {
+              all: [
+                {
+                  equal: [{ path: ["baseUrl"] }, { parameter: baseUrl }],
+                },
+                {
+                  equal: [{ path: ["version"] }, { parameter: "latest" }],
+                },
+              ],
+            },
+            temporalAxes: currentTimeInstantTemporalAxes,
+            limit: 1,
+          },
+        );
+
+        const existing = entityTypes[0];
+        if (!existing) {
+          return;
+        }
+
+        const { version } = componentsFromVersionedUrl(existing.schema.$id);
+        migrationState.entityTypeVersions[baseUrl] = version;
+      }),
+      ...propertyTypeBaseUrls.map(async (baseUrl) => {
+        if (migrationState.propertyTypeVersions[baseUrl]) {
+          return;
+        }
+
+        const { propertyTypes } = await queryPropertyTypes(
+          params.context.graphApi,
+          authentication,
+          {
+            filter: {
+              all: [
+                {
+                  equal: [{ path: ["baseUrl"] }, { parameter: baseUrl }],
+                },
+                {
+                  equal: [{ path: ["version"] }, { parameter: "latest" }],
+                },
+              ],
+            },
+            temporalAxes: currentTimeInstantTemporalAxes,
+            limit: 1,
+          },
+        );
+
+        const existing = propertyTypes[0];
+        if (!existing) {
+          return;
+        }
+
+        const { version } = componentsFromVersionedUrl(existing.schema.$id);
+        migrationState.propertyTypeVersions[baseUrl] = version;
+      }),
+      ...dataTypeBaseUrls.map(async (baseUrl) => {
+        if (migrationState.dataTypeVersions[baseUrl]) {
+          return;
+        }
+
+        const { dataTypes } = await queryDataTypes(
+          params.context.graphApi,
+          authentication,
+          {
+            filter: {
+              all: [
+                {
+                  equal: [{ path: ["baseUrl"] }, { parameter: baseUrl }],
+                },
+                {
+                  equal: [{ path: ["version"] }, { parameter: "latest" }],
+                },
+              ],
+            },
+            temporalAxes: currentTimeInstantTemporalAxes,
+            limit: 1,
+          },
+        );
+
+        const existing = dataTypes[0];
+        if (!existing) {
+          return;
+        }
+
+        const { version } = componentsFromVersionedUrl(existing.schema.$id);
+        migrationState.dataTypeVersions[baseUrl] = version;
+      }),
+    ]);
+  };
+
+  const migrationsCompleted: string[] = [];
+
+  try {
+    const hashInstance = await getHashInstance(params.context, authentication);
+    migrationsCompleted.push(...(hashInstance.migrationsCompleted ?? []));
+  } catch {
+    // HASH Instance entity not available, this may be the first time the instance is being initialized
+  }
+
+  await hydrateMigrationStateFromGraph();
+
   for (const migrationFileName of migrationFileNames) {
     if (migrationFileName.endsWith(".migration.ts")) {
       /**
@@ -57,15 +209,56 @@ export const migrateOntologyTypes = async (params: {
       // Expect the default export of a migration file to be of type `MigrationFunction`
       const migrationFunction = module.default as MigrationFunction;
 
-      /** @todo: consider persisting which migration files have been run */
+      const migrationNumber = migrationFileName.split("-")[0];
+
+      if (!migrationNumber) {
+        throw new Error(
+          `Migration file ${migrationFileName} has an invalid name. Migration files must be formatted as '{number}-{name}.migration.ts'`,
+        );
+      }
+
+      if (migrationsCompleted.includes(migrationNumber)) {
+        params.logger.info(
+          `Skipping migration ${migrationFileName} as it has already been processed`,
+        );
+        continue;
+      }
 
       migrationState = await migrationFunction({
         ...params,
         authentication,
         migrationState,
       });
 
-      params.logger.debug(`Processed migration ${migrationFileName}`);
+      migrationsCompleted.push(migrationNumber);
+
+      params.logger.info(`Processed migration ${migrationFileName}`);
     }
   }
+
+  const hashInstance = await getHashInstance(params.context, authentication);
+
+  await hashInstance.entity.patch(params.context.graphApi, authentication, {
+    propertyPatches: [
+      {
+        op: "add",
+        path: [systemPropertyTypes.migrationsCompleted.propertyTypeBaseUrl],
+        property: {
+          value: migrationsCompleted.map((migration) => ({
+            value: migration,
+            metadata: {
+              dataTypeId:
+                "https://blockprotocol.org/@blockprotocol/types/data-type/text/v/1",
+            },
+          })),
+        } satisfies MigrationsCompletedPropertyValueWithMetadata,
+      },
+    ],
+    provenance: {
+      actorType: "machine",
+      origin: {
+        type: "migration",
+      },
+    } satisfies ProvidedEntityEditionProvenance,
+  });
 };

apps/hash-api/src/graph/ensure-system-graph-is-initialized/migrate-ontology-types/migrations/001-create-hash-system-types.migration.ts

@@ -1,4 +1,5 @@
 import { getDataTypeById } from "@local/hash-graph-sdk/data-type";
+import { getEntityTypeById } from "@local/hash-graph-sdk/entity-type";
 import { fullTransactionTimeAxis } from "@local/hash-isomorphic-utils/graph-queries";
 import {
   blockProtocolEntityTypes,
@@ -25,6 +26,11 @@ const blockProtocolDataTypeIds = [
   "https://blockprotocol.org/@blockprotocol/types/data-type/value/v/1",
 ] as const;
 
+const blockProtocolEntityTypeIds = [
+  "https://blockprotocol.org/@hash/types/entity-type/query/v/1",
+  "https://blockprotocol.org/@hash/types/entity-type/has-query/v/1",
+] as const;
+
 const migrate: MigrationFunction = async ({
   context,
   authentication,
@@ -1732,8 +1738,8 @@ const migrate: MigrationFunction = async ({
   /**
    * Ensure the primitive BP data types are loaded
    */
-  await Promise.all(
-    blockProtocolDataTypeIds.map(async (dataTypeId) => {
+  await Promise.all([
+    ...blockProtocolDataTypeIds.map(async (dataTypeId) => {
       const existingDataType = await getDataTypeById(
         context.graphApi,
         authentication,
@@ -1751,7 +1757,25 @@ const migrate: MigrationFunction = async ({
         dataTypeId,
       });
     }),
-  );
+    ...blockProtocolEntityTypeIds.map(async (entityTypeId) => {
+      const existingEntityType = await getEntityTypeById(
+        context.graphApi,
+        authentication,
+        {
+          entityTypeId,
+          temporalAxes: fullTransactionTimeAxis,
+        },
+      );
+
+      if (existingEntityType) {
+        return;
+      }
+
+      return context.graphApi.loadExternalEntityType(authentication.actorId, {
+        entityTypeId,
+      });
+    }),
+  ]);
 
   return migrationState;
 };

apps/hash-api/src/graph/ensure-system-graph-is-initialized/migrate-ontology-types/migrations/005-create-hash-system-entities-and-web-bots.migration.ts

@@ -45,6 +45,11 @@ const migrate: MigrationFunction = async ({
     migrationState,
   });
 
+  const currentHashInstanceEntityTypeId = getCurrentHashSystemEntityTypeId({
+    entityTypeKey: "hashInstance",
+    migrationState,
+  });
+
   /**
    * Step 1: Create the system entities associated with the 'hash' web:
    * 1. The HASH org entity is required to create the HASH Instance entity in Step 2
@@ -68,7 +73,9 @@ const migrate: MigrationFunction = async ({
     await getHashInstance(context, systemAccountAuthentication);
   } catch (error) {
     if (error instanceof NotFoundError) {
-      await createHashInstance(context, systemAccountAuthentication, {});
+      await createHashInstance(context, systemAccountAuthentication, {
+        hashInstanceEntityTypeId: currentHashInstanceEntityTypeId,
+      });
       logger.info("Created hashInstance entity");
     } else {
       throw error;