feat: ignore GitOps collations, support TLS (#11)

* docs: document `indexStats` (#3)

* feat: support TLS (#6)

* docs: improve docs

* feat: support TLS connections

* docs: document TLS options

* feat(options): add `tlsCertificateKeyFilePassword` flag (#8)

* feat(options): add `tlsCertificateKeyFilePassword` flag

* ci: trigger ci

* added logic to ingnore default collations

* removed catch

* added specs

* added includeCollation flag

* pull request requested fixes

* marked includeCollations as param in docs

* fix docs

* ci: trigger ci

---------

Co-authored-by: alinashklyar <alina.shklyar@codefresh.io>

Author: masontikhonov

.gitignore

@@ -132,3 +132,7 @@ dist
 !indexes/**
 
 .DS_Store
+
+#Webstorm
+.idea/
+

package.json

@@ -1,6 +1,6 @@
 {
   "name": "index-alignment",
-  "version": "0.2.0-2.8",
+  "version": "0.3.0-2.8",
   "main": "dist/index.js",
   "private": true,
   "type": "module",

src/compare-dump.ts

@@ -7,22 +7,22 @@ import { readDump } from './read-dump.js';
 import type { CollectionDiff, CollectionIndexes, CompareOptions, DatabaseDiff, DatabaseIndexes, DbMapRaw, FullDiff } from './types.js';
 import { getTargetToDumpDb } from './utils.js';
 
-const compareCollections = (desired: CollectionIndexes, actual?: CollectionIndexes, dbMap?: DbMapRaw): CollectionDiff => {
+const compareCollections = (desired: CollectionIndexes, actual?: CollectionIndexes, dbMap?: DbMapRaw, options?: CompareOptions): CollectionDiff => {
   const dumpDbName = getTargetToDumpDb(dbMap).get(desired.databaseName) ?? desired.databaseName;
 
   const missingIndexes = desired.indexes.filter((desiredIndex) => {
     // Skip indexes that should be ignored in the dump
-    if (shouldIgnoreIndexInDump(dumpDbName, desired.collectionName, desiredIndex)) return false;
+    if (shouldIgnoreIndexInDump(dumpDbName, desired.collectionName, desiredIndex, options)) return false;
 
-    const match = actual?.indexes.find(actualIndex => isIndexEqual(desiredIndex, actualIndex));
+    const match = actual?.indexes.find(actualIndex => isIndexEqual(desiredIndex, actualIndex, options));
     return !match;
   });
 
   const extraIndexes = actual?.indexes.filter((actualIndex) => {
     // Skip indexes that should be ignored in the target
-    if (shouldIgnoreIndexInTarget(dumpDbName, actual.collectionName, actualIndex)) return false;
+    if (shouldIgnoreIndexInTarget(dumpDbName, actual.collectionName, actualIndex, options)) return false;
 
-    const match = desired.indexes.find(desiredIndex => isIndexEqual(desiredIndex, actualIndex));
+    const match = desired.indexes.find(desiredIndex => isIndexEqual(desiredIndex, actualIndex, options));
     return !match;
   });
 
@@ -33,12 +33,12 @@ const compareCollections = (desired: CollectionIndexes, actual?: CollectionIndex
   };
 };
 
-const compareDatabases = (desired: DatabaseIndexes, actual?: DatabaseIndexes, dbMap?: DbMapRaw): DatabaseDiff => {
+const compareDatabases = (desired: DatabaseIndexes, actual?: DatabaseIndexes, dbMap?: DbMapRaw, options?: CompareOptions): DatabaseDiff => {
   const dbDiff: DatabaseDiff = { databaseName: desired.databaseName, collections: {} };
 
   for (const desiredCol of desired.collections) {
     const actualCol = actual?.collections.find(actuaCol => actuaCol.collectionName === desiredCol.collectionName);
-    const collectionResult = compareCollections(desiredCol, actualCol, dbMap);
+    const collectionResult = compareCollections(desiredCol, actualCol, dbMap, options);
     if (collectionResult.missingIndexes || collectionResult.extraIndexes) {
       dbDiff.collections[desiredCol.collectionName] = collectionResult;
     }
@@ -56,7 +56,7 @@ export const compareDump = async (options: CompareOptions): Promise<FullDiff> =>
   const diff: FullDiff = { databases: {} };
   for (const desiredDb of desired) {
     const actualDb = actual.find(db => db.databaseName === desiredDb.databaseName);
-    const dbDiff = compareDatabases(desiredDb, actualDb, dbMap);
+    const dbDiff = compareDatabases(desiredDb, actualDb, dbMap, options);
     if (Object.keys(dbDiff.collections).length !== 0) {
       diff.databases[desiredDb.databaseName] = dbDiff;
     }

src/index.ts

@@ -94,6 +94,7 @@ program
     'Map the databases in the dump with the target databases. We have our own naming convention for the production databases, but it is up to the customers to name their databases',
     defaultDbMap,
   )
+  .option('--compareGitopsCollations', 'Compare collations for GitOps product. Only takes effect if --product=gitops', false)
   .action(compare);
 
 program

src/is-index-equal.spec.ts

@@ -102,4 +102,96 @@ describe('isIndexEqual', () => {
       expect(isIndexEqual(input.a, input.b)).toBe(false);
     });
   });
+
+  it('should return true when keys match and collation is default (for gitops)', () => {
+    const indexA = {
+      key: { a: 1 },
+      collation: {
+        locale: 'en_US',
+        caseLevel: false,
+        caseFirst: 'off',
+        strength: 1,
+        numericOrdering: false,
+        alternate: 'non-ignorable',
+        maxVariable: 'punct',
+        normalization: false,
+        backwards: false,
+      },
+    };
+
+    const indexB = {
+      key: { a: 1 },
+    };
+
+    expect(isIndexEqual(indexA, indexB, { product: 'gitops', uri: '' })).toBe(true);
+  });
+
+  it('should return false when keys match and collation is default (for classic)', () => {
+    const indexA = {
+      key: { a: 1 },
+      collation: {
+        locale: 'en_US',
+        caseLevel: false,
+        caseFirst: 'off',
+        strength: 1,
+        numericOrdering: false,
+        alternate: 'non-ignorable',
+        maxVariable: 'punct',
+        normalization: false,
+        backwards: false,
+      },
+    };
+
+    const indexB = {
+      key: { a: 1 },
+    };
+
+    expect(isIndexEqual(indexA, indexB, { product: 'classic', uri: '' })).toBe(false);
+  });
+
+  it('should return false when keys match and collation is default but compareGitopsCollations flag is true', () => {
+    const indexA = {
+      key: { a: 1 },
+      collation: {
+        locale: 'en_US',
+        caseLevel: false,
+        caseFirst: 'off',
+        strength: 1,
+        numericOrdering: false,
+        alternate: 'non-ignorable',
+        maxVariable: 'punct',
+        normalization: false,
+        backwards: false,
+      },
+    };
+
+    const indexB = {
+      key: { a: 1 },
+    };
+
+    expect(isIndexEqual(indexA, indexB, { product: 'gitops', uri: '', compareGitopsCollations: true })).toBe(false);
+  });
+
+  it('should return false when keys match but collation is not default', () => {
+    const indexA = {
+      key: { a: 1 },
+      collation: {
+        locale: 'en_US',
+        caseLevel: false,
+        caseFirst: 'off',
+        strength: 2,
+        numericOrdering: false,
+        alternate: 'non-ignorable',
+        maxVariable: 'punct',
+        normalization: false,
+        backwards: false,
+      },
+    };
+
+    const indexB = {
+      key: { a: 1 },
+    };
+
+    expect(isIndexEqual(indexA, indexB, { product: 'gitops', uri: '' })).toBe(false);
+  });
 });

src/is-index-equal.ts

@@ -1,5 +1,28 @@
 import { deepStrictEqual } from 'node:assert';
-import type { Index } from './types.js';
+import type { CompareOptions, Index } from './types.js';
+import { CollationOptions, Document } from 'mongodb';
+
+/**
+ * Default collation specified for collections in GitOps product
+ * (https://github.com/codefresh-io/argo-platform/blob/aa831b539c8434156c323db881ee7d44db87ac13/libs/db/src/helpers/helpers.ts#L81),
+ * extended with implicit collation options.
+ */
+const defaultCollation: CollationOptions = {
+  locale: 'en_US',
+  caseLevel: false,
+  caseFirst: 'off',
+  strength: 1,
+  numericOrdering: false,
+  alternate: 'non-ignorable',
+  maxVariable: 'punct',
+  normalization: false,
+  backwards: false,
+};
+
+function isDefaultCollation(collation: Document | undefined): boolean {
+  if (!collation) return false;
+  return JSON.stringify(collation) === JSON.stringify(defaultCollation);
+}
 
 /**
  * Compare two indexes to check if they are equal.
@@ -19,13 +42,23 @@ import type { Index } from './types.js';
  * that's why we use `deepStrictEqual` ({@link https://nodejs.org/api/assert.html#assertdeepstrictequalactual-expected-message|docs}).
  *
  * - If both `key` and options (except for `name`) are equal, the indexes are considered equal.
+ *
+ * `options.compareGitopsCollations` parameter allows to include collation options in the comparison for gitops.
+ * This check is temporary disabled by default due to a misalignment between production and on-prem environments.
+ * ({@link https://codefresh-io.atlassian.net/browse/CR-29948})
  */
-export const isIndexEqual = (a: Index, b: Index): boolean => {
+export const isIndexEqual = (a: Index, b: Index, options?: CompareOptions): boolean => {
   const aKey = a.key;
   const aOptions = { ...a, key: undefined, name: undefined };
   const bKey = b.key;
   const bOptions = { ...b, key: undefined, name: undefined };
   const isKeyEqual = JSON.stringify(aKey) === JSON.stringify(bKey);
+
+  if (options?.product === 'gitops' && !options?.compareGitopsCollations) {
+    if (isDefaultCollation(aOptions.collation)) delete aOptions.collation;
+    if (isDefaultCollation(bOptions.collation)) delete bOptions.collation;
+  }
+
   try {
     deepStrictEqual(aOptions, bOptions);
     return isKeyEqual;

src/overrides/should-ignore-index-in-dump.ts

@@ -1,5 +1,12 @@
 import { isIndexEqual } from '../is-index-equal.js';
-import type { CollectionName, DatabaseName, IgnoreInAllCollections, IgnoreList, Index } from '../types.js';
+import type {
+  CollectionName,
+  CompareOptions,
+  DatabaseName,
+  IgnoreInAllCollections,
+  IgnoreList,
+  Index,
+} from '../types.js';
 
 // TODO: Verify unique indexes, they should probably be ignored for now.
 
@@ -212,9 +219,9 @@ const ignoreList: IgnoreList = {
   },
 };
 
-export const shouldIgnoreIndexInDump = (dumpDbName: DatabaseName, collectionName: CollectionName, dumpIndex: Index): boolean => {
+export const shouldIgnoreIndexInDump = (dumpDbName: DatabaseName, collectionName: CollectionName, dumpIndex: Index, options?: CompareOptions): boolean => {
   // Check if the index should be ignored in all collections
-  if (ignoreInAllCollections.some(ignore => isIndexEqual(ignore, dumpIndex))) {
+  if (ignoreInAllCollections.some(ignore => isIndexEqual(ignore, dumpIndex, options))) {
     return true;
   }
 
@@ -226,7 +233,7 @@ export const shouldIgnoreIndexInDump = (dumpDbName: DatabaseName, collectionName
   }
 
   // Check if the index is in the ignore list for the specific collection
-  if (ignoreCollection?.indexes.some(ignore => isIndexEqual(ignore, dumpIndex))) {
+  if (ignoreCollection?.indexes.some(ignore => isIndexEqual(ignore, dumpIndex, options))) {
     return true;
   }
 
@@ -235,7 +242,7 @@ export const shouldIgnoreIndexInDump = (dumpDbName: DatabaseName, collectionName
     ...dumpIndex,
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     expireAfterSeconds: 'ANY' as any,
-  }))) {
+  }, options))) {
     return true;
   }
 

src/overrides/should-ignore-index-in-target.ts

@@ -1,5 +1,12 @@
 import { isIndexEqual } from '../is-index-equal.js';
-import type { CollectionName, DatabaseName, IgnoreInAllCollections, IgnoreList, Index } from '../types.js';
+import type {
+  CollectionName,
+  CompareOptions,
+  DatabaseName,
+  IgnoreInAllCollections,
+  IgnoreList,
+  Index,
+} from '../types.js';
 
 // TODO: Verify unique indexes, they should probably be ignored for now.
 
@@ -55,9 +62,9 @@ const ignoreList: IgnoreList = {
   },
 };
 
-export const shouldIgnoreIndexInTarget = (dumpDbName: DatabaseName, collectionName: CollectionName, targetIndex: Index): boolean => {
+export const shouldIgnoreIndexInTarget = (dumpDbName: DatabaseName, collectionName: CollectionName, targetIndex: Index, options?: CompareOptions): boolean => {
   // Check if the index should be ignored in all collections
-  if (ignoreInAllCollections.some(ignore => isIndexEqual(ignore, targetIndex))) {
+  if (ignoreInAllCollections.some(ignore => isIndexEqual(ignore, targetIndex, options))) {
     return true;
   }
 
@@ -69,7 +76,7 @@ export const shouldIgnoreIndexInTarget = (dumpDbName: DatabaseName, collectionNa
   }
 
   // Check if the index is in the ignore list for the specific collection
-  if (ignoreCollection?.indexes.some(ignore => isIndexEqual(ignore, targetIndex))) {
+  if (ignoreCollection?.indexes.some(ignore => isIndexEqual(ignore, targetIndex, options))) {
     return true;
   }
 
@@ -78,7 +85,7 @@ export const shouldIgnoreIndexInTarget = (dumpDbName: DatabaseName, collectionNa
     ...targetIndex,
     // eslint-disable-next-line @typescript-eslint/no-explicit-any
     expireAfterSeconds: 'ANY' as any,
-  }))) {
+  }, options))) {
     return true;
   }
 

src/types.ts

@@ -72,6 +72,7 @@ export interface CompareOptions extends Partial<MongoClientOptions> {
   uri: string;
   product: Product;
   dbMap?: DbMapRaw;
+  compareGitopsCollations?: boolean;
 }
 
 export interface StatsOptions extends Partial<MongoClientOptions> {