Merge pull request #404 from DataObjects-NET/7.1-pgsql-schema-extraction-imp

Improves schema extraction for PostgreSQL

Author: alex-kulakov

ChangeLog/7.1.3_dev.txt

@@ -1 +1,2 @@
-[main] Addressed race condition issue with TranslatorState.NonVisitableExpressions
+[main] Addressed race condition issue with TranslatorState.NonVisitableExpressions
+[postgresql] Improved database structucture extraction

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/DriverFactory.cs

@@ -111,19 +111,15 @@ private static SqlDriver CreateDriverInstance(
 
       // We support 8.3, 8.4 and any 9.0+
 
-      if (version.Major == 8) {
-        return version.Minor == 3 ? new v8_3.Driver(coreServerInfo) : new v8_4.Driver(coreServerInfo);
-      }
-
-      if (version.Major == 9) {
-        return version.Minor == 0 ? new v9_0.Driver(coreServerInfo) : new v9_1.Driver(coreServerInfo);
-      }
-
-      if (version.Major < 12) {
-        return new v10_0.Driver(coreServerInfo);
-      }
-
-      return new v12_0.Driver(coreServerInfo);
+      return version.Major switch {
+        8 when version.Minor == 3 => new v8_3.Driver(coreServerInfo),
+        8 when version.Minor > 3  => new v8_4.Driver(coreServerInfo),
+        9 when version.Minor == 0 => new v9_0.Driver(coreServerInfo),
+        9 when version.Minor > 0 => new v9_1.Driver(coreServerInfo),
+        10 => new v10_0.Driver(coreServerInfo),
+        11 => new v10_0.Driver(coreServerInfo),
+        _ => new v12_0.Driver(coreServerInfo)
+      };
     }
 
     /// <inheritdoc/>

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/Resources/Strings.Designer.cs

@@ -135,4 +135,7 @@
   <data name="ExSchemaXDoesNotExistOrBelongsToAnotherUser" xml:space="preserve">
     <value>Schema '{0}' either does not exist or belongs to another user.</value>
   </data>
+  <data name="ExCantFindSchemaXOwnerWithIdYInTheListOfRoles" xml:space="preserve">
+    <value>Can't find schema '{0}' owner with oid '{1}' in the list of roles.</value>
+  </data>
 </root>

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/Resources/Strings.resx

@@ -26,12 +26,17 @@ protected sealed class ExtractionContext
       /// <summary>
       /// Specific schemas to extract
       /// </summary>
-      public readonly Dictionary<string, Schema> TargetSchemes = new Dictionary<string, Schema>();
+      public readonly Dictionary<string, Schema> TargetSchemes = new();
 
       /// <summary>
-      /// Extracted users.
+      /// Extracted users (subset of <see cref="RoleLookup"/>).
       /// </summary>
-      public readonly Dictionary<long, string> UserLookup = new Dictionary<long, string>();
+      public readonly Dictionary<long, string> UserLookup = new();
+
+      /// <summary>
+      /// Extracted roles.
+      /// </summary>
+      public readonly Dictionary<long, string> RoleLookup = new();
 
       /// <summary>
       /// Catalog to extract information.
@@ -41,46 +46,54 @@ protected sealed class ExtractionContext
       /// <summary>
       /// Extracted schemas.
       /// </summary>
-      public readonly Dictionary<long, Schema> SchemaMap = new Dictionary<long, Schema>();
+      public readonly Dictionary<long, Schema> SchemaMap = new();
 
       /// <summary>
       /// Extracted schemas identifiers.
       /// </summary>
-      public readonly Dictionary<Schema, long> ReversedSchemaMap = new Dictionary<Schema, long>();
+      public readonly Dictionary<Schema, long> ReversedSchemaMap = new();
 
       /// <summary>
       /// Extracted tables.
       /// </summary>
-      public readonly Dictionary<long, Table> TableMap = new Dictionary<long, Table>();
+      public readonly Dictionary<long, Table> TableMap = new();
 
       /// <summary>
       /// Extracted views.
       /// </summary>
-      public readonly Dictionary<long, View> ViewMap = new Dictionary<long, View>();
+      public readonly Dictionary<long, View> ViewMap = new();
 
       /// <summary>
       /// Extracted sequences.
       /// </summary>
-      public readonly Dictionary<long, Sequence> SequenceMap = new Dictionary<long, Sequence>();
+      public readonly Dictionary<long, Sequence> SequenceMap = new();
 
       /// <summary>
       /// Extracted index expressions.
       /// </summary>
-      public readonly Dictionary<long, ExpressionIndexInfo> ExpressionIndexMap = new Dictionary<long, ExpressionIndexInfo>();
+      public readonly Dictionary<long, ExpressionIndexInfo> ExpressionIndexMap = new();
 
       /// <summary>
       /// Extracted domains.
       /// </summary>
-      public readonly Dictionary<long, Domain> DomainMap = new Dictionary<long, Domain>();
+      public readonly Dictionary<long, Domain> DomainMap = new();
 
       /// <summary>
       /// Extracted columns connected grouped by owner (table or view)
       /// </summary>
-      public readonly Dictionary<long, Dictionary<long, TableColumn>> TableColumnMap = new Dictionary<long, Dictionary<long, TableColumn>>();
+      public readonly Dictionary<long, Dictionary<long, TableColumn>> TableColumnMap = new();
 
+      /// <summary>
+      /// Roles in which current user is a member, self included.
+      /// </summary>
+      public readonly List<long> CurrentUserRoles = new();
+
+      public string CurrentUserName { get; set; }
       public long CurrentUserSysId { get; set; } = -1;
       public long? CurrentUserIdentifier { get; set; }
 
+      public bool IsMe(string name) => name == CurrentUserName;
+
       public ExtractionContext(Catalog catalog)
       {
         Catalog = catalog;
@@ -332,7 +345,7 @@ public override Catalog ExtractSchemes(string catalogName, string[] schemaNames)
     {
       var (catalog, context) = CreateCatalogAndContext(catalogName, schemaNames);
 
-      ExtractUsers(context);
+      _ = ExtractRoles(context, false);
       ExtractSchemas(context);
       return catalog;
     }
@@ -343,7 +356,7 @@ public override async Task<Catalog> ExtractSchemesAsync(
     {
       var (catalog, context) = CreateCatalogAndContext(catalogName, schemaNames);
 
-      await ExtractUsersAsync(context, token).ConfigureAwait(false);
+      await ExtractRoles(context, true, token).ConfigureAwait(false);
       await ExtractSchemasAsync(context, token).ConfigureAwait(false);
       return catalog;
     }
@@ -360,48 +373,62 @@ private static (Catalog catalog, ExtractionContext context) CreateCatalogAndCont
       return (catalog, context);
     }
 
-    private void ExtractUsers(ExtractionContext context)
+    private async ValueTask ExtractRoles(ExtractionContext context, bool isAsync, CancellationToken token = default)
     {
       context.UserLookup.Clear();
-      string me;
-      using (var command = Connection.CreateCommand("SELECT user")) {
-        me = (string) command.ExecuteScalar();
-      }
 
-      using (var cmd = Connection.CreateCommand("SELECT usename, usesysid FROM pg_user"))
-      using (var dr = cmd.ExecuteReader()) {
-        while (dr.Read()) {
-          ReadUserData(dr, context, me);
+      var extractCurentUserCommand = Connection.CreateCommand("SELECT user");
+      // Roles include users.
+      // Users also can have members for some reason and it doesn't make them groups,
+      // the only thing that defines user is ability to login :-)
+      const string ExtractRolesQueryTemplate = "SELECT rolname, oid, rolcanlogin, pg_has_role('{0}', oid,'member') FROM pg_roles";
+      
+
+      if (isAsync) {
+        await using (extractCurentUserCommand.ConfigureAwait(false)) {
+          context.CurrentUserName = (string) await extractCurentUserCommand.ExecuteScalarAsync(token).ConfigureAwait(false);
         }
-      }
-    }
 
-    private async Task ExtractUsersAsync(ExtractionContext context, CancellationToken token = default)
-    {
-      context.UserLookup.Clear();
-      string me;
-      var command = Connection.CreateCommand("SELECT user");
-      await using (command.ConfigureAwait(false)) {
-        me = (string) await command.ExecuteScalarAsync(token).ConfigureAwait(false);
+        var getAllUsersCommand = Connection.CreateCommand(string.Format(ExtractRolesQueryTemplate, context.CurrentUserName));
+        await using (getAllUsersCommand.ConfigureAwait(false)) {
+          var reader = await getAllUsersCommand.ExecuteReaderAsync(token).ConfigureAwait(false);
+          await using (reader.ConfigureAwait(false)) {
+            while (await reader.ReadAsync(token).ConfigureAwait(false)) {
+              ReadUserData(reader, context);
+            }
+          }
+        }
       }
+      else {
+        using (extractCurentUserCommand) {
+          context.CurrentUserName = (string) extractCurentUserCommand.ExecuteScalar();
+        }
 
-      command = Connection.CreateCommand("SELECT usename, usesysid FROM pg_user");
-      await using (command.ConfigureAwait(false)) {
-        var reader = await command.ExecuteReaderAsync(token).ConfigureAwait(false);
-        await using (reader.ConfigureAwait(false)) {
-          while (await reader.ReadAsync(token).ConfigureAwait(false)) {
-            ReadUserData(reader, context, me);
+        var getAllUsersCommand = Connection.CreateCommand(string.Format(ExtractRolesQueryTemplate, context.CurrentUserName));
+        using (getAllUsersCommand)
+        using (var dr = getAllUsersCommand.ExecuteReader()) {
+          while (dr.Read()) {
+            ReadUserData(dr, context);
           }
         }
       }
     }
 
-    private static void ReadUserData(DbDataReader dr, ExtractionContext context, string me)
+    private static void ReadUserData(DbDataReader dr, ExtractionContext context)
     {
-      var name = dr[0].ToString();
+      var name = dr.GetString(0);
+      // oid, which is basically a number, has its own type - oid! can't be read as int or long (facepalm)
       var sysId = Convert.ToInt64(dr[1]);
-      context.UserLookup.Add(sysId, name);
-      if (name == me) {
+      var canLogin = dr.GetBoolean(2);
+      var containsCurrentUser = dr.GetBoolean(3);
+      context.RoleLookup.Add(sysId, name);
+      if(containsCurrentUser) {
+        context.CurrentUserRoles.Add(sysId);
+      }
+      if (canLogin) {
+        context.UserLookup.Add(sysId, name);
+      }
+      if (context.IsMe(name)) {
         context.CurrentUserSysId = sysId;
       }
     }
@@ -499,7 +526,11 @@ protected virtual SqlQueryExpression BuildExtractSchemasQuery(ExtractionContext
       selectPublic.Columns.Add(namespaceTable1["nspowner"]);
 
       var selectMine = SqlDml.Select(namespaceTable2);
-      selectMine.Where = namespaceTable2["nspowner"] == context.CurrentUserIdentifier;
+      if (context.CurrentUserRoles.Count == 0)
+        selectMine.Where = namespaceTable2["nspowner"] == context.CurrentUserIdentifier;
+      else {
+        selectMine.Where = SqlDml.In(namespaceTable2["nspowner"], SqlDml.Array(context.CurrentUserRoles.ToArray()));
+      }
       selectMine.Columns.Add(namespaceTable2["nspname"]);
       selectMine.Columns.Add(namespaceTable2["oid"]);
       selectMine.Columns.Add(namespaceTable2["nspowner"]);
@@ -522,7 +553,12 @@ protected virtual void ReadSchemaData(DbDataReader dataReader, ExtractionContext
         catalog.DefaultSchema = schema;
       }
 
-      schema.Owner = context.UserLookup[owner];
+      if (context.RoleLookup.TryGetValue(owner, out var userName)) {
+        schema.Owner = userName;
+      }
+      else {
+        throw new InvalidOperationException(string.Format(Resources.Strings.ExCantFindSchemaXOwnerWithIdYInTheListOfRoles, name, owner));
+      }
       context.SchemaMap[oid] = schema;
       context.ReversedSchemaMap[schema] = oid;
     }
@@ -594,8 +630,8 @@ protected virtual ISqlCompileUnit BuildExtractSchemaContentsQuery(ExtractionCont
       select.Columns.Add(relationsTable["relnamespace"]);
       select.Columns.Add(tablespacesTable["spcname"]);
       select.Columns.Add(new Func<SqlCase>(() => {
-        var defCase = SqlDml.Case(relationsTable["relkind"]);
-        defCase.Add('v', SqlDml.FunctionCall("pg_get_viewdef", relationsTable["oid"]));
+        var defCase = SqlDml.Case(relationsTable["relkind"])
+          .Add('v', SqlDml.FunctionCall("pg_get_viewdef", relationsTable["oid"]));
         return defCase;
       })(), "definition");
       return select;
@@ -741,7 +777,7 @@ protected virtual void ReadColumnData(DbDataReader dataReader, ExtractionContext
       }
       else {
         var view = viewMap[columnOwnerId];
-        view.CreateColumn(columnName);
+        _ = view.CreateColumn(columnName);
       }
     }
 
@@ -912,8 +948,9 @@ protected virtual int ReadTableIndexData(DbDataReader dataReader, ExtractionCont
         else {
           for (int j = 0; j < indexKey.Length; j++) {
             int colIndex = indexKey[j];
-            if (colIndex > 0)
-              index.CreateIndexColumn(tableColumns[tableIdentifier][colIndex], true);
+            if (colIndex > 0) {
+              _ = index.CreateIndexColumn(tableColumns[tableIdentifier][colIndex], true);
+            }
             else {
               //column index is 0
               //this means that this index column is an expression
@@ -967,12 +1004,9 @@ protected virtual void ReadIndexColumnsData(DbDataReader dataReader, ExtractionC
       var exprIndexInfo = expressionIndexMap[Convert.ToInt64(dataReader[1])];
       for (var j = 0; j < exprIndexInfo.Columns.Length; j++) {
         int colIndex = exprIndexInfo.Columns[j];
-        if (colIndex > 0) {
-          exprIndexInfo.Index.CreateIndexColumn(tableColumns[Convert.ToInt64(dataReader[0])][colIndex], true);
-        }
-        else {
-          exprIndexInfo.Index.CreateIndexColumn(SqlDml.Native(dataReader[(j + 1).ToString()].ToString()));
-        }
+        _ = colIndex > 0
+          ? exprIndexInfo.Index.CreateIndexColumn(tableColumns[Convert.ToInt64(dataReader[0])][colIndex], true)
+          : exprIndexInfo.Index.CreateIndexColumn(SqlDml.Native(dataReader[(j + 1).ToString()].ToString()));
       }
     }
 

Orm/Xtensive.Orm.PostgreSql/Sql.Drivers.PostgreSql/v8_0/Extractor.cs

@@ -72,6 +72,9 @@
     <domain name="pgsql150"
             connectionUrl="postgresql://dotest:dotest@localhost:54150/dotest" />
 
+    <domain name="pgsql160"
+            connectionUrl="postgresql://dotest:dotest@localhost:54160/dotest" />
+
     <domain name="oracle10"
             connectionUrl="oracle://dotest:dotest@localhost:5510/ora10" />
 
@@ -167,6 +170,9 @@
     <domain name="pgsql150cs" provider="postgresql"
             connectionString="HOST=localhost;PORT=54150;DATABASE=dotest;USER ID=dotest;PASSWORD=dotest" />
 
+    <domain name="pgsql160cs" provider="postgresql"
+            connectionString="HOST=localhost;PORT=54160;DATABASE=dotest;USER ID=dotest;PASSWORD=dotest" />
+
     <domain name="oracle10cs" provider="oracle"
             connectionString="DATA SOURCE=&quot;(DESCRIPTION=(ADDRESS=(PROTOCOL=TCP)(HOST=localhost)(PORT=5510))(CONNECT_DATA=(SERVICE_NAME=ora10)))&quot;;USER ID=dotest;PASSWORD=dotest" />