From 80e9abaea8a4470ef8f3fcd76d764413331c53fe Mon Sep 17 00:00:00 2001 From: Haik Date: Thu, 30 May 2024 01:24:40 +0400 Subject: [PATCH 1/4] something --- .../EFCore.PostgresExtensions.csproj | 2 +- test/PandaNuGet.Demo/PandaNuGet.Demo.csproj | 6 +++--- test/PandaNuGet.Tests/PandaNuGet.Tests.csproj | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj b/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj index f19c6b5..ca78c93 100644 --- a/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj +++ b/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj @@ -23,7 +23,7 @@ - + diff --git a/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj b/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj index 2bc0e0f..2a21756 100644 --- a/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj +++ b/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj @@ -9,9 +9,9 @@ - - - + + + diff --git a/test/PandaNuGet.Tests/PandaNuGet.Tests.csproj b/test/PandaNuGet.Tests/PandaNuGet.Tests.csproj index 299c1f9..7107c1c 100644 --- a/test/PandaNuGet.Tests/PandaNuGet.Tests.csproj +++ b/test/PandaNuGet.Tests/PandaNuGet.Tests.csproj @@ -10,9 +10,9 @@ - - - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all From 0a07733bdc9ccd721274c137e1bcaa887b1c8552 Mon Sep 17 00:00:00 2001 From: Haik Date: Thu, 30 May 2024 01:42:08 +0400 Subject: [PATCH 2/4] nuget updates --- .../Extensions/QueryableExtensions.cs | 47 ++++++++++++++++++- test/PandaNuGet.Demo/PandaNuGet.Demo.csproj | 2 +- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs b/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs index e8fa218..500bec1 100644 --- a/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs +++ b/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs @@ -6,6 +6,7 @@ namespace EFCore.PostgresExtensions.Extensions public static class QueryableExtensions { internal const string ForUpdateKey = "for update "; + /// /// Use this method for selecting data with locking. /// Attention! Be aware that this method works only inside the transaction scope(dbContext.BeginTransaction) and you need to register it in startup. @@ -14,11 +15,53 @@ public static class QueryableExtensions /// Query to lock. /// Behavior organizes the way data should be locked, for more information check enum values. /// The same query with locking behavior added. - public static IQueryable ForUpdate(this IQueryable query, LockBehavior lockBehavior = LockBehavior.Default) + public static IQueryable ForUpdate(this IQueryable query, + LockBehavior lockBehavior = LockBehavior.Default) { query = query.TagWith(ForUpdateKey + lockBehavior.GetSqlKeyword()); return query.AsQueryable(); } + + + // public static Task FirstOrDefaultByBytesAsync( + // this IQueryable query, + // Expression> byteArrayProperty, + // byte[] searchBytes, + // int numberOfBytes, + // CancellationToken cancellationToken = default) where T : class + // { + // if (searchBytes == null || searchBytes.Length < numberOfBytes) + // { + // throw new ArgumentException($"Input array must be at least {numberOfBytes} bytes long."); + // } + // + // var firstBytes = searchBytes.Take(numberOfBytes).ToArray(); + // + // // Retrieve the DbContext from the IQueryable + // var context = query.GetDbContext(); + // + // // Get the table name from the DbContext model + // var entityType = context.Model.FindEntityType(typeof(T)); + // var tableName = entityType.GetTableName(); + // + // // Construct the SQL query + // var queryText = $@" + // SELECT * FROM ""{tableName}"" + // WHERE SUBSTRING(""Data"" FROM 1 FOR {numberOfBytes}) = @p0 + // LIMIT 1"; + // + // // Apply the byte array comparison using FromSqlRaw + // return query.FromSql(queryText, firstBytes).FirstOrDefaultAsync(cancellationToken); + // } + // + // private static DbContext GetDbContext(this IQueryable query) where T : class + // { + // var infrastructure = (IInfrastructure)query; + // var serviceProvider = infrastructure.Instance; + // var currentContext = serviceProvider.GetService(typeof(ICurrentDbContext)) as ICurrentDbContext; + // return currentContext!.Context; + // } + // } } -} +} \ No newline at end of file diff --git a/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj b/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj index 2a21756..063fad5 100644 --- a/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj +++ b/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj @@ -10,7 +10,7 @@ - + From d27ca43284f17da79c3ae67c8e0510b1edd72bf1 Mon Sep 17 00:00:00 2001 From: Haik Date: Thu, 30 May 2024 15:45:18 +0400 Subject: [PATCH 3/4] tests addded --- .../Extensions/QueryableExtensions.cs | 41 ------- .../Context/PostgresContext.cs | 14 ++- test/PandaNuGet.Demo/PandaNuGet.Demo.csproj | 2 + test/PandaNuGet.Demo/Program.cs | 13 ++ .../Services/GetByFirstBytesService.cs | 113 ++++++++++++++++++ 5 files changed, 140 insertions(+), 43 deletions(-) create mode 100644 test/PandaNuGet.Demo/Services/GetByFirstBytesService.cs diff --git a/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs b/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs index 500bec1..8c79b12 100644 --- a/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs +++ b/src/EFCore.PostgresExtensions/Extensions/QueryableExtensions.cs @@ -22,46 +22,5 @@ public static IQueryable ForUpdate(this IQueryable query, return query.AsQueryable(); } - - - // public static Task FirstOrDefaultByBytesAsync( - // this IQueryable query, - // Expression> byteArrayProperty, - // byte[] searchBytes, - // int numberOfBytes, - // CancellationToken cancellationToken = default) where T : class - // { - // if (searchBytes == null || searchBytes.Length < numberOfBytes) - // { - // throw new ArgumentException($"Input array must be at least {numberOfBytes} bytes long."); - // } - // - // var firstBytes = searchBytes.Take(numberOfBytes).ToArray(); - // - // // Retrieve the DbContext from the IQueryable - // var context = query.GetDbContext(); - // - // // Get the table name from the DbContext model - // var entityType = context.Model.FindEntityType(typeof(T)); - // var tableName = entityType.GetTableName(); - // - // // Construct the SQL query - // var queryText = $@" - // SELECT * FROM ""{tableName}"" - // WHERE SUBSTRING(""Data"" FROM 1 FOR {numberOfBytes}) = @p0 - // LIMIT 1"; - // - // // Apply the byte array comparison using FromSqlRaw - // return query.FromSql(queryText, firstBytes).FirstOrDefaultAsync(cancellationToken); - // } - // - // private static DbContext GetDbContext(this IQueryable query) where T : class - // { - // var infrastructure = (IInfrastructure)query; - // var serviceProvider = infrastructure.Instance; - // var currentContext = serviceProvider.GetService(typeof(ICurrentDbContext)) as ICurrentDbContext; - // return currentContext!.Context; - // } - // } } } \ No newline at end of file diff --git a/test/PandaNuGet.Demo/Context/PostgresContext.cs b/test/PandaNuGet.Demo/Context/PostgresContext.cs index 9bc1cf8..0954e81 100644 --- a/test/PandaNuGet.Demo/Context/PostgresContext.cs +++ b/test/PandaNuGet.Demo/Context/PostgresContext.cs @@ -1,9 +1,19 @@ -using Microsoft.EntityFrameworkCore; +using EFCoreQueryMagic.PostgresContext; +using Microsoft.EntityFrameworkCore; using PandaNuGet.Demo.Entities; namespace PandaNuGet.Demo.Context; -public class PostgresContext(DbContextOptions options) : DbContext(options) +public class PostgresContext(DbContextOptions options) : PostgresDbContext(options) { public DbSet Users { get; set; } = null!; +} + +public abstract class PostgresDbContext(DbContextOptions options) : DbContext(options) +{ + [DbFunction("substr", IsBuiltIn = true)] + public static byte[] substr(byte[] byteArray, int start, int length) + { + return byteArray.Skip(start).Take(length).ToArray(); + } } \ No newline at end of file diff --git a/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj b/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj index 063fad5..fb3aa9b 100644 --- a/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj +++ b/test/PandaNuGet.Demo/PandaNuGet.Demo.csproj @@ -11,6 +11,8 @@ + + diff --git a/test/PandaNuGet.Demo/Program.cs b/test/PandaNuGet.Demo/Program.cs index 706c81d..0881b9b 100644 --- a/test/PandaNuGet.Demo/Program.cs +++ b/test/PandaNuGet.Demo/Program.cs @@ -6,6 +6,7 @@ builder.AddPostgresContext(); builder.Services.AddScoped(); +builder.Services.AddScoped(); builder.Services.AddEndpointsApiExplorer(); builder.Services.AddSwaggerGen(); @@ -19,6 +20,18 @@ app.MapGet("ping", () => "pong"); +app.MapGet("/get-user-new", async (GetByFirstBytesService service) => +{ + await service.GetByFirstBytes(); + return "OK"; +}); + +app.MapGet("/get-user-old", async (GetByFirstBytesService service) => +{ + await service.GetByFirstBytesDavit(); + return "OK"; +}); + app.MapGet("/benchmark-sync/{minimumRows:int}", (BulkInsertService service, int minimumRows) => { var results = new List diff --git a/test/PandaNuGet.Demo/Services/GetByFirstBytesService.cs b/test/PandaNuGet.Demo/Services/GetByFirstBytesService.cs new file mode 100644 index 0000000..4309af4 --- /dev/null +++ b/test/PandaNuGet.Demo/Services/GetByFirstBytesService.cs @@ -0,0 +1,113 @@ +using System.Linq.Expressions; +using Microsoft.EntityFrameworkCore; +using PandaNuGet.Demo.Context; +using PandaNuGet.Demo.Entities; +using Pandatech.Crypto; +using PostgresDbContext = PandaNuGet.Demo.Context.PostgresDbContext; + +namespace PandaNuGet.Demo.Services; + +public class GetByFirstBytesService(PostgresContext context) +{ + private static byte[] DocumentNumber => Sha3.Hash("1234567890"); + + public async Task SeedUser() + { + var user = new UserEntity(); + var documentNumber = DocumentNumber; + var randomBytes = new byte[10]; + documentNumber = documentNumber.Concat(randomBytes).ToArray(); + user.Document = documentNumber; + context.Users.Add(user); + await context.SaveChangesAsync(); + return user.Id; + } + + public async Task GetByFirstBytes() + { + var userId = await SeedUser(); + + var userByDocument = await context + .Users + .WhereStartWithBytes(x => x.Document, 1, 64, DocumentNumber) + .FirstOrDefaultAsync(); + + Console.WriteLine($"AAAAAA {userByDocument.Id}"); + + var user = await context.Users.FindAsync(userId); + if (user != null) + { + context.Users.Remove(user); + await context.SaveChangesAsync(); + } + } + + public async Task GetByFirstBytesDavit() + { + var userId = await SeedUser(); + + var userByDocument = await context + .Users + .Where(u => PostgresDbContext.substr(u.Document, 1, 64).SequenceEqual(DocumentNumber)) + .FirstOrDefaultAsync(); + + Console.WriteLine($"AAAAAA {userByDocument.Id}"); + + var user = await context.Users.FindAsync(userId); + if (user != null) + { + context.Users.Remove(user); + await context.SaveChangesAsync(); + } + } +} + +public static class QueryableExtensions +{ + public static IQueryable WhereStartWithBytes( + this IQueryable source, + Expression> byteArraySelector, + int start, + int length, + byte[] prefix) where T : class + { + // Parameter expression for the source element + var parameter = Expression.Parameter(typeof(T), "x"); + + // Expression to get the byte array from the element + var member = Expression.Invoke(byteArraySelector, parameter); + + // MethodInfo for the substr method + var methodInfo = typeof(PostgresDbContext).GetMethod(nameof(PostgresDbContext.substr), new[] { typeof(byte[]), typeof(int), typeof(int) }); + + // Call to substr method + var call = Expression.Call( + methodInfo, + member, + Expression.Constant(start), + Expression.Constant(length)); + + // MethodInfo for the SequenceEqual method + var sequenceEqualMethod = typeof(Enumerable).GetMethods() + .First(m => m.Name == "SequenceEqual" && m.GetParameters().Length == 2) + .MakeGenericMethod(typeof(byte)); + + // Call to SequenceEqual method + var sequenceEqualCall = Expression.Call( + sequenceEqualMethod, + call, + Expression.Constant(prefix)); + + // Lambda expression for the final predicate + var lambda = Expression.Lambda>(sequenceEqualCall, parameter); + + // Apply the predicate to the source IQueryable with client-side evaluation + return source.AsEnumerable().AsQueryable().Where(lambda); + } +} + + + + + + From 01ba01423e9ba5eb9d2f56d19db00f90750826b0 Mon Sep 17 00:00:00 2001 From: Haik Date: Thu, 18 Jul 2024 00:15:59 +0400 Subject: [PATCH 4/4] added Random Incrementing Sequence Generation --- Readme.md | 106 ++++-- .../EFCore.PostgresExtensions.csproj | 4 +- .../EntityTypeConfigurationExtensions.cs | 19 ++ .../BulkInsertExtensionSync.cs | 312 ++++++++++-------- .../Extensions/MigrationBuilderExtensions.cs | 21 ++ .../Helpers/PgFunctionHelpers.cs | 58 ++++ 6 files changed, 354 insertions(+), 166 deletions(-) create mode 100644 src/EFCore.PostgresExtensions/EntityTypeConfigurationExtensions.cs create mode 100644 src/EFCore.PostgresExtensions/Extensions/MigrationBuilderExtensions.cs create mode 100644 src/EFCore.PostgresExtensions/Helpers/PgFunctionHelpers.cs diff --git a/Readme.md b/Readme.md index 55c737f..98a3e44 100644 --- a/Readme.md +++ b/Readme.md @@ -1,33 +1,24 @@ -- [1. Pandatech.EFCore.PostgresExtensions](#1-pandatechefcorepostgresextensions) - - [1.1. Features](#11-features) - - [1.2. Installation](#12-installation) - - [1.3. Usage](#13-usage) - - [1.3.1. Row-Level Locking](#131-row-level-locking) - - [1.3.2. Npgsql COPY Integration](#132-npgsql-copy-integration) - - [1.3.2.1. Benchmarks](#1321-benchmarks) - - [1.3.2.1.1. General Benchmark Results](#13211-general-benchmark-results) - - [1.3.2.1.2. Detailed Benchmark Results](#13212-detailed-benchmark-results) - - [1.3.2.1.3. Efficiency Comparison](#13213-efficiency-comparison) - - [1.3.2.1.4. Additional Notes](#13214-additional-notes) - - [1.4. License](#14-license) - -# 1. Pandatech.EFCore.PostgresExtensions +# Pandatech.EFCore.PostgresExtensions Pandatech.EFCore.PostgresExtensions is an advanced NuGet package designed to enhance PostgreSQL functionalities within Entity Framework Core, leveraging specific features not covered by the official Npgsql.EntityFrameworkCore.PostgreSQL -package. This package introduces optimized row-level locking mechanisms and an efficient, typed version of the -PostgreSQL COPY operation, adhering to EF Core syntax for seamless integration into your projects. +package. This package introduces optimized row-level locking mechanisms and PostgreSQL sequence random incrementing +features. -## 1.1. Features +## Features 1. **Row-Level Locking**: Implements the PostgreSQL `FOR UPDATE` feature, providing three lock behaviors - `Wait`, `Skip`, and `NoWait`, to facilitate advanced transaction control and concurrency management. -2. **Npgsql COPY Integration**: Offers a high-performance, typed interface for the PostgreSQL COPY command, allowing for +2. **Npgsql COPY Integration (Obsolete)**: Offers a high-performance, typed interface for the PostgreSQL COPY command, + allowing for bulk data operations within the EF Core framework. This feature significantly enhances data insertion speeds and efficiency. +3. **Random Incrementing Sequence Generation:** Provides a secure way to generate sequential IDs with random increments + to prevent predictability and potential data exposure. This ensures IDs are non-sequential and non-predictable, + enhancing security and balancing database load. -## 1.2. Installation +## Installation To install Pandatech.EFCore.PostgresExtensions, use the following NuGet command: @@ -35,9 +26,9 @@ To install Pandatech.EFCore.PostgresExtensions, use the following NuGet command: Install-Package Pandatech.EFCore.PostgresExtensions ``` -## 1.3. Usage +## Usage -### 1.3.1. Row-Level Locking +### Row-Level Locking Configure your DbContext to use Npgsql and enable query locks: @@ -71,7 +62,66 @@ catch (Exception ex) } ``` -### 1.3.2. Npgsql COPY Integration +### Random Incrementing Sequence Generation + +To configure a model to use the random ID sequence, use the `HasRandomIdSequence` extension method in your entity +configuration: + +```csharp +public class Animal +{ + public long Id { get; set; } + public string Name { get; set; } +} + +public class AnimalEntityConfiguration : IEntityTypeConfiguration +{ + public void Configure(EntityTypeBuilder builder) + { + builder.HasKey(x => x.Id); + builder.Property(x => x.Id) + .HasRandomIdSequence(); + } +} +``` + +After creating a migration, add the custom function **above create table** script in your migration class: + +```csharp +public partial class PgFunction : Migration +{ + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateRandomIdSequence("animal", "id", 5, 5, 10); //Add this line manually + + migrationBuilder.CreateTable( + name: "animal", + columns: table => new + { + id = table.Column(type: "bigint", nullable: false, defaultValueSql: "animal_random_id_generator()"), + name = table.Column(type: "text", nullable: false) + }, + constraints: table => + { + table.PrimaryKey("pk_animal", x => x.id); + }); + } + + /// + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "animal"); + } +} +``` +#### Additional notes +- The random incrementing sequence feature ensures the generated IDs are unique, non-sequential, and non-predictable, enhancing security. +- The feature supports only `long` data type (`bigint` in PostgreSQL). + + +### Npgsql COPY Integration (Obsolete: Use EFCore.BulkExtensions.PostgreSql instead) For bulk data operations, use the `BulkInsert` or `BulkInsertAsync` extension methods: @@ -89,12 +139,12 @@ public async Task BulkInsertExampleAsync() } ``` -#### 1.3.2.1. Benchmarks +#### Benchmarks The integration of the Npgsql COPY command showcases significant performance improvements compared to traditional EF Core and Dapper methods: -##### 1.3.2.1.1. General Benchmark Results +##### General Benchmark Results | Caption | Big O Notation | 1M Rows | Batch Size | |------------|----------------|-------------|------------| @@ -102,7 +152,7 @@ Core and Dapper methods: | Dapper | O(n) | 20.000 r/s | 1500 | | EFCore | O(n) | 10.600 r/s | 1500 | -##### 1.3.2.1.2. Detailed Benchmark Results +##### Detailed Benchmark Results | Operation | BulkInsert | Dapper | EF Core | |-------------|------------|--------|---------| @@ -110,7 +160,7 @@ Core and Dapper methods: | Insert 100K | 405ms | 5.47s | 8.58s | | Insert 1M | 2.87s | 55.85s | 94.57s | -##### 1.3.2.1.3. Efficiency Comparison +##### Efficiency Comparison | RowsCount | BulkInsert Efficiency | Dapper Efficiency | |-----------|----------------------------|---------------------------| @@ -118,13 +168,13 @@ Core and Dapper methods: | 100K | 21.17x faster than EF Core | 1.57x faster than EF Core | | 1M | 32.95x faster than EF Core | 1.69x faster than EF Core | -##### 1.3.2.1.4. Additional Notes +##### Additional Notes - The `BulkInsert` feature currently does not support entity properties intended for `JSON` storage. - The performance metrics provided above are based on benchmarks conducted under controlled conditions. Real-world performance may vary based on specific use cases and configurations. -## 1.4. License +## License Pandatech.EFCore.PostgresExtensions is licensed under the MIT License. diff --git a/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj b/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj index ca78c93..7bcd323 100644 --- a/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj +++ b/src/EFCore.PostgresExtensions/EFCore.PostgresExtensions.csproj @@ -8,13 +8,13 @@ Readme.md Pandatech MIT - 2.0.1 + 3.0.0 Pandatech.EFCore.PostgresExtensions Pandatech.EFCore.PostgresExtensions Pandatech, library, EntityFrameworkCore, PostgreSQL, For Update, Lock, LockingSyntax, Bulk insert, BinaryCopy The Pandatech.EFCore.PostgresExtensions library enriches Entity Framework Core applications with advanced PostgreSQL functionalities, starting with the ForUpdate locking syntax and BulkInsert function. Designed for seamless integration, this NuGet package aims to enhance the efficiency and capabilities of EF Core models when working with PostgreSQL, with the potential for further PostgreSQL-specific extensions. https://github.com/PandaTechAM/be-lib-efcore-postgres-extensions - NuGet updates + Random Incrementing Sequence Generation Feature diff --git a/src/EFCore.PostgresExtensions/EntityTypeConfigurationExtensions.cs b/src/EFCore.PostgresExtensions/EntityTypeConfigurationExtensions.cs new file mode 100644 index 0000000..4bc77d5 --- /dev/null +++ b/src/EFCore.PostgresExtensions/EntityTypeConfigurationExtensions.cs @@ -0,0 +1,19 @@ +using EFCore.PostgresExtensions.Helpers; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Metadata.Builders; + +namespace EFCore.PostgresExtensions; + +public static class EntityTypeConfigurationExtensions +{ + public static PropertyBuilder HasRandomIdSequence( + this PropertyBuilder propertyBuilder) + { + var tableName = propertyBuilder.Metadata.DeclaringType.GetTableName(); + var pgFunctionName = PgFunctionHelpers.GetPgFunctionName(tableName!); + propertyBuilder.HasDefaultValueSql(pgFunctionName); + + + return propertyBuilder; + } +} \ No newline at end of file diff --git a/src/EFCore.PostgresExtensions/Extensions/BulkInsertExtension/BulkInsertExtensionSync.cs b/src/EFCore.PostgresExtensions/Extensions/BulkInsertExtension/BulkInsertExtensionSync.cs index 849a59b..d62f61d 100644 --- a/src/EFCore.PostgresExtensions/Extensions/BulkInsertExtension/BulkInsertExtensionSync.cs +++ b/src/EFCore.PostgresExtensions/Extensions/BulkInsertExtension/BulkInsertExtensionSync.cs @@ -9,151 +9,191 @@ namespace EFCore.PostgresExtensions.Extensions.BulkInsertExtension; +[Obsolete("Use EfCore.BulkExtensions instead.")] public static class BulkInsertExtension { - public static ILogger? Logger { get; set; } + public static ILogger? Logger { get; set; } + + [Obsolete("Use EfCore.BulkExtensions instead.")] + public static async Task BulkInsertAsync(this DbSet dbSet, + List entities, + bool pkGeneratedByDb = true) where T : class + { + var context = PrepareBulkInsertOperation(dbSet, + entities, + pkGeneratedByDb, + out var sp, + out var properties, + out var columnCount, + out var sql, + out var propertyInfos, + out var propertyTypes); + + var connection = new NpgsqlConnection(context.Database.GetConnectionString()); + await connection.OpenAsync(); + + await using var writer = await connection.BeginBinaryImportAsync(sql); + + for (var entity = 0; entity < entities.Count; entity++) + { + var item = entities[entity]; + var values = propertyInfos.Select(property => property!.GetValue(item)) + .ToList(); + + ConvertEnumValue(columnCount, propertyTypes, properties, values); + + await writer.StartRowAsync(); + + for (var i = 0; i < columnCount; i++) + { + await writer.WriteAsync(values[i]); + } + } + + await writer.CompleteAsync(); + await connection.CloseAsync(); + sp.Stop(); + + Logger?.LogInformation("Binary copy completed successfully. Total time: {Milliseconds} ms", + sp.ElapsedMilliseconds); + } + + [Obsolete("Use EfCore.BulkExtensions instead.")] + public static void BulkInsert(this DbSet dbSet, + List entities, + bool pkGeneratedByDb = true) where T : class + { + var context = PrepareBulkInsertOperation(dbSet, + entities, + pkGeneratedByDb, + out var sp, + out var properties, + out var columnCount, + out var sql, + out var propertyInfos, + out var propertyTypes); + + var connection = new NpgsqlConnection(context.Database.GetConnectionString()); + connection.Open(); + + using var writer = connection.BeginBinaryImport(sql); + + for (var entity = 0; entity < entities.Count; entity++) + { + var item = entities[entity]; + var values = propertyInfos.Select(property => property!.GetValue(item)) + .ToList(); + + ConvertEnumValue(columnCount, propertyTypes, properties, values); + + writer.StartRow(); + + for (var i = 0; i < columnCount; i++) + { + writer.Write(values[i]); + } + } + + writer.Complete(); + connection.Close(); + sp.Stop(); + + Logger?.LogInformation("Binary copy completed successfully. Total time: {Milliseconds} ms", + sp.ElapsedMilliseconds); + } + + private static void ConvertEnumValue(int columnCount, + IReadOnlyList propertyTypes, + IReadOnlyList properties, + IList values) where T : class + { + for (var i = 0; i < columnCount; i++) + { + if (propertyTypes[i].IsEnum) + { + values[i] = Convert.ChangeType(values[i], Enum.GetUnderlyingType(propertyTypes[i])); + continue; + } + + // Check for generic types, specifically lists, and ensure the generic type is an enum + if (!propertyTypes[i].IsGenericType || propertyTypes[i] + .GetGenericTypeDefinition() != typeof(List<>) || + !propertyTypes[i] + .GetGenericArguments()[0].IsEnum) continue; + + var enumMapping = properties[i] + .FindTypeMapping(); + + // Only proceed if the mapping is for an array type, as expected for lists + if (enumMapping is not NpgsqlArrayTypeMapping) continue; + + var list = (IList)values[i]!; + var underlyingType = Enum.GetUnderlyingType(propertyTypes[i] + .GetGenericArguments()[0]); + + var convertedList = (from object item in list + select Convert.ChangeType(item, underlyingType)).ToList(); + values[i] = convertedList; + } + } + + + private static DbContext PrepareBulkInsertOperation(DbSet dbSet, + List entities, + bool pkGeneratedByDb, + out Stopwatch sp, + out List properties, + out int columnCount, + out string sql, + out List propertyInfos, + out List propertyTypes) where T : class + { + sp = Stopwatch.StartNew(); + var context = dbSet.GetDbContext(); + + + if (entities == null || entities.Count == 0) + throw new ArgumentException("The model list cannot be null or empty."); + + if (context == null) throw new ArgumentNullException(nameof(context), "The DbContext instance cannot be null."); + + + var entityType = context.Model.FindEntityType(typeof(T))! ?? + throw new InvalidOperationException("Entity type not found."); + + var tableName = entityType.GetTableName() ?? + throw new InvalidOperationException("Table name is null or empty."); + + properties = entityType.GetProperties() + .ToList(); - public static async Task BulkInsertAsync(this DbSet dbSet, List entities, - bool pkGeneratedByDb = true) where T : class - { - var context = PrepareBulkInsertOperation(dbSet, entities, pkGeneratedByDb, out var sp, out var properties, - out var columnCount, out var sql, out var propertyInfos, out var propertyTypes); + if (pkGeneratedByDb) + properties = properties.Where(x => !x.IsKey()) + .ToList(); - var connection = new NpgsqlConnection(context.Database.GetConnectionString()); - await connection.OpenAsync(); + var columnNames = properties.Select(x => $"\"{x.GetColumnName()}\"") + .ToList(); - await using var writer = await connection.BeginBinaryImportAsync(sql); + if (columnNames.Count == 0) + throw new InvalidOperationException("Column names are null or empty."); - for (var entity = 0; entity < entities.Count; entity++) - { - var item = entities[entity]; - var values = propertyInfos.Select(property => property!.GetValue(item)).ToList(); - ConvertEnumValue(columnCount, propertyTypes, properties, values); + columnCount = columnNames.Count; + var rowCount = entities.Count; - await writer.StartRowAsync(); + Logger?.LogDebug( + "Column names found successfully. \n Total column count: {ColumnCount} \n Total row count: {RowCount}", + columnCount, + rowCount); - for (var i = 0; i < columnCount; i++) - { - await writer.WriteAsync(values[i]); - } - } + sql = $"COPY \"{tableName}\" ({string.Join(", ", columnNames)}) FROM STDIN (FORMAT BINARY)"; - await writer.CompleteAsync(); - await connection.CloseAsync(); - sp.Stop(); + Logger?.LogInformation("SQL query created successfully. Sql query: {Sql}", sql); - Logger?.LogInformation("Binary copy completed successfully. Total time: {Milliseconds} ms", - sp.ElapsedMilliseconds); - } - - public static void BulkInsert(this DbSet dbSet, List entities, - bool pkGeneratedByDb = true) where T : class - { - var context = PrepareBulkInsertOperation(dbSet, entities, pkGeneratedByDb, out var sp, out var properties, - out var columnCount, out var sql, out var propertyInfos, out var propertyTypes); - - var connection = new NpgsqlConnection(context.Database.GetConnectionString()); - connection.Open(); - - using var writer = connection.BeginBinaryImport(sql); - - for (var entity = 0; entity < entities.Count; entity++) - { - var item = entities[entity]; - var values = propertyInfos.Select(property => property!.GetValue(item)).ToList(); - - ConvertEnumValue(columnCount, propertyTypes, properties, values); - - writer.StartRow(); - - for (var i = 0; i < columnCount; i++) - { - writer.Write(values[i]); - } - } - - writer.Complete(); - connection.Close(); - sp.Stop(); - - Logger?.LogInformation("Binary copy completed successfully. Total time: {Milliseconds} ms", - sp.ElapsedMilliseconds); - } - - private static void ConvertEnumValue(int columnCount, IReadOnlyList propertyTypes, - IReadOnlyList properties, IList values) where T : class - { - for (var i = 0; i < columnCount; i++) - { - if (propertyTypes[i].IsEnum) - { - values[i] = Convert.ChangeType(values[i], Enum.GetUnderlyingType(propertyTypes[i])); - continue; - } - - // Check for generic types, specifically lists, and ensure the generic type is an enum - if (!propertyTypes[i].IsGenericType || propertyTypes[i].GetGenericTypeDefinition() != typeof(List<>) || - !propertyTypes[i].GetGenericArguments()[0].IsEnum) continue; - - var enumMapping = properties[i].FindTypeMapping(); - - // Only proceed if the mapping is for an array type, as expected for lists - if (enumMapping is not NpgsqlArrayTypeMapping) continue; - - var list = (IList)values[i]!; - var underlyingType = Enum.GetUnderlyingType(propertyTypes[i].GetGenericArguments()[0]); - - var convertedList = (from object item in list select Convert.ChangeType(item, underlyingType)).ToList(); - values[i] = convertedList; - } - } - - - private static DbContext PrepareBulkInsertOperation(DbSet dbSet, List entities, bool pkGeneratedByDb, - out Stopwatch sp, out List properties, out int columnCount, out string sql, - out List propertyInfos, out List propertyTypes) where T : class - { - sp = Stopwatch.StartNew(); - var context = dbSet.GetDbContext(); - - - if (entities == null || entities.Count == 0) - throw new ArgumentException("The model list cannot be null or empty."); - - if (context == null) throw new ArgumentNullException(nameof(context), "The DbContext instance cannot be null."); - - - var entityType = context.Model.FindEntityType(typeof(T))! ?? - throw new InvalidOperationException("Entity type not found."); - - var tableName = entityType.GetTableName() ?? - throw new InvalidOperationException("Table name is null or empty."); - - properties = entityType.GetProperties().ToList(); - - if (pkGeneratedByDb) - properties = properties.Where(x => !x.IsKey()).ToList(); - - var columnNames = properties.Select(x => $"\"{x.GetColumnName()}\"").ToList(); - - if (columnNames.Count == 0) - throw new InvalidOperationException("Column names are null or empty."); - - - columnCount = columnNames.Count; - var rowCount = entities.Count; - - Logger?.LogDebug( - "Column names found successfully. \n Total column count: {ColumnCount} \n Total row count: {RowCount}", - columnCount, rowCount); - - sql = $"COPY \"{tableName}\" ({string.Join(", ", columnNames)}) FROM STDIN (FORMAT BINARY)"; - - Logger?.LogInformation("SQL query created successfully. Sql query: {Sql}", sql); - - propertyInfos = properties.Select(x => x.PropertyInfo).ToList(); - propertyTypes = propertyInfos.Select(x => x!.PropertyType).ToList(); - return context; - } + propertyInfos = properties.Select(x => x.PropertyInfo) + .ToList(); + propertyTypes = propertyInfos.Select(x => x!.PropertyType) + .ToList(); + return context; + } } \ No newline at end of file diff --git a/src/EFCore.PostgresExtensions/Extensions/MigrationBuilderExtensions.cs b/src/EFCore.PostgresExtensions/Extensions/MigrationBuilderExtensions.cs new file mode 100644 index 0000000..2580c00 --- /dev/null +++ b/src/EFCore.PostgresExtensions/Extensions/MigrationBuilderExtensions.cs @@ -0,0 +1,21 @@ +using EFCore.PostgresExtensions.Helpers; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace EFCore.PostgresExtensions.Extensions; + +public static class MigrationBuilderExtensions +{ + public static void CreateRandomIdSequence(this MigrationBuilder migrationBuilder, + string tableName, + string pkName, + long startValue, + int minRandIncrementValue, + int maxRandIncrementValue) + { + migrationBuilder.Sql(PgFunctionHelpers.GetPgFunction(tableName, + pkName, + startValue, + minRandIncrementValue, + maxRandIncrementValue)); + } +} \ No newline at end of file diff --git a/src/EFCore.PostgresExtensions/Helpers/PgFunctionHelpers.cs b/src/EFCore.PostgresExtensions/Helpers/PgFunctionHelpers.cs new file mode 100644 index 0000000..53b49f0 --- /dev/null +++ b/src/EFCore.PostgresExtensions/Helpers/PgFunctionHelpers.cs @@ -0,0 +1,58 @@ +namespace EFCore.PostgresExtensions.Helpers; + +public static class PgFunctionHelpers +{ + public static string GetPgFunction(string tableName, + string pkName, + long startValue, + int minRandIncrementValue, + int maxRandIncrementValue) + { + var sequenceName = GetSequenceName(tableName, pkName); + var pgFunctionName = GetPgFunctionName(tableName); + + return $""" + -- Create sequence if not exists + DO $$ + BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = '{sequenceName}') THEN + CREATE SEQUENCE {sequenceName} START WITH {startValue}; + END IF; + END + $$; + + -- Create or replace the function + CREATE OR REPLACE FUNCTION {pgFunctionName} + RETURNS bigint AS $$ + DECLARE + current_value bigint; + increment_value integer; + new_value bigint; + BEGIN + -- Acquire an advisory lock + PERFORM pg_advisory_lock(1); + + -- Get the next value of the sequence atomically + current_value := nextval('{sequenceName}'); -- name of the sequence + + -- Generate a random increment between {minRandIncrementValue} and {maxRandIncrementValue} + increment_value := floor(random() * ({maxRandIncrementValue} - {minRandIncrementValue} + 1) + {minRandIncrementValue})::integer; + + -- Set the new value with the random increment + new_value := current_value + increment_value; + + -- Update the sequence to the new value + PERFORM setval('{sequenceName}', new_value, true); -- name of the sequence + + -- Release the advisory lock + PERFORM pg_advisory_unlock(1); + + RETURN new_value; + END; + $$ LANGUAGE plpgsql; + """; + } + + private static string GetSequenceName(string tableName, string pkName) => $"{tableName}_{pkName}_seq"; + public static string GetPgFunctionName(string tableName) => $"{tableName}_random_id_generator()"; +} \ No newline at end of file