Skip to content

Commit

Permalink
Remove creator and seller CIDs from the database (#1345)
Browse files Browse the repository at this point in the history
  • Loading branch information
@karashiiro
karashiiro authored Jul 22, 2024
1 parent f251498 commit 60d613a
Show file tree
Hide file tree
Showing 3 changed files with 74 additions and 118 deletions.
119 changes: 28 additions & 91 deletions src/Universalis.DbAccess.Tests/MarketBoard/ListingStoreTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,23 +44,7 @@ public async Task ReplaceLiveRetrieveLive_Works()
Assert.All(currentlyShown.Listings.OrderBy(l => l.PricePerUnit).Zip(results), pair =>
{
var (expected, actual) = pair;
Assert.Equal(expected.ListingId, actual.ListingId);
Assert.Equal(3, actual.ItemId);
Assert.Equal(93, actual.WorldId);
Assert.Equal(expected.Hq, actual.Hq);
Assert.Equal(expected.OnMannequin, actual.OnMannequin);
Assert.Equal(expected.PricePerUnit, actual.PricePerUnit);
Assert.Equal(expected.Quantity, actual.Quantity);
Assert.Equal(expected.RetainerName, actual.RetainerName);
Assert.Equal(expected.RetainerId, actual.RetainerId);
Assert.Equal(expected.RetainerCityId, actual.RetainerCityId);
Assert.Equal(expected.DyeId, actual.DyeId);
Assert.Equal(expected.CreatorId, actual.CreatorId);
Assert.Equal(expected.CreatorName, actual.CreatorName);
Assert.Equal(new DateTimeOffset(expected.LastReviewTime).ToUnixTimeSeconds(),
new DateTimeOffset(actual.LastReviewTime).ToUnixTimeSeconds());
Assert.Equal(DateTimeKind.Utc, actual.LastReviewTime.Kind);
Assert.Equal(expected.SellerId, actual.SellerId);
AssertEqual(expected, actual);
});
}

Expand All @@ -79,23 +63,7 @@ public async Task ReplaceLiveRetrieveLive_Cached_Works()
Assert.All(currentlyShown.Listings.OrderBy(l => l.PricePerUnit).Zip(results), pair =>
{
var (expected, actual) = pair;
Assert.Equal(expected.ListingId, actual.ListingId);
Assert.Equal(3, actual.ItemId);
Assert.Equal(92, actual.WorldId);
Assert.Equal(expected.Hq, actual.Hq);
Assert.Equal(expected.OnMannequin, actual.OnMannequin);
Assert.Equal(expected.PricePerUnit, actual.PricePerUnit);
Assert.Equal(expected.Quantity, actual.Quantity);
Assert.Equal(expected.RetainerName, actual.RetainerName);
Assert.Equal(expected.RetainerId, actual.RetainerId);
Assert.Equal(expected.RetainerCityId, actual.RetainerCityId);
Assert.Equal(expected.DyeId, actual.DyeId);
Assert.Equal(expected.CreatorId, actual.CreatorId);
Assert.Equal(expected.CreatorName, actual.CreatorName);
Assert.Equal(new DateTimeOffset(expected.LastReviewTime).ToUnixTimeSeconds(),
new DateTimeOffset(actual.LastReviewTime).ToUnixTimeSeconds());
Assert.Equal(DateTimeKind.Utc, actual.LastReviewTime.Kind);
Assert.Equal(expected.SellerId, actual.SellerId);
AssertEqual(expected, actual);
});
}

Expand Down Expand Up @@ -132,25 +100,7 @@ public async Task ReplaceLiveRetrieveLiveMultiple_Works()
Assert.All(currentlyShown.Listings.OrderBy(l => l.PricePerUnit).Zip(results), pair =>
{
var (expected, actual) = pair;
Assert.Equal(expected.ListingId, actual.ListingId);
Assert.Equal(5, actual.ItemId);
Assert.Equal(93, actual.WorldId);
Assert.Equal(expected.Hq, actual.Hq);
Assert.Equal(expected.OnMannequin, actual.OnMannequin);
Assert.Equal(expected.PricePerUnit, actual.PricePerUnit);
Assert.Equal(expected.Quantity, actual.Quantity);
Assert.Equal(expected.RetainerName, actual.RetainerName);
Assert.Equal(expected.RetainerId, actual.RetainerId);
Assert.Equal(expected.RetainerCityId, actual.RetainerCityId);
Assert.Equal(expected.DyeId, actual.DyeId);
Assert.Equal(expected.CreatorId, actual.CreatorId);
Assert.Equal(expected.CreatorName, actual.CreatorName);
Assert.Equal(new DateTimeOffset(expected.LastReviewTime).ToUnixTimeSeconds(),
new DateTimeOffset(actual.LastReviewTime).ToUnixTimeSeconds());
Assert.Equal(DateTimeKind.Utc, actual.LastReviewTime.Kind);
Assert.Equal(expected.SellerId, actual.SellerId);
Assert.Equal(DateTimeKind.Utc, actual.UpdatedAt.Kind);
Assert.Equal(expected.Source, actual.Source);
AssertEqual(expected, actual);
});
}
}
Expand Down Expand Up @@ -185,25 +135,7 @@ public async Task ReplaceLiveRetrieveManyLive_Works()
Assert.All(expectedListings[i].OrderBy(l => l.PricePerUnit).Zip(results[new WorldItemPair(93, i)]), pair =>
{
var (expected, actual) = pair;
Assert.Equal(expected.ListingId, actual.ListingId);
Assert.Equal(expected.ItemId, actual.ItemId);
Assert.Equal(expected.WorldId, actual.WorldId);
Assert.Equal(expected.Hq, actual.Hq);
Assert.Equal(expected.OnMannequin, actual.OnMannequin);
Assert.Equal(expected.PricePerUnit, actual.PricePerUnit);
Assert.Equal(expected.Quantity, actual.Quantity);
Assert.Equal(expected.RetainerName, actual.RetainerName);
Assert.Equal(expected.RetainerId, actual.RetainerId);
Assert.Equal(expected.RetainerCityId, actual.RetainerCityId);
Assert.Equal(expected.DyeId, actual.DyeId);
Assert.Equal(expected.CreatorId, actual.CreatorId);
Assert.Equal(expected.CreatorName, actual.CreatorName);
Assert.Equal(new DateTimeOffset(expected.LastReviewTime).ToUnixTimeSeconds(),
new DateTimeOffset(actual.LastReviewTime).ToUnixTimeSeconds());
Assert.Equal(DateTimeKind.Utc, actual.LastReviewTime.Kind);
Assert.Equal(expected.SellerId, actual.SellerId);
Assert.Equal(DateTimeKind.Utc, actual.UpdatedAt.Kind);
Assert.Equal(expected.Source, actual.Source);
AssertEqual(expected, actual);
});
}

Expand Down Expand Up @@ -245,25 +177,7 @@ await store.RetrieveManyLive(new ListingManyQuery
Assert.All(expectedListings[i].OrderBy(l => l.PricePerUnit).Zip(results[new WorldItemPair(93, i)]), pair =>
{
var (expected, actual) = pair;
Assert.Equal(expected.ListingId, actual.ListingId);
Assert.Equal(expected.ItemId, actual.ItemId);
Assert.Equal(expected.WorldId, actual.WorldId);
Assert.Equal(expected.Hq, actual.Hq);
Assert.Equal(expected.OnMannequin, actual.OnMannequin);
Assert.Equal(expected.PricePerUnit, actual.PricePerUnit);
Assert.Equal(expected.Quantity, actual.Quantity);
Assert.Equal(expected.RetainerName, actual.RetainerName);
Assert.Equal(expected.RetainerId, actual.RetainerId);
Assert.Equal(expected.RetainerCityId, actual.RetainerCityId);
Assert.Equal(expected.DyeId, actual.DyeId);
Assert.Equal(expected.CreatorId, actual.CreatorId);
Assert.Equal(expected.CreatorName, actual.CreatorName);
Assert.Equal(new DateTimeOffset(expected.LastReviewTime).ToUnixTimeSeconds(),
new DateTimeOffset(actual.LastReviewTime).ToUnixTimeSeconds());
Assert.Equal(DateTimeKind.Utc, actual.LastReviewTime.Kind);
Assert.Equal(expected.SellerId, actual.SellerId);
Assert.Equal(DateTimeKind.Utc, actual.UpdatedAt.Kind);
Assert.Equal(expected.Source, actual.Source);
AssertEqual(expected, actual);
});
}

Expand Down Expand Up @@ -300,4 +214,27 @@ public async Task RetrieveManyLive_ReturnsEmpty_WhenMissing()
Assert.Empty(value);
});
}

private static void AssertEqual(Listing expected, Listing actual)
{
Assert.Equal(expected.ListingId, actual.ListingId);
Assert.Equal(expected.ItemId, actual.ItemId);
Assert.Equal(expected.WorldId, actual.WorldId);
Assert.Equal(expected.Hq, actual.Hq);
Assert.Equal(expected.OnMannequin, actual.OnMannequin);
Assert.Equal(expected.PricePerUnit, actual.PricePerUnit);
Assert.Equal(expected.Quantity, actual.Quantity);
Assert.Equal(expected.RetainerName, actual.RetainerName);
Assert.Equal(expected.RetainerId, actual.RetainerId);
Assert.Equal(expected.RetainerCityId, actual.RetainerCityId);
Assert.Equal(expected.DyeId, actual.DyeId);
Assert.Null(actual.CreatorId);
Assert.Equal(expected.CreatorName, actual.CreatorName);
Assert.Equal(new DateTimeOffset(expected.LastReviewTime).ToUnixTimeSeconds(),
new DateTimeOffset(actual.LastReviewTime).ToUnixTimeSeconds());
Assert.Equal(DateTimeKind.Utc, actual.LastReviewTime.Kind);
Assert.Null(actual.SellerId);
Assert.Equal(DateTimeKind.Utc, actual.UpdatedAt.Kind);
Assert.Equal(expected.Source, actual.Source);
}
}
53 changes: 26 additions & 27 deletions src/Universalis.DbAccess/MarketBoard/ListingStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,10 +113,10 @@ public async Task ReplaceLive(IEnumerable<Listing> listings, CancellationToken c
batch.BatchCommands.Add(new NpgsqlBatchCommand(
"""
INSERT INTO listing
(listing_id, item_id, world_id, hq, on_mannequin, materia, unit_price, quantity, dye_id, creator_id,
creator_name, last_review_time, retainer_id, retainer_name, retainer_city_id, seller_id, uploaded_at,
(listing_id, item_id, world_id, hq, on_mannequin, materia, unit_price, quantity, dye_id,
creator_name, last_review_time, retainer_id, retainer_name, retainer_city_id, uploaded_at,
source)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (listing_id) DO NOTHING;
""")
{
Expand All @@ -131,13 +131,11 @@ INSERT INTO listing
new NpgsqlParameter<int> { TypedValue = listing.PricePerUnit },
new NpgsqlParameter<int> { TypedValue = listing.Quantity },
new NpgsqlParameter<int> { TypedValue = listing.DyeId },
new NpgsqlParameter<string> { TypedValue = listing.CreatorId },
new NpgsqlParameter<string> { TypedValue = listing.CreatorName },
new NpgsqlParameter<DateTime> { TypedValue = listing.LastReviewTime },
new NpgsqlParameter<string> { TypedValue = listing.RetainerId },
new NpgsqlParameter<string> { TypedValue = listing.RetainerName },
new NpgsqlParameter<int> { TypedValue = listing.RetainerCityId },
new NpgsqlParameter<string> { TypedValue = listing.SellerId },
new NpgsqlParameter<DateTime> { TypedValue = uploadedAt.UtcDateTime },
new NpgsqlParameter<string> { TypedValue = listing.Source },
},
Expand All @@ -147,6 +145,7 @@ INSERT INTO listing
try
{
rowsUpdated += await batch.ExecuteNonQueryAsync(cancellationToken);
await _easyCachingProvider.RemoveAsync(ListingsKey(worldID, itemID), cancellationToken);
}
catch (Exception e)
{
Expand Down Expand Up @@ -176,9 +175,9 @@ public async Task<IEnumerable<Listing>> RetrieveLive(ListingQuery query,
await using var command = _dataSource.CreateCommand(
"""
SELECT t.listing_id, t.item_id, t.world_id, t.hq, t.on_mannequin, t.materia,
t.unit_price, t.quantity, t.dye_id, t.creator_id, t.creator_name,
t.unit_price, t.quantity, t.dye_id, t.creator_name,
t.last_review_time, t.retainer_id, t.retainer_name, t.retainer_city_id,
t.seller_id, t.uploaded_at, t.source
t.uploaded_at, t.source
FROM listing t
WHERE t.item_id = $1 AND t.world_id = $2
ORDER BY unit_price
Expand All @@ -205,15 +204,15 @@ ORDER BY unit_price
PricePerUnit = reader.GetInt32(6),
Quantity = reader.GetInt32(7),
DyeId = reader.GetInt32(8),
CreatorId = string.Intern(reader.GetString(9)),
CreatorName = reader.GetString(10),
LastReviewTime = reader.GetDateTime(11),
RetainerId = string.Intern(reader.GetString(12)),
RetainerName = reader.GetString(13),
RetainerCityId = reader.GetInt32(14),
SellerId = string.Intern(reader.GetString(15)),
UpdatedAt = reader.GetDateTime(16),
Source = string.Intern(reader.GetString(17)),
CreatorId = null,
CreatorName = reader.GetString(9),
LastReviewTime = reader.GetDateTime(10),
RetainerId = string.Intern(reader.GetString(11)),
RetainerName = reader.GetString(12),
RetainerCityId = reader.GetInt32(13),
SellerId = null,
UpdatedAt = reader.GetDateTime(14),
Source = string.Intern(reader.GetString(15)),
});
}

Expand Down Expand Up @@ -263,9 +262,9 @@ public async Task<IDictionary<WorldItemPair, IList<Listing>>> RetrieveManyLive(L
await using var command = _dataSource.CreateCommand(
"""
SELECT t.listing_id, t.item_id, t.world_id, t.hq, t.on_mannequin, t.materia,
t.unit_price, t.quantity, t.dye_id, t.creator_id, t.creator_name,
t.unit_price, t.quantity, t.dye_id, t.creator_name,
t.last_review_time, t.retainer_id, t.retainer_name, t.retainer_city_id,
t.seller_id, t.uploaded_at, t.source
t.uploaded_at, t.source
FROM listing t
WHERE t.item_id = ANY($1) AND t.world_id = ANY($2)
""");
Expand Down Expand Up @@ -311,16 +310,16 @@ FROM listing t
DyeId = reader.GetInt32(8),
// Large hashed IDs are interned as they will likely be reused when
// the same item is requested again
CreatorId = string.Intern(reader.GetString(9)),
CreatorName = reader.GetString(10),
LastReviewTime = reader.GetDateTime(11),
RetainerId = string.Intern(reader.GetString(12)),
RetainerName = reader.GetString(13),
RetainerCityId = reader.GetInt32(14),
SellerId = string.Intern(reader.GetString(15)),
UpdatedAt = reader.GetDateTime(16),
CreatorId = null,
CreatorName = reader.GetString(9),
LastReviewTime = reader.GetDateTime(10),
RetainerId = string.Intern(reader.GetString(11)),
RetainerName = reader.GetString(12),
RetainerCityId = reader.GetInt32(13),
SellerId = null,
UpdatedAt = reader.GetDateTime(14),
// Small strings, but with only a few possible values
Source = string.Intern(reader.GetString(17)),
Source = string.Intern(reader.GetString(15)),
});
}
}
Expand Down
20 changes: 20 additions & 0 deletions src/Universalis.DbAccess/Migrations/0014_DeleteIdentifyingColumns.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
using FluentMigrator;

namespace Universalis.DbAccess.Migrations;

[Migration(14)]
public class DeleteIdentifyingColumns : Migration
{
public override void Up()
{
Delete.Column("creator_id").FromTable("listing");
Delete.Column("seller_id").FromTable("listing");
}

public override void Down()
{
Alter.Table("listing")
.AddColumn("creator_id").AsString().Nullable()
.AddColumn("seller_id").AsString().NotNullable().WithDefaultValue("");
}
}

0 comments on commit 60d613a

Please sign in to comment.