Skip to content

Commit

Permalink
Merge pull request #427 from penguin-statistics/dev
Browse files Browse the repository at this point in the history 
Release v3.11.5
  • Loading branch information
@AlvISsReimu
AlvISsReimu authored Mar 18, 2023
2 parents debf182 + 4529b79 commit 7d975e4
Show file tree
Hide file tree
Showing 6 changed files with 126 additions and 77 deletions.
11 changes: 11 additions & 0 deletions internal/model/drop_report_query_context.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,3 +28,14 @@ func (queryCtx *DropReportQueryContext) GetStageIds() []int {
}
return stageIds
}

func (queryCtx *DropReportQueryContext) GetItemIdsSet(stageId int) map[int]struct{} {
if queryCtx.StageItemFilter == nil {
return nil
}
itemIdsSet := make(map[int]struct{})
for _, itemId := range (*queryCtx.StageItemFilter)[stageId] {
itemIdsSet[itemId] = struct{}{}
}
return itemIdsSet
}
65 changes: 19 additions & 46 deletions internal/repo/drop_report.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,62 +52,35 @@ func (s *DropReport) UpdateDropReportReliability(ctx context.Context, tx bun.Tx,
return err
}

func (s *DropReport) CalcTotalQuantityForDropMatrix(
// only filtered by stage_id, not item_id, needs post-filtering
func (s *DropReport) CalcQuantityUniqCount(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.TotalQuantityResultForDropMatrix, error) {
results := make([]*model.TotalQuantityResultForDropMatrix, 0)
) ([]*model.QuantityUniqCountResultForDropMatrix, error) {
results := make([]*model.QuantityUniqCountResultForDropMatrix, 0)

subq1 := s.DB.NewSelect().
TableExpr("drop_reports AS dr").
Column("dr.stage_id", "dr.source_name", "dpe.item_id", "dpe.quantity").
Column("dr.stage_id", "dpe.item_id", "dpe.quantity").
Join("JOIN drop_pattern_elements AS dpe ON dpe.drop_pattern_id = dr.pattern_id")
s.handleAccountAndReliability(subq1, queryCtx.AccountID)
if queryCtx.ExcludeNonOneTimes {
s.handleTimes(subq1, 1)
}
s.handleCreatedAtWithTime(subq1, queryCtx.StartTime, queryCtx.EndTime)
s.handleServer(subq1, queryCtx.Server)
s.handleStagesAndItems(subq1, *queryCtx.StageItemFilter)

mainq := s.DB.NewSelect().
TableExpr("(?) AS a", subq1).
Column("stage_id", "item_id").
ColumnExpr("SUM(quantity) AS total_quantity")
s.handleSourceName(mainq, queryCtx.SourceCategory)

if err := mainq.
Group("stage_id", "item_id").
Scan(ctx, &results); err != nil {
return nil, err
if queryCtx.SourceCategory != constant.SourceCategoryAll {
subq1 = subq1.Column("dr.source_name")
}
return results, nil
}

func (s *DropReport) CalcTotalQuantityForPatternMatrix(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.TotalQuantityResultForPatternMatrix, error) {
results := make([]*model.TotalQuantityResultForPatternMatrix, 0)

subq1 := s.DB.NewSelect().
TableExpr("drop_reports AS dr").
Column("dr.source_name", "dr.stage_id", "dr.pattern_id")
s.handleAccountAndReliability(subq1, queryCtx.AccountID)
if queryCtx.ExcludeNonOneTimes {
s.handleTimes(subq1, 1)
}
s.handleCreatedAtWithTime(subq1, queryCtx.StartTime, queryCtx.EndTime)
s.handleServer(subq1, queryCtx.Server)
s.handleStages(subq1, queryCtx.GetStageIds())
s.handleTimes(subq1, 1)

mainq := s.DB.NewSelect().
TableExpr("(?) AS a", subq1).
Column("stage_id", "pattern_id").
ColumnExpr("COUNT(*) AS total_quantity")
Column("stage_id", "item_id", "quantity").
ColumnExpr("COUNT(*) AS count")
s.handleSourceName(mainq, queryCtx.SourceCategory)

if err := mainq.
Group("stage_id", "pattern_id").
Group("stage_id", "item_id", "quantity").
Scan(ctx, &results); err != nil {
return nil, err
}
Expand Down Expand Up @@ -147,31 +120,31 @@ func (s *DropReport) CalcTotalTimes(
return results, nil
}

func (s *DropReport) CalcQuantityUniqCount(
func (s *DropReport) CalcTotalQuantityForPatternMatrix(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.QuantityUniqCountResultForDropMatrix, error) {
results := make([]*model.QuantityUniqCountResultForDropMatrix, 0)
) ([]*model.TotalQuantityResultForPatternMatrix, error) {
results := make([]*model.TotalQuantityResultForPatternMatrix, 0)

subq1 := s.DB.NewSelect().
TableExpr("drop_reports AS dr").
Column("dr.source_name", "dr.stage_id", "dpe.item_id", "dpe.quantity").
Join("JOIN drop_pattern_elements AS dpe ON dpe.drop_pattern_id = dr.pattern_id")
Column("dr.source_name", "dr.stage_id", "dr.pattern_id")
s.handleAccountAndReliability(subq1, queryCtx.AccountID)
if queryCtx.ExcludeNonOneTimes {
s.handleTimes(subq1, 1)
}
s.handleCreatedAtWithTime(subq1, queryCtx.StartTime, queryCtx.EndTime)
s.handleServer(subq1, queryCtx.Server)
s.handleStagesAndItems(subq1, *queryCtx.StageItemFilter)
s.handleStages(subq1, queryCtx.GetStageIds())
s.handleTimes(subq1, 1)

mainq := s.DB.NewSelect().
TableExpr("(?) AS a", subq1).
Column("stage_id", "item_id", "quantity").
ColumnExpr("COUNT(*) AS count")
Column("stage_id", "pattern_id").
ColumnExpr("COUNT(*) AS total_quantity")
s.handleSourceName(mainq, queryCtx.SourceCategory)

if err := mainq.
Group("stage_id", "item_id", "quantity").
Group("stage_id", "pattern_id").
Scan(ctx, &results); err != nil {
return nil, err
}
Expand Down
11 changes: 5 additions & 6 deletions internal/service/analytics.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,19 @@ import (
"context"
"time"

"exusiai.dev/gommon/constant"
"github.com/pkg/errors"

v2 "exusiai.dev/backend-next/internal/model/v2"
"exusiai.dev/backend-next/internal/repo"
"exusiai.dev/gommon/constant"
)

type Analytics struct {
DropReportRepo *repo.DropReport
DropReportService *DropReport
}

func NewAnalytics(dropReportRepo *repo.DropReport) *Analytics {
func NewAnalytics(dropReportService *DropReport) *Analytics {
return &Analytics{
DropReportRepo: dropReportRepo,
DropReportService: dropReportService,
}
}

Expand All @@ -33,7 +32,7 @@ func (s *Analytics) GetRecentUniqueUserCountBySource(ctx context.Context, recent
if duration > maxDuration {
return nil, errors.New("duration is too long")
}
uniqueUserCount, err := s.DropReportRepo.CalcRecentUniqueUserCountBySource(ctx, duration)
uniqueUserCount, err := s.DropReportService.CalcRecentUniqueUserCountBySource(ctx, duration)
if err != nil {
return nil, err
}
Expand Down
42 changes: 34 additions & 8 deletions internal/service/drop_matrix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,6 @@ func (s *DropMatrix) calcDropMatrixByGivenDate(

func (s *DropMatrix) calcDropMatrix(ctx context.Context, queryCtx *model.DropReportQueryContext) ([]*model.DropMatrixElement, error) {
var combinedResults []*model.CombinedResultForDropMatrix
quantityResults, err := s.DropReportService.CalcTotalQuantityForDropMatrix(ctx, queryCtx)
if err != nil {
return nil, err
}
timesResults, err := s.DropReportService.CalcTotalTimesForDropMatrix(ctx, queryCtx)
if err != nil {
return nil, err
Expand All @@ -231,6 +227,7 @@ func (s *DropMatrix) calcDropMatrix(ctx context.Context, queryCtx *model.DropRep
if err != nil {
return nil, err
}
quantityResults := calcTotalQuantityForDropMatrix(quantityUniqCountResults)

oneBatch := s.combineQuantityAndTimesResults(quantityResults, timesResults, quantityUniqCountResults, nil)
combinedResults = append(combinedResults, oneBatch...)
Expand Down Expand Up @@ -536,10 +533,6 @@ func (s *DropMatrix) calcDropMatrixForTimeRanges(
SourceCategory: sourceCategory,
ExcludeNonOneTimes: false,
}
quantityResults, err := s.DropReportService.CalcTotalQuantityForDropMatrix(ctx, queryCtx)
if err != nil {
return nil, err
}
timesResults, err := s.DropReportService.CalcTotalTimesForDropMatrix(ctx, queryCtx)
if err != nil {
return nil, err
Expand All @@ -548,6 +541,7 @@ func (s *DropMatrix) calcDropMatrixForTimeRanges(
if err != nil {
return nil, err
}
quantityResults := calcTotalQuantityForDropMatrix(quantityUniqCountResults)
oneBatch := s.combineQuantityAndTimesResults(quantityResults, timesResults, quantityUniqCountResults, timeRange)
combinedResults = append(combinedResults, oneBatch...)
}
Expand Down Expand Up @@ -735,6 +729,38 @@ func (s *DropMatrix) combineQuantityAndTimesResults(
return combinedResults
}

func calcTotalQuantityForDropMatrix(uniqCountResults []*model.QuantityUniqCountResultForDropMatrix) []*model.TotalQuantityResultForDropMatrix {
uniqCountResultsMapByStageIdAndItemId := make(map[int]map[int][]*model.QuantityUniqCountResultForDropMatrix)
for _, result := range uniqCountResults {
if _, ok := uniqCountResultsMapByStageIdAndItemId[result.StageID]; !ok {
uniqCountResultsMapByStageIdAndItemId[result.StageID] = make(map[int][]*model.QuantityUniqCountResultForDropMatrix)
}
if _, ok := uniqCountResultsMapByStageIdAndItemId[result.StageID][result.ItemID]; !ok {
uniqCountResultsMapByStageIdAndItemId[result.StageID][result.ItemID] = make([]*model.QuantityUniqCountResultForDropMatrix, 0)
}
uniqCountResultsMapByStageIdAndItemId[result.StageID][result.ItemID] = append(uniqCountResultsMapByStageIdAndItemId[result.StageID][result.ItemID], result)
}
results := make([]*model.TotalQuantityResultForDropMatrix, 0)
for _, uniqCountResultsMapByItemId := range uniqCountResultsMapByStageIdAndItemId {
for _, uniqCountResults := range uniqCountResultsMapByItemId {
results = append(results, convertUniqCountResultsForOneItemToQuantityResult(uniqCountResults))
}
}
return results
}

func convertUniqCountResultsForOneItemToQuantityResult(uniqCountResults []*model.QuantityUniqCountResultForDropMatrix) *model.TotalQuantityResultForDropMatrix {
sum := 0
for _, result := range uniqCountResults {
sum += result.Quantity * result.Count
}
return &model.TotalQuantityResultForDropMatrix{
StageID: uniqCountResults[0].StageID,
ItemID: uniqCountResults[0].ItemID,
TotalQuantity: sum,
}
}

func (s *DropMatrix) validateQuantityBucketsAndTimes(quantityBuckets map[int]int, times int) bool {
sum := 0
for _, quantity := range quantityBuckets {
Expand Down
65 changes: 53 additions & 12 deletions internal/service/drop_report.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"gopkg.in/guregu/null.v3"

"exusiai.dev/backend-next/internal/model"
modelv2 "exusiai.dev/backend-next/internal/model/v2"
"exusiai.dev/backend-next/internal/repo"
)

Expand All @@ -20,16 +21,40 @@ func NewDropReport(dropReportRepo *repo.DropReport) *DropReport {
}
}

func (s *DropReport) CalcTotalQuantityForDropMatrix(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.TotalQuantityResultForDropMatrix, error) {
return s.DropReportRepo.CalcTotalQuantityForDropMatrix(ctx, queryCtx)
}
// DropMatrix

func (s *DropReport) CalcTotalQuantityForPatternMatrix(
func (s *DropReport) CalcQuantityUniqCount(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.TotalQuantityResultForPatternMatrix, error) {
return s.DropReportRepo.CalcTotalQuantityForPatternMatrix(ctx, queryCtx)
) ([]*model.QuantityUniqCountResultForDropMatrix, error) {
results, err := s.DropReportRepo.CalcQuantityUniqCount(ctx, queryCtx)
if err != nil {
return nil, err
}
if queryCtx.StageItemFilter == nil {
return results, nil
}
// filter the results by stageIdItemId map, because in repo layer we only filter by stageId
filteredResults := make([]*model.QuantityUniqCountResultForDropMatrix, 0)
resultsMapByStageId := make(map[int][]*model.QuantityUniqCountResultForDropMatrix)
for _, result := range results {
if _, ok := resultsMapByStageId[result.StageID]; !ok {
resultsMapByStageId[result.StageID] = make([]*model.QuantityUniqCountResultForDropMatrix, 0)
}
resultsMapByStageId[result.StageID] = append(resultsMapByStageId[result.StageID], result)
}
for stageId, stageResults := range resultsMapByStageId {
itemIdsSet := queryCtx.GetItemIdsSet(stageId)
if itemIdsSet == nil {
filteredResults = append(filteredResults, stageResults...)
} else {
for _, result := range stageResults {
if _, ok := itemIdsSet[result.ItemID]; ok {
filteredResults = append(filteredResults, result)
}
}
}
}
return filteredResults, nil
}

func (s *DropReport) CalcTotalTimesForDropMatrix(
Expand All @@ -38,12 +63,22 @@ func (s *DropReport) CalcTotalTimesForDropMatrix(
return s.DropReportRepo.CalcTotalTimes(ctx, queryCtx)
}

// PatternMatrix

func (s *DropReport) CalcTotalQuantityForPatternMatrix(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.TotalQuantityResultForPatternMatrix, error) {
return s.DropReportRepo.CalcTotalQuantityForPatternMatrix(ctx, queryCtx)
}

func (s *DropReport) CalcTotalTimesForPatternMatrix(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.TotalTimesResult, error) {
return s.DropReportRepo.CalcTotalTimes(ctx, queryCtx)
}

// Trend

func (s *DropReport) CalcTotalQuantityForTrend(
ctx context.Context, server string, startTime *time.Time, intervalLength time.Duration, intervalNum int, stageIdItemIdMap map[int][]int, accountId null.Int, sourceCategory string,
) ([]*model.TotalQuantityResultForTrend, error) {
Expand All @@ -56,8 +91,14 @@ func (s *DropReport) CalcTotalTimesForTrend(
return s.DropReportRepo.CalcTotalTimesForTrend(ctx, server, startTime, intervalLength, intervalNum, stageIds, accountId, sourceCategory)
}

func (s *DropReport) CalcQuantityUniqCount(
ctx context.Context, queryCtx *model.DropReportQueryContext,
) ([]*model.QuantityUniqCountResultForDropMatrix, error) {
return s.DropReportRepo.CalcQuantityUniqCount(ctx, queryCtx)
// Sitestats

func (s *DropReport) CalcTotalStageQuantityForShimSiteStats(ctx context.Context, server string, isRecent24h bool) ([]*modelv2.TotalStageTime, error) {
return s.DropReportRepo.CalcTotalStageQuantityForShimSiteStats(ctx, server, isRecent24h)
}

// Others

func (s *DropReport) CalcRecentUniqueUserCountBySource(ctx context.Context, duration time.Duration) ([]*modelv2.UniqueUserCountBySource, error) {
return s.DropReportRepo.CalcRecentUniqueUserCountBySource(ctx, duration)
}
9 changes: 4 additions & 5 deletions internal/service/site_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,19 @@ import (

"exusiai.dev/backend-next/internal/model/cache"
modelv2 "exusiai.dev/backend-next/internal/model/v2"
"exusiai.dev/backend-next/internal/repo"
)

type SiteStats struct {
DropReportRepo *repo.DropReport
DropReportService *DropReport
DropMatrixElementService *DropMatrixElement
}

func NewSiteStats(
dropReportRepo *repo.DropReport,
dropReportService *DropReport,
dropMatrixElementService *DropMatrixElement,
) *SiteStats {
return &SiteStats{
DropReportRepo: dropReportRepo,
DropReportService: dropReportService,
DropMatrixElementService: dropMatrixElementService,
}
}
Expand All @@ -42,7 +41,7 @@ func (s *SiteStats) RefreshShimSiteStats(ctx context.Context, server string) (*m
return nil, err
}

stageTimes24h, err := s.DropReportRepo.CalcTotalStageQuantityForShimSiteStats(ctx, server, true)
stageTimes24h, err := s.DropReportService.CalcTotalStageQuantityForShimSiteStats(ctx, server, true)
if err != nil {
return nil, err
}
Expand Down

0 comments on commit 7d975e4

Please sign in to comment.