From a829e4858cdd1b05111bff5f34fdcc039bec5cd7 Mon Sep 17 00:00:00 2001 From: DimVlas Date: Wed, 25 Dec 2024 19:52:04 +0000 Subject: [PATCH 1/4] HW10 is completed --- hw10_program_optimization/.sync | 0 hw10_program_optimization/bench_new.txt | 16 +++ hw10_program_optimization/bench_old.txt | 16 +++ hw10_program_optimization/bench_stat.txt | 15 +++ hw10_program_optimization/go.mod | 10 +- hw10_program_optimization/go.sum | 8 ++ hw10_program_optimization/stats.go | 63 +++++----- hw10_program_optimization/stats_easyjson.go | 127 ++++++++++++++++++++ hw10_program_optimization/stats_test.go | 28 +++++ 9 files changed, 246 insertions(+), 37 deletions(-) delete mode 100644 hw10_program_optimization/.sync create mode 100644 hw10_program_optimization/bench_new.txt create mode 100644 hw10_program_optimization/bench_old.txt create mode 100644 hw10_program_optimization/bench_stat.txt create mode 100644 hw10_program_optimization/stats_easyjson.go diff --git a/hw10_program_optimization/.sync b/hw10_program_optimization/.sync deleted file mode 100644 index e69de29..0000000 diff --git a/hw10_program_optimization/bench_new.txt b/hw10_program_optimization/bench_new.txt new file mode 100644 index 0000000..21023fa --- /dev/null +++ b/hw10_program_optimization/bench_new.txt @@ -0,0 +1,16 @@ +goos: linux +goarch: amd64 +pkg: github.com/DimVlas/otus_hw/hw10_program_optimization +cpu: AMD Ryzen 5 3500X 6-Core Processor +BenchmarkGetDomainStat-6 6 180000872 ns/op 29449422 B/op 723161 allocs/op +BenchmarkGetDomainStat-6 6 179475854 ns/op 29449080 B/op 723159 allocs/op +BenchmarkGetDomainStat-6 6 179211242 ns/op 29449562 B/op 723161 allocs/op +BenchmarkGetDomainStat-6 6 179568154 ns/op 29449693 B/op 723162 allocs/op +BenchmarkGetDomainStat-6 6 183618851 ns/op 29449421 B/op 723160 allocs/op +BenchmarkGetDomainStat-6 6 186144564 ns/op 29449717 B/op 723163 allocs/op +BenchmarkGetDomainStat-6 6 179541616 ns/op 29449549 B/op 723161 allocs/op +BenchmarkGetDomainStat-6 6 179762574 ns/op 29449098 B/op 723159 allocs/op +BenchmarkGetDomainStat-6 6 179707295 ns/op 29449813 B/op 723162 allocs/op +BenchmarkGetDomainStat-6 6 179366661 ns/op 29449816 B/op 723162 allocs/op +PASS +ok github.com/DimVlas/otus_hw/hw10_program_optimization 12.673s diff --git a/hw10_program_optimization/bench_old.txt b/hw10_program_optimization/bench_old.txt new file mode 100644 index 0000000..277102e --- /dev/null +++ b/hw10_program_optimization/bench_old.txt @@ -0,0 +1,16 @@ +goos: linux +goarch: amd64 +pkg: github.com/DimVlas/otus_hw/hw10_program_optimization +cpu: AMD Ryzen 5 3500X 6-Core Processor +BenchmarkGetDomainStat-6 2 552224837 ns/op 316920320 B/op 2845406 allocs/op +BenchmarkGetDomainStat-6 2 554340655 ns/op 316939656 B/op 2845410 allocs/op +BenchmarkGetDomainStat-6 2 548643641 ns/op 316883512 B/op 2845403 allocs/op +BenchmarkGetDomainStat-6 2 541981412 ns/op 316864984 B/op 2845401 allocs/op +BenchmarkGetDomainStat-6 2 551823022 ns/op 316993964 B/op 2845409 allocs/op +BenchmarkGetDomainStat-6 2 556719799 ns/op 316884424 B/op 2845408 allocs/op +BenchmarkGetDomainStat-6 2 543636506 ns/op 316920640 B/op 2845407 allocs/op +BenchmarkGetDomainStat-6 2 553364894 ns/op 316957360 B/op 2845408 allocs/op +BenchmarkGetDomainStat-6 2 550641171 ns/op 316939488 B/op 2845408 allocs/op +BenchmarkGetDomainStat-6 2 555527000 ns/op 316957552 B/op 2845409 allocs/op +PASS +ok github.com/DimVlas/otus_hw/hw10_program_optimization 16.542s diff --git a/hw10_program_optimization/bench_stat.txt b/hw10_program_optimization/bench_stat.txt new file mode 100644 index 0000000..d0be7bd --- /dev/null +++ b/hw10_program_optimization/bench_stat.txt @@ -0,0 +1,15 @@ +goos: linux +goarch: amd64 +pkg: github.com/DimVlas/otus_hw/hw10_program_optimization +cpu: AMD Ryzen 5 3500X 6-Core Processor + │ bench_old.txt │ bench_new.txt │ + │ sec/op │ sec/op vs base │ +GetDomainStat-6 552.0m ± 2% 179.6m ± 2% -67.46% (p=0.000 n=10) + + │ bench_old.txt │ bench_new.txt │ + │ B/op │ B/op vs base │ +GetDomainStat-6 302.25Mi ± 0% 28.09Mi ± 0% -90.71% (p=0.000 n=10) + + │ bench_old.txt │ bench_new.txt │ + │ allocs/op │ allocs/op vs base │ +GetDomainStat-6 2845.4k ± 0% 723.2k ± 0% -74.58% (p=0.000 n=10) diff --git a/hw10_program_optimization/go.mod b/hw10_program_optimization/go.mod index 8bccd17..0521407 100644 --- a/hw10_program_optimization/go.mod +++ b/hw10_program_optimization/go.mod @@ -1,11 +1,17 @@ -module github.com/fixme_my_friend/hw10_program_optimization +module github.com/DimVlas/otus_hw/hw10_program_optimization go 1.22 -require github.com/stretchr/testify v1.7.0 +require ( + github.com/mailru/easyjson v0.9.0 + github.com/stretchr/testify v1.7.0 +) require ( + github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + golang.org/x/perf v0.0.0-20241204221936-711ff2ab7231 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/hw10_program_optimization/go.sum b/hw10_program_optimization/go.sum index c221f64..7469e4f 100644 --- a/hw10_program_optimization/go.sum +++ b/hw10_program_optimization/go.sum @@ -1,11 +1,19 @@ +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= +github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/perf v0.0.0-20241204221936-711ff2ab7231 h1:LrlW0UtZ/i5iwthL3HvtiFwcGwgtopumPLfOwdutbzQ= +golang.org/x/perf v0.0.0-20241204221936-711ff2ab7231/go.mod h1:Ha9gd2gpRTtN8P+nXRQCFfD8JnfyIrp8vL362lG7AnU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index affb108..014c7d2 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -1,11 +1,11 @@ package hw10programoptimization import ( - "encoding/json" - "fmt" + "bufio" "io" - "regexp" "strings" + + easyjson "github.com/mailru/easyjson" ) type User struct { @@ -21,46 +21,39 @@ type User struct { type DomainStat map[string]int func GetDomainStat(r io.Reader, domain string) (DomainStat, error) { - u, err := getUsers(r) - if err != nil { - return nil, fmt.Errorf("get users error: %w", err) - } - return countDomains(u, domain) + return domainsStat(r, domain) } -type users [100_000]User - -func getUsers(r io.Reader) (result users, err error) { - content, err := io.ReadAll(r) - if err != nil { - return - } +func domainsStat(r io.Reader, domain string) (DomainStat, error) { + rd := bufio.NewReader(r) + var br bool + var user User + stat := make(DomainStat) - lines := strings.Split(string(content), "\n") - for i, line := range lines { - var user User - if err = json.Unmarshal([]byte(line), &user); err != nil { - return + for { + line, err := rd.ReadBytes('\n') + if err != nil { + switch err { + case io.EOF: + br = true + default: + return nil, err + } } - result[i] = user - } - return -} - -func countDomains(u users, domain string) (DomainStat, error) { - result := make(DomainStat) - for _, user := range u { - matched, err := regexp.Match("\\."+domain, []byte(user.Email)) - if err != nil { + if err = easyjson.Unmarshal(line, &user); err != nil { return nil, err } - if matched { - num := result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])] - num++ - result[strings.ToLower(strings.SplitN(user.Email, "@", 2)[1])] = num + if strings.HasSuffix(user.Email, "."+domain) { + d := strings.ToLower(strings.SplitN(user.Email, "@", 2)[1]) + stat[d]++ + } + + if br { + break } } - return result, nil + + return stat, nil } diff --git a/hw10_program_optimization/stats_easyjson.go b/hw10_program_optimization/stats_easyjson.go new file mode 100644 index 0000000..2bccabf --- /dev/null +++ b/hw10_program_optimization/stats_easyjson.go @@ -0,0 +1,127 @@ +// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. + +package hw10programoptimization + +import ( + json "encoding/json" + easyjson "github.com/mailru/easyjson" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" +) + +// suppress unused package warning +var ( + _ *json.RawMessage + _ *jlexer.Lexer + _ *jwriter.Writer + _ easyjson.Marshaler +) + +func easyjsonE3ab7953DecodeGithubComFixmeMyFriendHw10ProgramOptimization(in *jlexer.Lexer, out *User) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "ID": + out.ID = int(in.Int()) + case "Name": + out.Name = string(in.String()) + case "Username": + out.Username = string(in.String()) + case "Email": + out.Email = string(in.String()) + case "Phone": + out.Phone = string(in.String()) + case "Password": + out.Password = string(in.String()) + case "Address": + out.Address = string(in.String()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjsonE3ab7953EncodeGithubComFixmeMyFriendHw10ProgramOptimization(out *jwriter.Writer, in User) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"ID\":" + out.RawString(prefix[1:]) + out.Int(int(in.ID)) + } + { + const prefix string = ",\"Name\":" + out.RawString(prefix) + out.String(string(in.Name)) + } + { + const prefix string = ",\"Username\":" + out.RawString(prefix) + out.String(string(in.Username)) + } + { + const prefix string = ",\"Email\":" + out.RawString(prefix) + out.String(string(in.Email)) + } + { + const prefix string = ",\"Phone\":" + out.RawString(prefix) + out.String(string(in.Phone)) + } + { + const prefix string = ",\"Password\":" + out.RawString(prefix) + out.String(string(in.Password)) + } + { + const prefix string = ",\"Address\":" + out.RawString(prefix) + out.String(string(in.Address)) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v User) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjsonE3ab7953EncodeGithubComFixmeMyFriendHw10ProgramOptimization(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v User) MarshalEasyJSON(w *jwriter.Writer) { + easyjsonE3ab7953EncodeGithubComFixmeMyFriendHw10ProgramOptimization(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *User) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjsonE3ab7953DecodeGithubComFixmeMyFriendHw10ProgramOptimization(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *User) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjsonE3ab7953DecodeGithubComFixmeMyFriendHw10ProgramOptimization(l, v) +} diff --git a/hw10_program_optimization/stats_test.go b/hw10_program_optimization/stats_test.go index f2c20a7..5e4a4e5 100644 --- a/hw10_program_optimization/stats_test.go +++ b/hw10_program_optimization/stats_test.go @@ -1,9 +1,12 @@ +//go:build !bench // +build !bench package hw10programoptimization import ( + "archive/zip" "bytes" + "log" "testing" "github.com/stretchr/testify/require" @@ -37,3 +40,28 @@ func TestGetDomainStat(t *testing.T) { require.Equal(t, DomainStat{}, result) }) } + +func BenchmarkGetDomainStat(b *testing.B) { + z, err := zip.OpenReader("testdata/users.dat.zip") + if err != nil { + log.Println(err) + return + } + defer z.Close() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + func() { + b.StopTimer() + r, err := z.File[0].Open() + if err != nil { + log.Println(err) + return + } + defer r.Close() + + b.StartTimer() + _, _ = GetDomainStat(r, "biz") + }() + } +} From 804a6ce0ac34ad4a3c2a02542bdcfe923e4371ec Mon Sep 17 00:00:00 2001 From: DimVlas Date: Wed, 25 Dec 2024 19:58:44 +0000 Subject: [PATCH 2/4] HW10 linter --- .golangci.yml | 1 + hw10_program_optimization/stats.go | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 824c997..6ceb3f9 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -15,6 +15,7 @@ linters-settings: - github.com/DimVlas/otus_hw/hw09_struct_validator/rules - github.com/stretchr/testify/require - github.com/stretchr/testify/assert + - github.com/mailru/easyjson - $gostd exhaustive: explicit-exhaustive-switch: true diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index 014c7d2..860ae80 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -2,6 +2,7 @@ package hw10programoptimization import ( "bufio" + "errors" "io" "strings" @@ -33,10 +34,9 @@ func domainsStat(r io.Reader, domain string) (DomainStat, error) { for { line, err := rd.ReadBytes('\n') if err != nil { - switch err { - case io.EOF: + if !errors.Is(err, io.EOF) { br = true - default: + } else { return nil, err } } From d8a90f29dcddacde2979722158e2c12984b1ede7 Mon Sep 17 00:00:00 2001 From: DimVlas Date: Wed, 25 Dec 2024 20:02:19 +0000 Subject: [PATCH 3/4] HW10 tests --- hw10_program_optimization/stats.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hw10_program_optimization/stats.go b/hw10_program_optimization/stats.go index 860ae80..892641c 100644 --- a/hw10_program_optimization/stats.go +++ b/hw10_program_optimization/stats.go @@ -34,7 +34,7 @@ func domainsStat(r io.Reader, domain string) (DomainStat, error) { for { line, err := rd.ReadBytes('\n') if err != nil { - if !errors.Is(err, io.EOF) { + if errors.Is(err, io.EOF) { br = true } else { return nil, err From ac0de95a1810acad2b6afd52994a37932d6fce33 Mon Sep 17 00:00:00 2001 From: DimVlas Date: Wed, 25 Dec 2024 20:05:04 +0000 Subject: [PATCH 4/4] HW10 mod tidy --- hw10_program_optimization/go.mod | 2 -- hw10_program_optimization/go.sum | 4 ---- 2 files changed, 6 deletions(-) diff --git a/hw10_program_optimization/go.mod b/hw10_program_optimization/go.mod index 0521407..57c7414 100644 --- a/hw10_program_optimization/go.mod +++ b/hw10_program_optimization/go.mod @@ -8,10 +8,8 @@ require ( ) require ( - github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - golang.org/x/perf v0.0.0-20241204221936-711ff2ab7231 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect ) diff --git a/hw10_program_optimization/go.sum b/hw10_program_optimization/go.sum index 7469e4f..7d979c2 100644 --- a/hw10_program_optimization/go.sum +++ b/hw10_program_optimization/go.sum @@ -1,5 +1,3 @@ -github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 h1:xlwdaKcTNVW4PtpQb8aKA4Pjy0CdJHEqvFbAnvR5m2g= -github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794/go.mod h1:7e+I0LQFUI9AXWxOfsQROs9xPhoJtbsyWcjJqDd4KPY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -12,8 +10,6 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/perf v0.0.0-20241204221936-711ff2ab7231 h1:LrlW0UtZ/i5iwthL3HvtiFwcGwgtopumPLfOwdutbzQ= -golang.org/x/perf v0.0.0-20241204221936-711ff2ab7231/go.mod h1:Ha9gd2gpRTtN8P+nXRQCFfD8JnfyIrp8vL362lG7AnU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=