diff --git a/README.md b/README.md index e66793ab..bbb6c734 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ Download the binary package directly from the [Releases](https://github.com/tair To compile from source, ensure that you have a Golang environment set up on your local machine: ```shell -git clone https://github.com/alibaba/RedisShake +git clone https://RedisShake/ cd RedisShake sh build.sh ``` diff --git a/cmd/redis-shake/main.go b/cmd/redis-shake/main.go index 106d044a..aa8a3868 100644 --- a/cmd/redis-shake/main.go +++ b/cmd/redis-shake/main.go @@ -60,6 +60,15 @@ func main() { } theReader = reader.NewRDBReader(opts) log.Infof("create RdbReader: %v", opts.Filepath) + } else if v.IsSet("aof_reader") { // 修改aof reader + opts := new(reader.AOFReaderOptions) + defaults.SetDefaults(opts) + err := v.UnmarshalKey("rdb_reader", opts) + if err != nil { + log.Panicf("failed to read the AOFReader config entry. err: %v", err) + } + theReader = reader.NewAOFReader(opts) + log.Infof("create AOFReader: %v", opts.Filepath) } else { log.Panicf("no reader config entry found") } diff --git a/docs/src/zh/guide/getting-started.md b/docs/src/zh/guide/getting-started.md index 1df9f343..c01eb95f 100644 --- a/docs/src/zh/guide/getting-started.md +++ b/docs/src/zh/guide/getting-started.md @@ -11,7 +11,7 @@ 要从源代码编译，确保您在本地机器上设置了 Golang 环境： ```shell -git clone https://github.com/alibaba/RedisShake +git clone https://RedisShake/ cd RedisShake sh build.sh ``` diff --git a/go.mod b/go.mod index 6c90ddca..b20e995a 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.20 require ( github.com/dustin/go-humanize v1.0.1 + github.com/go-redis/redis v6.15.9+incompatible github.com/go-stack/stack v1.8.1 github.com/mcuadros/go-defaults v1.2.0 github.com/rs/zerolog v1.28.0 @@ -20,14 +21,16 @@ require ( github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/onsi/ginkgo v1.16.5 // indirect + github.com/onsi/gomega v1.27.10 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.4.2 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 23056550..a5086803 100644 --- a/go.sum +++ b/go.sum @@ -59,13 +59,18 @@ github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5y github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg= +github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= @@ -128,6 +133,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= @@ -151,6 +157,17 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= github.com/pelletier/go-toml/v2 v2.0.6/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -242,6 +259,7 @@ golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -262,6 +280,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= @@ -271,6 +290,7 @@ golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -291,6 +311,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -299,7 +320,10 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -320,6 +344,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -327,8 +352,8 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -336,8 +361,8 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -385,6 +410,7 @@ golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82u golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= @@ -485,9 +511,14 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/aof/aof.go b/internal/aof/aof.go new file mode 100644 index 00000000..1b1bf44e --- /dev/null +++ b/internal/aof/aof.go @@ -0,0 +1,1049 @@ +package aof + +import ( + "bufio" + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "time" + "unicode" + + "RedisShake/internal/entry" + "RedisShake/internal/log" + "RedisShake/internal/rdb" +) + +const ( + AOFManifestFileTypeBase = "b" /* Base File */ + AOFManifestTypeHist = "h" /* History File */ + AOFManifestTypeIncr = "i" /* INCR File */ + RDBFormatSuffix = ".RDB" + AOFFormatSuffix = ".AOF" + BaseFileSuffix = ".Base" + IncrFileSuffix = ".incr" + TempFileNamePrefix = "temp-" + COK = 1 + CERR = -1 + EINTR = 4 + ManifestNameSuffix = ".manifest" + AOFNotExist = 1 + AOFOpenErr = 3 + AOFOK = 0 + AOFEmpty = 2 + AOFFailed = 4 + AOFTruncated = 5 + SizeMax = 128 + RDBFlagsAOFPreamble = 1 << 0 +) + +var RDBFileBeingLoaded string = "" + +func Ustime() int64 { + tv := time.Now() + ust := int64(tv.UnixNano()) / 1000 + return ust + +} + +func StringNeedsRepr(s string) int { + len := len(s) + point := 0 + for len > 0 { + if s[point] == '\\' || s[point] == '"' || s[point] == '\n' || s[point] == '\r' || + s[point] == '\t' || s[point] == '\a' || s[point] == '\b' || !unicode.IsPrint(rune(s[point])) || unicode.IsSpace(rune(s[point])) { + return 1 + } + len-- + point++ + } + + return 0 +} + +func DirExists(dName string) int { + _, err := os.Stat(dName) + if err != nil { + return 0 + } + + return 1 +} + +func FileExist(FileName string) int { + _, err := os.Stat(FileName) + if err != nil { + return 0 + } + + return 1 +} + +func IsHexDigit(c byte) bool { + return (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || + (c >= 'A' && c <= 'F') +} + +func HexDigitToInt(c byte) int { + switch c { + case '0': + return 0 + case '1': + return 1 + case '2': + return 2 + case '3': + return 3 + case '4': + return 4 + case '5': + return 5 + case '6': + return 6 + case '7': + return 7 + case '8': + return 8 + case '9': + return 9 + case 'a', 'A': + return 10 + case 'b', 'B': + return 11 + case 'c', 'C': + return 12 + case 'd', 'D': + return 13 + case 'e', 'E': + return 14 + case 'f', 'F': + return 15 + default: + return 0 + } +} + +func SplitArgs(line string) ([]string, int) { + var p string = line + var Current string + var vector []string + argc := 0 + i := 0 + lens := len(p) + for { //SKIP BLANKS + for i < lens && unicode.IsSpace((rune(p[i]))) { + i++ + } + if i < lens { + inq := false // Set to true if we are in "quotes" + insq := false // Set to true if we are in 'single quotes' + done := false + + for !done { + if inq { + + if p[i] == '\\' && (p[i+1]) == 'x' && IsHexDigit(p[i+2]) && IsHexDigit(p[i+3]) { + _, err1 := strconv.ParseInt(string(p[i+2]), 16, 64) + _, err2 := strconv.ParseInt(string(p[i+3]), 16, 64) + if err1 == nil && err2 == nil { + int16 := (HexDigitToInt((p[i+2])) * 16) + HexDigitToInt(p[i+3]) + Current = Current + fmt.Sprint(int16) + i += 3 + } + + } else if p[i] == '\\' && i+1 < lens { + var c byte + i++ + switch p[i] { + case 'n': + c = '\n' + case 'r': + c = 'r' + case 'a': + c = '\a' + default: + c = p[i] + } + Current += string(c) + } else if p[i] == '"' { + if i+1 < lens && !unicode.IsSpace((rune(p[i+1]))) { + return nil, 0 + } + done = true + } else if i >= lens { + return nil, 0 + } else { + Current += string(p[i]) + } + } else if insq { + if p[i] == '\\' && p[i+1] == '\'' { + i++ + Current += "'" + } else if p[i] == '\'' { + if i+1 < lens && !unicode.IsSpace((rune(p[i+1]))) { + return nil, 0 + } + done = true + } else if i >= lens { + return nil, 0 + } else { + Current += string(p[i]) + } + + } else { + switch p[i] { + case ' ', '\n', '\r', '\t', '\000': + done = true + case '"': + inq = true + case '\'': + insq = true + default: + Current += string(p[i]) + } + } + if i < lens { + i++ + } + if i == lens { + done = true + } + } + + vector = append(vector, Current) + argc++ + Current = "" + + } else { + return vector, argc + } + + } +} + +func Stringcatlen(s string, t []byte, lent int) string { + curlen := len(s) + + if curlen == 0 { + return "" + } + + buf := make([]byte, curlen+lent) + + copy(buf[:curlen], []byte(s)) + copy(buf[curlen:], t) + return string(buf) +} + +func Stringcatprintf(s string, fmtStr string, args ...interface{}) string { + result := fmt.Sprintf(fmtStr, args...) + if s == "" { + return result + } else { + return s + result + } +} + +func Stringcatrepr(s string, p string, length int) string { + s = s + string("\"") + for i := 0; i < length; i++ { + switch p[i] { + case '\\', '"': + s = Stringcatprintf(s, "\\%c", p[i]) + case '\n': + s = s + "\\n" + case '\r': + s = s + "\\r" + case '\t': + s = s + "\\t" + case '\a': + s = s + "\\a" + case '\b': + s = s + "\\b" + default: + if strconv.IsPrint(rune(p[i])) { + s = s + string(p[i]) + } else { + s = s + "\\x%02x" + } + } + } + return s + "\"" +} + +func UpdateLoadingFileName(FileName string) { + RDBFileBeingLoaded = FileName +} + +/* AOF manifest definition */ +type AOFInfo struct { + FileName string + FileSeq int64 + AOFFileType string +} + +func AOFInfoCreate() *AOFInfo { + return new(AOFInfo) +} + +var AOF_Info AOFInfo = *AOFInfoCreate() + +func (a *AOFInfo) GetAOFInfoName() string { + return a.FileName +} + +func AOFInfoDup(orig *AOFInfo) *AOFInfo { + if orig == nil { + log.Panicf("Assertion failed: orig != nil") + } + ai := AOFInfoCreate() + ai.FileName = orig.FileName + ai.FileSeq = orig.FileSeq + ai.AOFFileType = orig.AOFFileType + return ai +} + +func AOFInfoFormat(buf string, ai *AOFInfo) string { + var AOFManifestcreate string + if StringNeedsRepr(ai.FileName) == 1 { + AOFManifestcreate = Stringcatrepr("", ai.FileName, len(ai.FileName)) + } + var ret string + if AOFManifestcreate != "" { + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, AOFManifestcreate, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) + } else { + ret = Stringcatprintf(buf, "%s %s %s %d %s %s\n", AOFManifestKeyFileName, ai.FileName, AOFManifestKeyFileSeq, ai.FileSeq, AOFManifestKeyFileType, ai.AOFFileType) + } + return ret +} + +type INFO struct { + AOFDirName string + AOFUseRDBPreamble int + AOFManifest *AOFManifest + AOFFileName string + AOFCurrentSize int64 + AOFRewriteBaseSize int64 +} + +var AOFFileInfo INFO + +func (a *INFO) GetAOFDirName() string { + return a.AOFDirName +} + +func NewAOFFileInfo(aofFilePath string) *INFO { + return &INFO{ + AOFDirName: filepath.Dir(aofFilePath), + AOFUseRDBPreamble: 0, + AOFManifest: nil, + AOFFileName: filepath.Base(aofFilePath), + AOFCurrentSize: 0, + AOFRewriteBaseSize: 0, + } +} + +func (a *INFO) SetAOFDirName(DirName string) { + a.AOFDirName = DirName +} + +func (a *INFO) GetAOFUseRDBPreamble() int { + return a.AOFUseRDBPreamble +} + +func (a *INFO) SetAOFUseRDBPreamble(useRDBPreamble int) { + a.AOFUseRDBPreamble = useRDBPreamble +} + +func (a *INFO) GetAOFManifest() *AOFManifest { + return a.AOFManifest +} +func (a *INFO) SetAOFManifest(manifest *AOFManifest) { + a.AOFManifest = manifest +} + +func (a *INFO) GetAOFFileName() string { + return a.AOFFileName +} + +func (a *INFO) SetAOFFileName(FileName string) { + a.AOFFileName = FileName +} + +func (a *INFO) GetAOFCurrentSize() int64 { + return a.AOFCurrentSize +} + +func (a *INFO) SetAOFCurrentSize(size int64) { + a.AOFCurrentSize = size +} + +func (a *INFO) GetAOFRewriteBaseSize() int64 { + return a.AOFRewriteBaseSize +} + +func (a *INFO) SetAOFRewriteBaseSize(size int64) { + a.AOFRewriteBaseSize = size +} + +type listIter struct { + next *listNode + Direction int +} + +type lists struct { + head, tail *listNode + len uint64 +} + +type listNode struct { + prev *listNode + next *listNode + value interface{} +} + +func ListCreate() *lists { + lists := &lists{} + lists.head = nil + lists.tail = nil + lists.len = 0 + return lists +} +func ListNext(iter *listIter) *listNode { + Current := iter.next + + if Current != nil { + if iter.Direction == 0 { + iter.next = Current.next + } else { + iter.next = Current.prev + } + } + return Current +} + +func (list *lists) ListsRewind(li *listIter) { + li.next = list.head + li.Direction = 0 +} + +func ListLinkNodeTail(lists *lists, node *listNode) { + if lists.len == 0 { + lists.head = node + lists.tail = node + node.prev = nil + node.next = nil + } else { + node.prev = lists.tail + node.next = nil + lists.tail.next = node + lists.tail = node + } + lists.len++ +} + +func ListAddNodeTail(lists *lists, value interface{}) *lists { + node := &listNode{ + value: value, + prev: nil, + next: nil, + } + ListLinkNodeTail(lists, node) + return lists +} + +func ListsRewindTail(list *lists, li *listIter) { + li.next = list.tail + li.Direction = 1 +} + +func ListDup(orig *lists) *lists { + var copy *lists + var iter listIter + var node *listNode + copy = ListCreate() + if copy == nil { + return nil + } + copy.ListsRewind(&iter) + node = ListNext(&iter) + var value interface{} + for node != nil { + value = node.value + } + + if ListAddNodeTail(copy, value) == nil { + return nil + } + return copy +} + +func ListIndex(list *lists, index int64) *listNode { + var n *listNode + + if index < 0 { + index = (-index) - 1 + n = list.tail + for ; index > 0 && n != nil; index-- { + n = n.prev + } + } else { + n = list.head + for ; index > 0 && n != nil; index-- { + n = n.next + } + } + return n +} + +func ListLinkNodeHead(list *lists, node *listNode) { + if list.len == 0 { + list.head = node + list.tail = node + node.prev = nil + node.next = nil + } else { + node.prev = nil + node.next = list.head + list.head.prev = node + list.head = node + } + list.len++ +} + +func ListAddNodeHead(list *lists, value interface{}) *lists { + node := &listNode{ + value: value, + } + ListLinkNodeHead(list, node) + + return list +} + +func ListUnlinkNode(list *lists, node *listNode) { + if node.prev != nil { + node.prev.next = node.next + } else { + list.head = node.next + } + if node.next != nil { + node.next.prev = node.prev + } else { + list.tail = node.prev + } + node.next = nil + node.prev = nil + + list.len-- +} +func ListDelNode(list *lists, node *listNode) { + ListUnlinkNode(list, node) + +} + +type Loader struct { + filPath string + ch chan *entry.Entry +} + +func NewLoader(filPath string, ch chan *entry.Entry) *Loader { + ld := new(Loader) + ld.ch = ch + ld.filPath = filPath + return ld +} + +type AOFManifest struct { + BaseAOFInfo *AOFInfo + incrAOFList *lists + HistoryList *lists + CurrBaseFileSeq int64 + CurrIncrFileSeq int64 + Dirty int64 +} + +func AOFManifestcreate() *AOFManifest { + am := &AOFManifest{ + incrAOFList: ListCreate(), + HistoryList: ListCreate(), + } + return am +} + +func AOFManifestDup(orig *AOFManifest) *AOFManifest { + if orig == nil { + panic("orig is nil") + } + + am := &AOFManifest{ + CurrBaseFileSeq: orig.CurrBaseFileSeq, + CurrIncrFileSeq: orig.CurrIncrFileSeq, + Dirty: orig.Dirty, + } + + if orig.BaseAOFInfo != nil { + am.BaseAOFInfo = AOFInfoDup(orig.BaseAOFInfo) + } + + am.incrAOFList = ListDup(orig.incrAOFList) + am.HistoryList = ListDup(orig.HistoryList) + + if am.incrAOFList == nil || am.HistoryList == nil { + log.Panicf("IncrAOFlist or HistoryAOFlist is nil") + } + return am +} + +func GetAOFManifestAsString(am *AOFManifest) string { + if am == nil { + panic("am is nil") + } + var buf string + var ln *listNode + var li listIter + + if am.BaseAOFInfo != nil { + buf = AOFInfoFormat(buf, am.BaseAOFInfo) + } + am.HistoryList.ListsRewind(&li) + ln = ListNext(&li) + for ln != nil { + ai, ok := ln.value.(*AOFInfo) + if ok { + buf = AOFInfoFormat(buf, ai) + } + ln = ListNext(&li) + } + + am.incrAOFList.ListsRewind(&li) + ln = ListNext(&li) + for ln != nil { + ai, ok := ln.value.(*AOFInfo) + if ok { + buf = AOFInfoFormat(buf, ai) + } + ln = ListNext(&li) + } + + return buf + +} + +func GetNewBaseFileNameAndMarkPreAsHistory(am *AOFManifest) string { + if am == nil { + log.Panicf("AOFManifest is nil") + } + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType != AOFManifestFileTypeBase { + log.Panicf("Base_AOF_info has invalid File_type") + } + am.BaseAOFInfo.AOFFileType = AOFManifestTypeHist + } + var formatSuffix string + if AOFFileInfo.AOFUseRDBPreamble == 1 { + formatSuffix = RDBFormatSuffix + } else { + formatSuffix = AOFFormatSuffix + } + ai := AOFInfoCreate() + ai.FileName = Stringcatprintf("%s.%d%s%d", AOF_Info.GetAOFInfoName(), am.CurrBaseFileSeq+1, BaseFileSuffix, formatSuffix) + ai.FileSeq = am.CurrBaseFileSeq + 1 + ai.AOFFileType = AOFManifestFileTypeBase + am.BaseAOFInfo = ai + am.Dirty = 1 + return am.BaseAOFInfo.FileName +} + +func AOFLoadManifestFromDisk() { + AOFFileInfo.AOFManifest = AOFManifestcreate() + if DirExists(AOFFileInfo.AOFDirName) == 0 { + log.Infof("The AOF Directory %v doesn't exist\n", AOFFileInfo.AOFDirName) + return + } + + am_Name := GetAOFManifestFileName() + am_Filepath := MakePath(AOFFileInfo.AOFDirName, am_Name) + if FileExist(am_Filepath) == 0 { + log.Infof("The AOF Directory %v doesn't exist\n", AOFFileInfo.AOFDirName) + return + } + + am := AOFLoadManifestFromFile(am_Filepath) + //if am != nil { + AOFFileInfo.AOFManifest = am + //} +} + +func GetNewIncrAOFName(am *AOFManifest) string { + ai := AOFInfoCreate() + ai.AOFFileType = AOFManifestTypeIncr + ai.FileName = Stringcatprintf("", "%s.%d%s%s", AOFFileInfo.AOFFileName, am.CurrIncrFileSeq+1, IncrFileSuffix, AOFFormatSuffix) + ai.FileSeq = am.CurrIncrFileSeq + 1 + ListAddNodeTail(am.incrAOFList, ai) + am.Dirty = 1 + return ai.FileName +} + +func GetTempIncrAOFNanme() string { + return Stringcatprintf("", "%s%s%s", TempFileNamePrefix, AOFFileInfo.AOFFileName, IncrFileSuffix) +} + +func GetLastIncrAOFName(am *AOFManifest) string { + if am == nil { + log.Panicf(("AOFManifest is nil")) + } + + if am.incrAOFList.len == 0 { + return GetNewIncrAOFName(am) + } + + lastnode := ListIndex(am.incrAOFList, -1) + + ai, ok := lastnode.value.(AOFInfo) + if !ok { + log.Panicf("Failed to convert lastnode.value to AOFInfo") + } + return ai.FileName +} + +func GetAOFManifestFileName() string { + return AOFFileInfo.AOFFileName +} + +func GetTempAOFManifestFileName() string { + return Stringcatprintf("", "%s%s", TempFileNamePrefix, AOFFileInfo.AOFFileName) +} + +func StartLoading(size int64, RDBflags int, async int) { + /* Load the DB */ + log.Infof("The AOF File starts loading.\n") +} + +func StopLoading(ret int) { + + if ret == AOFOK || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded\n") + } else { + log.Infof("There was an error opening the AOF File.\n") + } +} + +func AOFFileExist(FileName string) int { + Filepath := MakePath(AOFFileInfo.AOFDirName, FileName) + ret := FileExist(Filepath) + return ret +} + +func GetAppendOnlyFileSize(FileName string, status *int) int64 { + var size int64 + + AOFFilePath := MakePath(AOFFileInfo.AOFDirName, FileName) + + stat, err := os.Stat(AOFFilePath) + if err != nil { + if status != nil { + if os.IsNotExist(err) { + *status = AOFNotExist + } else { + *status = AOFOpenErr + } + } + log.Panicf("Unable to obtain the AOF File %v length. stat: %v", FileName, err.Error()) + size = 0 + } else { + if status != nil { + *status = AOFOK + } + size = stat.Size() + } + return size +} + +func GetBaseAndIncrAppendOnlyFilesSize(am *AOFManifest, status *int) int64 { + var size int64 + var ln *listNode = new(listNode) + var li *listIter = new(listIter) + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType != AOFManifestFileTypeBase { + log.Panicf("File type must be Base.") + } + size += GetAppendOnlyFileSize(am.BaseAOFInfo.FileName, status) + if *status != AOFOK { + return 0 + } + } + + am.incrAOFList.ListsRewind(li) + ln = ListNext(li) + for ln != nil { + ai := ln.value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { + log.Panicf("File type must be Incr") + } + size += GetAppendOnlyFileSize(ai.FileName, status) + if *status != AOFOK { + return 0 + } + ln = ListNext(li) + } + return size +} + +func GetBaseAndIncrAppendOnlyFilesNum(am *AOFManifest) int { + num := 0 + if am.BaseAOFInfo != nil { + num++ + } + if am.incrAOFList != nil { + num += int(am.incrAOFList.len) + } + return num +} + +func (ld *Loader) LoadSingleAppendOnlyFile(FileName string, ch chan *entry.Entry, LastFile bool) int { + ret := AOFOK + AOFFilepath := MakePath(AOFFileInfo.AOFDirName, FileName) + var sizes int64 = 0 + fp, err := os.Open(AOFFilepath) + if err != nil { + if os.IsNotExist(err) { + if _, err := os.Stat(AOFFilepath); err == nil || !os.IsNotExist(err) { + log.Infof("Fatal error: can't open the append log File %v for reading: %v", FileName, err.Error()) + return AOFOpenErr + } else { + log.Infof("The append log File %v doesn't exist: %v", FileName, err.Error()) + return AOFNotExist + } + + } + defer fp.Close() + + stat, _ := fp.Stat() + if stat.Size() == 0 { + return AOFEmpty + } + } + sig := make([]byte, 5) + if n, err := fp.Read(sig); err != nil || n != 5 || !bytes.Equal(sig, []byte("REDIS")) { + if _, err := fp.Seek(0, 0); err != nil { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } + } else { + sizes += 5 + log.Infof("Reading RDB Base File on AOF loading...") + ldRDB := rdb.NewLoader(AOFFilepath, ch) // TODO:需要修改 + ldRDB.ParseRDB() + return AOFOK + //Skipped RDB checksum and has not been processed yet. + } + reader := bufio.NewReader(fp) + for { + + line, err := reader.ReadBytes('\n') + { + if err != nil { + if err == io.EOF { + break + } + } else { + _, errs := fp.Seek(0, io.SeekCurrent) + if errs != nil { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } + } + sizes += int64(len(line)) + + if line[0] == '#' { + continue + } + if line[0] != '*' { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + argc, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) + if argc < 1 { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + if argc > int64(SizeMax) { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + e := entry.NewEntry() + argv := []string{} + + for j := 0; j < int(argc); j++ { + line, err := reader.ReadString('\n') + if err != nil || line[0] != '$' { + if err == io.EOF { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } else { + log.Infof("Bad File format reading the append only File %v:make a backup of your AOF File, then use ./redis-check-AOF --fix ", FileName) + } + } + sizes += int64(len(line)) + len, _ := strconv.ParseInt(string(line[1:len(line)-2]), 10, 64) + argstring := make([]byte, len+2) + argstring, err = reader.ReadBytes('\n') + if err != nil || argstring[len+1] != '\n' { + log.Infof("Unrecoverable error reading the append only File %v: %v", FileName, err) + ret = AOFFailed + return ret + } + /*if ConsumeNewline(argstring[len-2:]) == 0 { + return 0 + }*/ + argstring = argstring[:len] + argv = append(argv, string(argstring)) + + sizes += len + 2 + } + for _, value := range argv { + e.Argv = append(e.Argv, value) + } + ld.ch <- e + if sizes >= CheckAOFInfof.pos && LastFile { + break + } + } + + } + return ret +} + +func (ld *Loader) LoadAppendOnlyFile(am *AOFManifest, ch chan *entry.Entry) int { + if am == nil { + log.Panicf("AOFManifest is null") + } + status := AOFOK + ret := AOFOK + var start int64 + var totalSize int64 = 0 + var BaseSize int64 = 0 + var AOFName string + var totalNum, AOFNum, lastFile int + + if AOFFileExist(AOFFileInfo.AOFFileName) == 1 { + if DirExists(AOFFileInfo.AOFDirName) == 0 || + (am.BaseAOFInfo == nil && am.incrAOFList.len == 0) || + (am.BaseAOFInfo != nil && am.incrAOFList.len == 0 && + strings.Compare(am.BaseAOFInfo.FileName, AOFFileInfo.AOFFileName) == 0 && AOFFileExist(AOFFileInfo.AOFFileName) == 0) { + log.Panicf("This is an old version of the AOF File") + } + } + + if am.BaseAOFInfo == nil && am.incrAOFList == nil { + return AOFNotExist + } + + totalNum = GetBaseAndIncrAppendOnlyFilesNum(am) + if totalNum <= 0 { + log.Panicf("Assertion failed: IncrAppendOnlyFilestotalNum > 0") + } + + totalSize = GetBaseAndIncrAppendOnlyFilesSize(am, &status) + if status != AOFOK { + if status == AOFNotExist { + status = AOFFailed + } + return status + } else if totalSize == 0 { + return AOFEmpty + } + + StartLoading(totalSize, RDBFlagsAOFPreamble, 0) + if am.BaseAOFInfo != nil { + if am.BaseAOFInfo.AOFFileType == AOFManifestFileTypeBase { + AOFName = string(am.BaseAOFInfo.FileName) + UpdateLoadingFileName(AOFName) + BaseSize = GetAppendOnlyFileSize(AOFName, nil) + lastFile = totalNum + start = Ustime() + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, false) + if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { + log.Infof("DB loaded from Base File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) + } + + if ret == AOFEmpty { + ret = AOFOK + } + + if ret == AOFTruncated && lastFile == 0 { + ret = AOFFailed + log.Infof("Fatal error: the truncated File is not the last File") + } + + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOK || ret == AOFTruncated { + log.Infof("The AOF File was successfully loaded\n") + } else { + if ret == AOFOpenErr { + log.Panicf("There was an error opening the AOF File.\n") + } else { + log.Panicf("Failed to open AOF File.\n") + } + } + return ret + } + } + totalNum-- + } + + if am.incrAOFList.len > 0 { + var ln *listNode = new(listNode) + var li listIter + + am.incrAOFList.ListsRewind(&li) + ln = ListNext(&li) + for ln != nil { + ai := ln.value.(*AOFInfo) + if ai.AOFFileType != AOFManifestTypeIncr { + log.Panicf("The manifestType must be Incr") + } + AOFName = ai.FileName + UpdateLoadingFileName(AOFName) + lastFile = totalNum + AOFNum++ + start = Ustime() + if lastFile == 1 { + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, true) + } else { + ret = ld.LoadSingleAppendOnlyFile(AOFName, ch, false) + } + if ret == AOFOK || (ret == AOFTruncated && lastFile == 1) { + log.Infof("DB loaded from incr File %v: %.3f seconds", AOFName, float64(Ustime()-start)/1000000) + } + + if ret == AOFEmpty { + ret = AOFOK + } + + if ret == AOFTruncated && lastFile == 0 { + ret = AOFFailed + log.Infof("Fatal error: the truncated File is not the last File\n") + } + + if ret == AOFOpenErr || ret == AOFFailed { + if ret == AOFOpenErr { + log.Infof("There was an error opening the AOF File.\n") + } else { + log.Infof("Failed to open AOF File.\n") + } + return ret + } + ln = ListNext(&li) + } + totalNum-- + } + + AOFFileInfo.AOFCurrentSize = totalSize + AOFFileInfo.AOFRewriteBaseSize = BaseSize + return ret + +} diff --git a/internal/aof/aof_check.go b/internal/aof/aof_check.go new file mode 100644 index 00000000..93c67aa4 --- /dev/null +++ b/internal/aof/aof_check.go @@ -0,0 +1,565 @@ +package aof + +import ( + "RedisShake/internal/config" + "bufio" + "fmt" + "io" + + "math" + "os" + "path" + "path/filepath" + "strconv" + "strings" + + "RedisShake/internal/log" + "RedisShake/internal/rdb" +) + +type AOFFileType string + +const ( + AOFResp AOFFileType = "AOF_RESP" + AOFRdbPreamble AOFFileType = "AOF_RDB_PREAMBLE" + AOFMultiPart AOFFileType = "AOF_MULTI_PART" + ManifestMaxLine = 1024 + AOFCheckOk = 0 + AOFCheckEmpty = 1 + AOFCheckTruncated = 2 + AOFCheckTimeStampTruncated = 3 + AOFManifestKeyFileName = "File" + AOFManifestKeyFileSeq = "seq" + AOFManifestKeyFileType = "type" + AOFAnnoTationLineMaxLen = 1024 +) + +type CheckAOFINFOF struct { + line int64 + fp *os.File + pos int64 + toTimestamp int64 +} + +func NewCheckAOFINFOF() *CheckAOFINFOF { + return &CheckAOFINFOF{ + line: 0, + fp: nil, + pos: 0, + toTimestamp: 0, + } +} + +var CheckAOFInfof = NewCheckAOFINFOF() + +// check 里面的主函数 +func CheckAOFMain(AOFFilePath string) (checkResult bool, FileType AOFFileType, err error) { + var Filepaths string + var dirpath string + fix := 1 + Filepaths = AOFFilePath + dirpath = filepath.Dir(string(Filepaths)) + + FileType = GetInputAOFFileType(Filepaths) + switch FileType { + case "AOF_MULTI_PART": + CheckMultipartAOF(dirpath, Filepaths, fix) + case "AOF_RESP": + CheckOldStyleAOF(Filepaths, fix, false) + case "AOF_RDB_PREAMBLE": + CheckOldStyleAOF(Filepaths, fix, true) + } + return true, AOFMultiPart, nil +} + +func GetInputAOFFileType(AOFFilePath string) AOFFileType { + if FilelsManifest(AOFFilePath) { + return "AOF_MULTI_PART" + } else if FileIsRDB(AOFFilePath) { + return "AOF_RDB_PREAMBLE" + } else { + return "AOF_RESP" + } +} + +func FilelsManifest(AOFFilePath string) bool { + var is_manifest bool = false + log.Infof("FIleLsMainifest:%v", AOFFilePath) + fp, err := os.Open(AOFFilePath) + if err != nil { + log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) + } + sb, err := os.Stat(AOFFilePath) + if err != nil { + log.Panicf("cannot stat File: %v\n", AOFFilePath) + } + size := sb.Size() + if size == 0 { + fp.Close() + return false + } + reader := bufio.NewReader(fp) + for { + lines, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + break + } else { + log.Panicf("cannot read File: %v\n", AOFFilePath) + } + } + if lines[0] == '#' || len(lines) < 4 { + continue + } else if lines[:4] == "file" { + is_manifest = true + } + } + fp.Close() + return is_manifest +} + +func FileIsRDB(AOFFilePath string) bool { + fp, err := os.Open(AOFFilePath) + + if err != nil { + log.Panicf("Cannot open File %v:%v\n", AOFFilePath, err.Error()) + } + + defer fp.Close() + sb, err := os.Stat(AOFFilePath) + if err != nil { + log.Panicf("cannot stat File: %v\n", AOFFilePath) + } + size := sb.Size() + if size == 0 { + return false + } + if size >= 8 { + sig := make([]byte, 5) + _, err := fp.Read(sig) + if err == nil && string(sig) == "REDIS" { + return true + } + } + return false +} + +func OutPutAOFStyle(ret int, AOFFileName string, AOFType string) { + switch ret { + case AOFCheckOk: + log.Infof("%v %v is valid\n", AOFType, AOFFileName) + case AOFCheckEmpty: + log.Infof("%v %v is empty\n", AOFType, AOFFileName) + case AOFCheckTimeStampTruncated: + log.Infof("Successfully truncated AOF %v to timestamp %d\n", AOFFileName, CheckAOFInfof.toTimestamp) + case AOFCheckTruncated: + log.Infof("Successfully truncated AOF %v\n", AOFFileName) + } + +} + +func MakePath(Paths string, FileName string) string { + return path.Join(Paths, FileName) +} + +func PathIsBaseName(Path string) bool { + return strings.IndexByte(Path, '/') == -1 && strings.IndexByte(Path, '\\') == -1 +} + +func ReadArgc(rd *bufio.Reader, target *int64) int { + return ReadLong(rd, ' ', target) +} + +func ReadString(rd *bufio.Reader, target *string) int { + var len int64 + *target = "" + if ReadLong(rd, '$', &len) == 0 { + return 0 + } + + if len < 0 || len > math.MaxInt64-2 { + log.Infof("Expected to read string of %d bytes, which is not in the suitable range\n", len) + return 0 + } + len += 2 + // Increase length to also consume \r\n + data := make([]byte, len) + if ReadBytes(rd, &data, len) == 0 { + return 0 + } + + if ConsumeNewline(data[len-2:]) == 0 { + return 0 + } + data = data[:len-2] //\r\n + *target = string(data) + return 1 +} + +func ReadBytes(rd *bufio.Reader, target *[]byte, length int64) int { + var err error + *target, err = rd.ReadBytes('\n') + if err != nil || (*target)[length-1] != '\n' { + log.Infof("AOF format error:%s", *target) + return 0 + } + CheckAOFInfof.pos += length + return 1 +} + +func ConsumeNewline(buf []byte) int { + if buf[0] != '\r' || buf[1] != '\n' { + log.Infof("Expected \\r\\n, got: %02x%02x", buf[0], buf[1]) + return 0 + } + CheckAOFInfof.line += 1 + return 1 +} + +func ReadLong(rd *bufio.Reader, prefix byte, target *int64) int { + + var err error + var value int64 + buf, err := rd.ReadBytes('\n') + if err != nil { + log.Infof("Failed to read line from File") + return 0 + } + CheckAOFInfof.pos += int64(len(buf)) + if prefix != ' ' { + if buf[0] != prefix { + log.Infof("Expected prefix '%c', got: '%c'\n", prefix, buf[0]) + return 0 + } + value, err = strconv.ParseInt(string(buf[1:len(buf)-2]), 10, 64) //Removed line breaks\r\n + if err != nil { + log.Infof("Failed to parse long value") + return 0 + } + } else { + value, err = strconv.ParseInt(string(buf[0:len(buf)-2]), 10, 64) //Removed line breaks\r\n + if err != nil { + log.Infof("Failed to parse long value") + return 0 + } + } + *target = value + CheckAOFInfof.line += 1 + return 1 + +} + +func AOFLoadManifestFromFile(am_Filepath string) *AOFManifest { + var maxseq int64 + am := AOFManifestcreate() + fp, err := os.Open(am_Filepath) + if err != nil { + log.Panicf("Fatal error:can't open the AOF manifest %v for reading: %v", am_Filepath, err) + } + var argv []string + var ai *AOFInfo + var line string + linenum := 0 + reader := bufio.NewReader(fp) + for { + buf, err := reader.ReadString('\n') + if err != nil { + if err == io.EOF { + if linenum == 0 { + log.Infof("Found an empty AOF manifest") + am = nil + return am + } else { + break + } + + } else { + log.Infof("Read AOF manifest failed") + am = nil + return am + + } + } + + linenum++ + if buf[0] == '#' { + continue + } + if !strings.Contains(buf, "\n") { + log.Infof("The AOF manifest File contains too long line") + return nil + } + line = strings.Trim(buf, " \t\r\n") + if len(line) == 0 { + log.Infof("Invalid AOF manifest File format") + return nil + } + argc := 0 + argv, argc = SplitArgs(line) + + if argc < 6 || argc%2 != 0 { + log.Infof("Invalid AOF manifest File format") + am = nil + return am + } + ai = AOFInfoCreate() + for i := 0; i < argc; i += 2 { + if strings.EqualFold(argv[i], AOFManifestKeyFileName) { + ai.FileName = string(argv[i+1]) + if !PathIsBaseName(string(ai.FileName)) { + log.Panicf("File can't be a path, just a Filename") + } + } else if strings.EqualFold(argv[i], AOFManifestKeyFileSeq) { + ai.FileSeq, _ = strconv.ParseInt(argv[i+1], 10, 64) + } else if strings.EqualFold(argv[i], AOFManifestKeyFileType) { + ai.AOFFileType = string(argv[i+1][0]) + } + } + if ai.FileName == "" || ai.FileSeq == 0 || ai.AOFFileType == "" { + log.Panicf("Invalid AOF manifest File format") + } + if ai.AOFFileType == AOFManifestFileTypeBase { + if am.BaseAOFInfo != nil { + log.Panicf("Found duplicate Base File information") + } + am.BaseAOFInfo = ai + am.CurrBaseFileSeq = ai.FileSeq + } else if ai.AOFFileType == AOFManifestTypeHist { + am.HistoryList = ListAddNodeTail(am.HistoryList, ai) + } else if ai.AOFFileType == AOFManifestTypeIncr { + if ai.FileSeq <= maxseq { + log.Panicf("Found a non-monotonic sequence number") + } + am.incrAOFList = ListAddNodeTail(am.HistoryList, ai) + am.CurrIncrFileSeq = ai.FileSeq + maxseq = ai.FileSeq + } else { + log.Panicf("Unknown AOF File type") + } + line = " " + ai = nil + } + fp.Close() + return am +} + +func ProcessRESP(rd *bufio.Reader, Filename string, outMulti *int) int { + var argc int64 + var str string + + if ReadArgc(rd, &argc) == 0 { + return 0 + } + + for i := int64(0); i < argc; i++ { + if ReadString(rd, &str) == 0 { + return 0 + } + if i == 0 { + if strings.EqualFold(str, "multi") { + if (*outMulti) != 0 { + log.Infof("Unexpected MULTI in AOF %v", Filename) + return 0 + } + (*outMulti)++ + } else if strings.EqualFold(str, "exec") { + (*outMulti)-- + if (*outMulti) != 0 { + log.Infof("Unexpected EXEC in AOF %v", Filename) + return 0 + } + } + } + } + + return 1 +} + +func ProcessAnnotations(rd *bufio.Reader, Filename string, lastFile bool) int { + buf, _, err := rd.ReadLine() + if err != nil { + log.Panicf("Failed to read annotations from AOF %v, aborting...\n", Filename) + } + if CheckAOFInfof.toTimestamp != 0 && strings.HasPrefix(string(buf), "TS:") { + var ts int64 + ts, err = strconv.ParseInt(strings.TrimPrefix(string(buf), "TS:"), 10, 64) + if err != nil { + log.Panicf("Invalid timestamp annotation") + } + + if ts <= CheckAOFInfof.toTimestamp { + CheckAOFInfof.pos += int64(len(buf)) + 2 + return 1 + } + + if CheckAOFInfof.pos == 0 { + log.Panicf("AOF %v has nothing before timestamp %d, aborting...\n", Filename, CheckAOFInfof.toTimestamp) + } + + if !lastFile { + log.Infof("Failed to truncate AOF %v to timestamp %d to offset %d because it is not the last File.\n", Filename, CheckAOFInfof.toTimestamp, CheckAOFInfof.pos) + log.Panicf("If you insist, please delete all Files after this File according to the manifest File and delete the corresponding records in manifest File manually. Then re-run redis-check-AOF.") + } + + // Truncate remaining AOF if exceeding 'toTimestamp' + /*if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { + log.Panicf("Failed to truncate AOF %v to timestamp %d\n", Filename, CheckAOFInfof.toTimestamp) + } else {*/ + //CheckAOFInfof.pos += int64(len(buf)) + 2 + return 0 + //} + } + CheckAOFInfof.pos += int64(len(buf)) + 2 + return 1 +} + +func CheckMultipartAOF(DirPath string, ManifestFilePath string, fix int) { + totalNum := 0 + AOFNum := 0 + var ret int + am := AOFLoadManifestFromFile(ManifestFilePath) + if am.BaseAOFInfo != nil { + totalNum++ + } + if am.incrAOFList != nil { + totalNum += int(am.incrAOFList.len) + } + if am.BaseAOFInfo != nil { + AOFFileName := am.BaseAOFInfo.FileName + AOFFilePath := MakePath(DirPath, AOFFileName) + AOFNum++ + lastFile := AOFNum == totalNum + AOFPreable := FileIsRDB(AOFFilePath) + if AOFPreable { + log.Infof("Start to check Base AOF (RDB format).\n") + } else { + log.Infof("Start to check Base AOF (AOF format).\n") + } + ret = CheckSingleAOF(AOFFileName, AOFFilePath, lastFile, fix, AOFPreable) + OutPutAOFStyle(ret, AOFFileName, "Base AOF") + + } + if am.incrAOFList.len != 0 { + log.Infof("start to check INCR INCR Files.\n") + var ln *listNode + ln = am.incrAOFList.head + for ln != nil { + ai := ln.value.(*AOFInfo) + AOFFileName := ai.FileName + AOFFilePath := MakePath(DirPath, AOFFileName) + AOFNum++ + lastFile := AOFNum == totalNum + ret = CheckSingleAOF(AOFFileName, AOFFilePath, lastFile, fix, false) + OutPutAOFStyle(ret, AOFFileName, "INCR AOF") + ln = ln.next + } + } + + log.Infof("All AOF Files and manifest are vaild") +} + +func CheckOldStyleAOF(AOFFilePath string, fix int, preamble bool) { + log.Infof("Start checking Old-Style AOF\n") + var ret = CheckSingleAOF(AOFFilePath, AOFFilePath, true, fix, preamble) + OutPutAOFStyle(ret, AOFFilePath, "AOF") + +} +func CheckSingleAOF(AOFFileName, AOFFilePath string, lastFile bool, fix int, preamble bool) int { + var rdbpos int64 = 0 + CheckAOFInfof.toTimestamp = config.Config.Source.AOFTruncateToTimestamp + multi := 0 + CheckAOFInfof.pos = 0 + buf := make([]byte, 1) + var err error + CheckAOFInfof.fp, err = os.OpenFile(AOFFilePath, os.O_RDWR, 0666) + if err != nil { + log.Panicf("Cannot open File %v:%v,aborting...\n", AOFFilePath, err) + } + sb, err := CheckAOFInfof.fp.Stat() + if err != nil { + log.Panicf("Cannot stat File: %v,aborting...\n", AOFFileName) + } + size := sb.Size() + if size == 0 { + return AOFCheckEmpty + } + rd := bufio.NewReader(CheckAOFInfof.fp) + if preamble { + + rdbpos = rdb.RedisCheckRDBMain(AOFFilePath, CheckAOFInfof.fp) + rdbpos += 8 //The RDB checksum has not been processed yet. + if rdbpos == -1 { + log.Panicf("RDB preamble of AOF File is not sane, aborting.\n") + } else { + log.Infof("RDB preamble is OK, proceeding with AOF tail...\n") + _, err = CheckAOFInfof.fp.Seek(rdbpos, io.SeekStart) + if err != nil { + + log.Panicf(("Failed to seek in AOF %v: %v"), AOFFileName, err) + } + CheckAOFInfof.pos = rdbpos + } + } + + for { + + if _, err := rd.Read(buf); err != nil { + + if err == io.EOF { + + break + } + log.Panicf("Failed to read from AOF %v, aborting...\n", AOFFileName) + } + CheckAOFInfof.pos += int64(len(buf)) + if buf[0] == '#' { + if ProcessAnnotations(rd, AOFFilePath, lastFile) == 0 { + CheckAOFInfof.fp.Close() + return AOFCheckTimeStampTruncated + } + } else if buf[0] == '*' { + if ProcessRESP(rd, AOFFilePath, &multi) == 0 { + break + } + } else { + log.Infof("AOF %v format error\n", AOFFileName) + break + } + } + + diff := size - CheckAOFInfof.pos + if diff == 0 && CheckAOFInfof.toTimestamp == 1 { + log.Infof("Truncate nothing in AOF %v to timestamp %d\n", AOFFileName, CheckAOFInfof.toTimestamp) + return AOFCheckOk + } + log.Infof("AOF analyzed: Filename=%v, size=%d, ok_up_to=%d, ok_up_to_line=%d, diff=%d\n", AOFFileName, size, CheckAOFInfof.pos, CheckAOFInfof.line, diff) + if diff > 0 { + if fix == 1 { + if !lastFile { + log.Panicf("Failed to truncate AOF %v because it is not the last File\n", AOFFileName) + os.Exit(1) + } + + fmt.Printf("this will shrink the AOF %v from %d bytes,with %d bytes,to %d bytes\n", AOFFileName, size, diff, CheckAOFInfof.pos) + fmt.Print("Continue? [y/N]: ") + reader := bufio.NewReader(os.Stdin) + input, err := reader.ReadString('\n') + if err != nil || strings.ToLower(string(input[0])) != "y" { + log.Panicf("Aborting...") + + } + + if err := CheckAOFInfof.fp.Truncate(CheckAOFInfof.pos); err != nil { + log.Panicf("Failed to truncate AOF %v\n", AOFFileName) + + } else { + return AOFCheckTruncated + } + } else { + log.Panicf("AOF %v is not valid.Use the --fix potion to try fixing it.\n", AOFFileName) + } + } + CheckAOFInfof.fp.Close() + + return AOFCheckOk +} diff --git a/internal/reader/aof_reader.go b/internal/reader/aof_reader.go new file mode 100644 index 00000000..d82b38aa --- /dev/null +++ b/internal/reader/aof_reader.go @@ -0,0 +1,93 @@ +package reader + +import ( + "os" + "path" + "path/filepath" + + "RedisShake/internal/aof" + "RedisShake/internal/entry" + "RedisShake/internal/log" +) + +type AOFReaderOptions struct { // TODO：修改 + Filepath string `mapstructure:"filepath" default:""` + AOFTimestamp string +} + +type aofReader struct { + path string + ch chan *entry.Entry +} + +// TODO:需要实现 +func (r *aofReader) Status() interface{} { + //TODO implement me + panic("implement me") +} + +func (r *aofReader) StatusString() string { + //TODO implement me + panic("implement me") +} + +func (r *aofReader) StatusConsistent() bool { + //TODO implement me + panic("implement me") +} + +func NewAOFReader(opts *AOFReaderOptions) Reader { + log.Infof("NewAOFReader: path=[%s]", opts.Filepath) + absolutePath, err := filepath.Abs(opts.Filepath) + if err != nil { + log.Panicf("NewAOFReader: filepath.Abs error: %s", err.Error()) + } + log.Infof("NewAOFReader: absolute path=[%s]", absolutePath) + r := &aofReader{ + path: absolutePath, + ch: make(chan *entry.Entry), + } + return r +} + +func (r *aofReader) StartRead() chan *entry.Entry { + r.ch = make(chan *entry.Entry, 1024) + + go func() { + aof.AOFFileInfo = *(aof.NewAOFFileInfo(r.path)) + + aof.AOFLoadManifestFromDisk() + am := aof.AOFFileInfo.GetAOFManifest() + + if am == nil { + paths := path.Join(aof.AOFFileInfo.GetAOFDirName(), aof.AOFFileInfo.GetAOFFileName()) + aof.CheckAOFMain(r.path) + log.Infof("start send AOF path=[%s]", r.path) + fi, err := os.Stat(r.path) + if err != nil { + log.Panicf("NewAOFReader: os.Stat error：%s", err.Error()) + } + log.Infof("the file stat:%v", fi) + aofLoader := aof.NewLoader(r.path, r.ch) + _ = aofLoader.LoadSingleAppendOnlyFile(paths, r.ch, true) + log.Infof("Send AOF finished. path=[%s]", r.path) + close(r.ch) + } else { + paths := path.Join(aof.AOFFileInfo.GetAOFDirName(), aof.GetAOFManifestFileName()) + aof.CheckAOFMain(paths) + log.Infof("start send AOF。path=[%s]", r.path) + fi, err := os.Stat(r.path) + if err != nil { + log.Panicf("NewAOFReader: os.Stat error：%s", err.Error()) + } + log.Infof("the file stat:%v", fi) + aofLoader := aof.NewLoader(r.path, r.ch) + _ = aofLoader.LoadAppendOnlyFile(aof.AOFFileInfo.GetAOFManifest(), r.ch) + log.Infof("Send AOF finished. path=[%s]", r.path) + close(r.ch) + } + + }() + + return r.ch +} diff --git a/test/aof_test/aof_test.go b/test/aof_test/aof_test.go new file mode 100644 index 00000000..f1ab591a --- /dev/null +++ b/test/aof_test/aof_test.go @@ -0,0 +1,116 @@ +package aof_test + +import ( + "fmt" + "os" + "testing" + + "github.com/go-redis/redis" +) + +const ( + AOFRestorePath = "/test_aof_restore.toml" + AppendOnlyAoFPath = "/appendonlydir/appendonly.aof.manifest" +) + +// if you use this test you need start redis in port 6379s +func TestMainFunction(t *testing.T) { + + // os.Args = []string{"redis-shake", "/home/hwy/kaiyuan/restore.toml"} + wdPath, err := os.Getwd() + if err != nil { + panic(err) + } + + configPath := wdPath + AOFRestorePath + aofFilePath := wdPath + AppendOnlyAoFPath + fmt.Printf("configPath:%v, aofFilepath:%v\n", configPath, aofFilePath) + + // AOFMain(configPath, aofFilePath) //restore aof + + client := redis.NewClient(&redis.Options{ + Addr: "localhost:6379", + Password: "", + DB: 0, + }) + + pong, err := client.Ping().Result() + if err != nil { + t.Fatalf("Failed to connect to Redis: %v", err) + } + fmt.Println("Connected to Redis:", pong) + /* + for i := 11; i <= 10000; i++ { + value := strconv.Itoa(i) + score := float64(i) + z := redis.Z{Score: score, Member: value} + err := client.ZAdd("myzset", z).Err() + fmt.Println(value) + if err != nil { + fmt.Println("Failed to write data to Redis:", err) + return + } + } + */ + // 读取整个有序集合 + zsetValues, err := client.ZRangeWithScores("myzset", 0, -1).Result() + if err != nil { + fmt.Println("Failed to read data from Redis:", err) + return + } + + // 遍历有序集合中的元素和分数 + for _, z := range zsetValues { + member := z.Member.(string) + score := z.Score + fmt.Printf("Member: %s, Score: %f\n", member, score) + } + /* + expected := map[string]string{ + "kl": "kl", + "key0": "2022-03-29 17:25:54.592593", + "key1": "2022-03-29 17:25:54.876326", + "key2": "2022-03-29 17:25:52.871918", + "key3": "2022-03-29 17:25:53.034060", + "key4": "2022-03-29 17:25:53.196913", + "key5": "2022-03-29 17:25:53.356234", + "key6": "2022-03-29 17:25:53.513544", + "key7": "2022-03-29 17:25:53.671556", + "key8": "2022-03-29 17:25:53.861237", + "key9": "2022-03-29 17:25:54.020518", + "key10": "2022-03-29 17:25:54.177881", + "key11": "2022-03-29 17:25:54.337640", + } + */ + /*for key, value := range expected { + result, err := client.Get(key).Result() + if err != nil { + t.Fatalf("Failed to read key %s from Redis: %v", key, err) + } + + if result != value { + t.Errorf("Value for key %s is incorrect. Expected: %s, Got: %s", key, value, result) + } + } + + for key := 11; key <= 10000; key++ { + result, err := client.Get(strconv.Itoa(key)).Result() + if err != nil { + t.Fatalf("Failed to read key %v from Redis: %v", key, err) + } + + if result != strconv.Itoa(key) { + t.Errorf("Value for key %v is incorrect. Expected: %v, Got: %v", key, key, result) + } + }*/ + + /*result, err := client.SMembers("superpowers").Result() + if err != nil { + t.Fatalf("Failed to read set from Redis: %v", err) + } + strings := result[0] + if strings != "reflexes" { + t.Errorf("read set wrong") + }*/ + +} diff --git a/test/aof_test/appendonlydir/appendonly.aof.manifest b/test/aof_test/appendonlydir/appendonly.aof.manifest new file mode 100644 index 00000000..eca380fc --- /dev/null +++ b/test/aof_test/appendonlydir/appendonly.aof.manifest @@ -0,0 +1,2 @@ +file appendonly.aof.2.base.rdb seq 2 type b +file appendonly.aof.2.incr.aof seq 2 type i diff --git a/test/aof_test/test_aof_restore.toml b/test/aof_test/test_aof_restore.toml new file mode 100644 index 00000000..e5d2f762 --- /dev/null +++ b/test/aof_test/test_aof_restore.toml @@ -0,0 +1,55 @@ +type = "restore" + +[source] +version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... +# Path to the dump.rdb file. Absolute path or relative path. Note +# that relative paths are relative to the dir directory. +rdb_file_path = "" +aof_file_path="/appendonly" #Please use an absolute path. +aof_truncate_to_timestamp=0 +[target] +type = "standalone" # standalone or cluster +# When the target is a cluster, write the address of one of the nodes. +# redis-shake will obtain other nodes through the `cluster nodes` command. +version = 5.0 # redis version, such as 2.8, 4.0, 5.0, 6.0, 6.2, 7.0, ... +address = "127.0.0.1:6379" +username = "" # keep empty if not using ACL +password = "" # keep empty if no authentication is required +tls = false + +[advanced] +dir = "data" + +# runtime.GOMAXPROCS, 0 means use runtime.NumCPU() cpu cores +ncpu = 3 + +# pprof port, 0 means disable +pprof_port = 0 + +# metric port, 0 means disable +metrics_port = 0 + +# log +log_file = "redis-shake.log" +log_level = "info" # debug, info or warn +log_interval = 5 # in seconds + +# redis-shake gets key and value from rdb file, and uses RESTORE command to +# create the key in target redis. Redis RESTORE will return a "Target key name +# is busy" error when key already exists. You can use this configuration item +# to change the default behavior of restore: +# panic: redis-shake will stop when meet "Target key name is busy" error. +# rewrite: redis-shake will replace the key with new value. +# ignore: redis-shake will skip restore the key when meet "Target key name is busy" error. +rdb_restore_command_behavior = "rewrite" # panic, rewrite or skip + +# pipeline +pipeline_count_limit = 1024 + +# Client query buffers accumulate new commands. They are limited to a fixed +# amount by default. This amount is normally 1gb. +target_redis_client_max_querybuf_len = 1024_000_000 + +# In the Redis protocol, bulk requests, that are, elements representing single +# strings, are normally limited to 512 mb. +target_redis_proto_max_bulk_len = 512_000_000 \ No newline at end of file