initial commit

.gitignore

@@ -0,0 +1,27 @@
+target/
+
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+nbproject/private/
+build/
+nbbuild/
+dist/
+nbdist/
+.nb-gradle/
+
+.DS_Store
+*.log

README.md

@@ -0,0 +1,10 @@
+# export-select-to-insert-golang
+
+
+go run main.go config test.yaml
+
+go build main.go
+
+GO111MODULE=on go run main.go -config test.yaml
+
+

config.yaml

@@ -0,0 +1,18 @@
+mysql:
+  user : root
+  password :
+  host : 127.0.0.1
+  port : 3306
+  name : test
+logfile:
+  filename: Logs.sql
+export:
+  tableName: Logs
+  exportSQL: SELECT * FROM Logs
+  exportOpenPaging: true
+  exportPagingStart: 0
+  exportPagingEnd: 400
+  exportPagingLimit: 20000
+
+
+

go.mod

@@ -0,0 +1,9 @@
+module golaer
+
+go 1.13
+
+require (
+	github.com/go-sql-driver/mysql v1.5.0
+	gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5
+	gopkg.in/yaml.v2 v2.2.8
+)

go.sum

@@ -0,0 +1,7 @@
+github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs=
+github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5 h1:E846t8CnR+lv5nE+VuiKTDG/v1U2stad0QzddfJC7kY=
+gopkg.in/robfig/cron.v2 v2.0.0-20150107220207-be2e0b0deed5/go.mod h1:hiOFpYm0ZJbusNj2ywpbrXowU3G8U6GIQzqn2mw1UIE=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -0,0 +1,103 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"golaer/util"
+	"log"
+	"strconv"
+	"strings"
+)
+
+func replaceStr(str string) string{
+	str = strings.Replace(str, "'","\\'",-1)
+	return str
+}
+
+func foreachData(result []map[string]string, columns []string){
+
+	fmt.Println("columns: ",columns)
+	for _, r := range result {
+		exportColumns := "("
+		values := "("
+		sql := ""
+		dataLength := len(r)
+		i := 0
+		//sortedKeys := make([]string, 0)
+		//for k, _ := range r {
+		//	sortedKeys = append(sortedKeys, k)
+		//}
+		//
+		////sort 'string' key in increasing order
+		//sort.Strings(sortedKeys)
+		for _, k := range columns {
+			i++
+			exportColumns += k
+			values += "'"+replaceStr(r[k])+"'"
+			if dataLength != i {
+				exportColumns += ","
+				values += ","
+			}
+		}
+		exportColumns += ")"
+		values += ")"
+
+		exportTable:= util.Config.Export.TableName
+		sql = "INSERT "+exportTable+" " + exportColumns + " VALUES " + values + "; \n"
+		//fmt.Print(sql)
+		util.Tracefile(sql)
+	}
+
+}
+
+
+func getData(exportSQL string) int{
+	fmt.Println("exportSQL: ",exportSQL)
+	var result, columns = util.RunSelect(exportSQL)
+	fmt.Println("columns: ",columns)
+	resultLength := len(result)
+	fmt.Println("dataLength: ", resultLength)
+	foreachData(result,columns)
+	return resultLength
+}
+
+func task() {
+	log.Println("running ...")
+	//fmt.Println(util.Config.Crontab.Period)
+	//util.Run()
+	exportSQL:= util.Config.Export.ExportSQL
+	exportOpenPaging:= util.Config.Export.ExportOpenPaging;
+
+	if exportOpenPaging {
+		exportPagingEnd:= util.Config.Export.ExportPagingEnd
+		exportPagingStart:= util.Config.Export.ExportPagingStart
+		exportPagingLimit:= util.Config.Export.ExportPagingLimit
+
+		for i := exportPagingStart; i <= exportPagingEnd; i++ {
+			limitStart := i * exportPagingLimit
+			fmt.Println("current page: ",i,"page limit: ",exportPagingLimit)
+			exportSQLPaging := exportSQL+" LIMIT " + strconv.Itoa(limitStart) + ","+ strconv.Itoa(exportPagingLimit)
+			dataLength := getData(exportSQLPaging)
+			if 0 == dataLength {
+				fmt.Println("export complete , the page is:",i,"data length is :",dataLength)
+				break
+			}
+		}
+	}else{
+		getData(exportSQL)
+	}
+
+	log.Println("export done please check logfile " + util.Config.LogFile.FileName)
+
+}
+
+
+
+func main() {
+	config := flag.String("config", "./config.yaml", "the config")
+	flag.Parse()
+	fmt.Println("config:", *config)
+	util.ConfigFile = *config
+	util.InitConfig()
+	task()
+}

modile/yaml.go

@@ -0,0 +1,23 @@
+package module
+// Yaml struct of yaml
+type Yaml struct {
+	Mysql struct {
+		User string `yaml:"user"`
+		Host string `yaml:"host"`
+		Password string `yaml:"password"`
+		Port string `yaml:"port"`
+		Name string `yaml:"name"`
+
+	}
+	LogFile struct {
+		FileName string `yaml:"filename"`
+	}
+	Export struct {
+		TableName string `yaml:"tableName"`
+		ExportSQL string `yaml:"exportSQL"`
+		ExportOpenPaging bool `yaml:"exportOpenPaging"`
+		ExportPagingStart int `yaml:"exportPagingStart"`
+		ExportPagingEnd int `yaml:"exportPagingEnd"`
+		ExportPagingLimit int `yaml:"exportPagingLimit"`
+	}
+}

util/configUtil.go

@@ -0,0 +1,27 @@
+package util
+
+import (
+	module "golaer/modile"
+	"gopkg.in/yaml.v2"
+	"io/ioutil"
+	"log"
+)
+
+var ConfigFile string;
+var Config = new(module.Yaml)
+
+func InitConfig() {
+	conf := new(module.Yaml)
+	yamlFile, err := ioutil.ReadFile(ConfigFile)
+	if err != nil {
+		log.Printf("yamlFile.Get err #%v ", err)
+	}
+	err = yaml.Unmarshal(yamlFile, conf)
+	// err = yaml.Unmarshal(yamlFile, &resultMap)
+	if err != nil {
+		log.Fatalf("Unmarshal: %v", err)
+	}
+	log.Println("conf", conf)
+	Config = conf
+}
+

util/fileUtil.go

@@ -0,0 +1,12 @@
+package util
+
+import (
+	"os"
+)
+
+func Tracefile(str_content string)  {
+	fd,_:=os.OpenFile(Config.LogFile.FileName,os.O_RDWR|os.O_CREATE|os.O_APPEND,0644)
+	buf:=[]byte(str_content)
+	fd.Write(buf)
+	fd.Close()
+}

util/mysqlUtil.go

@@ -0,0 +1,58 @@
+package util
+
+import (
+	"database/sql"
+	_ "github.com/go-sql-driver/mysql"
+	"log"
+)
+
+var dbConn *sql.DB
+
+func getConnection() *sql.DB {
+	mysqlUrl := ""+ Config.Mysql.User+":"+ Config.Mysql.Password+"@tcp("+ Config.Mysql.Host+":"+ Config.Mysql.Port+")"+"/"+Config.Mysql.Name
+		//connect database
+		db, err := sql.Open("mysql", mysqlUrl)
+		dbConn = db
+		if err != nil {
+			log.Fatal(err.Error())
+		}
+
+	return dbConn
+}
+
+
+
+func RunSelect(mSql string) ([]map[string]string,[]string ){
+	db := getConnection()
+	defer db.Close()
+
+	rows, err := db.Query(mSql)
+	if err != nil {
+		log.Fatal(err.Error())
+	}
+	columns, _ := rows.Columns()
+
+	values := make([]sql.RawBytes, len(columns))
+	scanArgs := make([]interface{}, len(values))
+
+
+	for i := range values {
+		scanArgs[i] = &values[i]
+	}
+
+	var result []map[string]string
+	for rows.Next() {
+		res := make(map[string]string)
+		rows.Scan(scanArgs...)
+		for i, col := range values {
+			res[columns[i]] = string(col)
+		}
+		result = append(result, res)
+	}
+	rows.Close()
+
+	return result,columns
+}
+
+
+