Initial integration of relational and temporal databases, initial sup…

.gitignore

@@ -7,6 +7,10 @@
 *.dll
 *.so
 *.dylib
+*.back
+output.html
+nohup
+main
 
 # Test binary, built with `go test -c`
 *.test

cmd/root.go

@@ -0,0 +1,50 @@
+/*
+Copyright © 2024 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"os"
+	"webprobe/internal"
+
+	"github.com/spf13/cobra"
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "webprobe",
+	Short: "支持IPv4、IPv6的网站探测工具，	支持探测多级子链并计算支持度",
+	Long: `支持IPv4、IPv6的网站探测工具，能够探测多级子链并计算支持度，同时提供网络性能优化、报告和可视化、灵活配置选项和安全性考虑。
+
+	`,
+	// Uncomment the following line if your bare application
+	// has an action associated with it:
+	// Run: func(cmd *cobra.Command, args []string) {},
+	Run: func(cmd *cobra.Command, args []string) {
+		// Do Stuff Here
+		internal.ScrapyWeb()
+	},
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+
+}
+
+func init() {
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+
+	// rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.webprobe.yaml)")
+
+	// Cobra also supports local flags, which will only run
+	// when this action is called directly.
+	rootCmd.PersistentFlags().StringVarP(&internal.ConfigPath, "config", "c", ".", "config file directory")
+	//rootCmd.MarkPersistentFlagRequired("config") // 标记为必需
+}

cmd/run.go

@@ -0,0 +1,39 @@
+/*
+Copyright © 2024 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"webprobe/utility"
+
+	"github.com/spf13/cobra"
+)
+
+// runCmd represents the run command
+var runCmd = &cobra.Command{
+	Use:   "run",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		utility.Cmdline()
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(runCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// runCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// runCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

cmd/serve.go

@@ -0,0 +1,40 @@
+/*
+Copyright © 2024 NAME HERE <EMAIL ADDRESS>
+
+*/
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// serveCmd represents the serve command
+var serveCmd = &cobra.Command{
+	Use:   "serve",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("serve called")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(serveCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// serveCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// serveCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

cmd/toDB.go

@@ -0,0 +1,67 @@
+/*
+Copyright © 2024 NAME HERE <EMAIL ADDRESS>
+*/
+package cmd
+
+import (
+	"log"
+	"webprobe/handler"
+	"webprobe/internal"
+	"webprobe/utility"
+
+	"github.com/spf13/cobra"
+)
+
+// toDBCmd represents the toDB command
+var toDBCmd = &cobra.Command{
+	Use:   "toDB",
+	Short: "Specify the output database type (relational or time-series). Default is time-series.",
+	Long: `Specify the type of database to save the scan results. The available options are:
+
+	-- RDB: Save the scan results to a relational database.
+	-- TSDB: Save the scan results to a time-series database (default).`,
+	Run: func(cmd *cobra.Command, args []string) {
+		internal.ScrapyWeb()
+
+		RDBItems, _ := cmd.Flags().GetStringArray("RDB")
+		TSDBItems, _ := cmd.Flags().GetStringArray("TSDB")
+		//for _, item := range RDBItems {
+		if len(RDBItems) > 0 {
+
+			if internal.Cfg.DBConfig.UseDB {
+				log.Printf("正在写入数据库...")
+				handler.SaveData(internal.DB, internal.UrlDataWithReachability, internal.Cfg.ScannerConfig.Crawl.Depth)
+				log.Printf("写入关系数据库" + RDBItems[0] + "完成.")
+			}
+		}
+
+		//for _, item := range TSDBItems {
+		if len(TSDBItems) > 0 {
+
+			utility.PushToPrometheus(internal.Cfg.Push.PushGatewayURL, internal.UrlDataWithReachability)
+			log.Printf("写入时序数据库" + TSDBItems[0] + "完成.")
+		}
+
+	},
+}
+
+func init() {
+
+	rootCmd.AddCommand(toDBCmd)
+	// 在这里定义您的标志和配置设置。
+
+	// Cobra支持持久性标志，这些标志将适用于此命令
+	// 和所有子命令，例如：
+	// toDBCmd.PersistentFlags().String("foo", "", "A help for foo")
+	// 创建toDBCmd命令的Flag，作用为指定关系性数据库的名称 (默认为tsdb)
+	toDBCmd.PersistentFlags().StringArrayP("RDB", "r", []string{""}, "result to RDB")
+	// 创建toDBCmd命令的Flag，作用为指定时序数据库的名称 (默认为tsdb)
+	toDBCmd.PersistentFlags().StringArrayP("TSDB", "t", []string{""}, "result to toTSDB")
+
+	// Cobra支持仅在直接调用此命令时运行的本地标志，例如：
+	// toDBCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+
+}
+
+//
+//

config.yaml

@@ -1,7 +1,7 @@
-url: ["https://www.ecnu.edu.cn", "https://www.baidu.com"]
+url: ["https://eoffice.ecnu.edu.cn"]
 scanner:
     crawl:
-        depth: 2 # 爬虫时的链接深度，0 代表只检测url自身，以此类推
+        depth: 1 # 爬虫时的链接深度，0 代表只检测url自身，以此类推
         timeout: 5 # 爬虫时访问每个url的超时时间
     check:
         ipv6: false # 默认仅使用ipv4检测，设置为true后进行ipv4和ipv6双检测
@@ -13,9 +13,16 @@ scanner:
         http_client_timeout: 15 # 请求每个URL的超时时间
         context_timeout: 1000 # 完成所有检测的超时时间，应足够大，否则会在检测某个url上一直挂起
 push:
-    push_to_prometheus: false
-    push_gateway_url: "http://10.24.70.20:9091"
+    push_to_prometheus: true
+    push_gateway_url: "http://127.0.0.1:9091"
 output: "output.html"
 db:
-    use_db: false
-    db_path: "LinkPulse.db"
+    use_db: true
+    dialect: "mysql"
+    username: "root"
+    password: "xxxxxx"
+    host: "127.0.0.1"
+    port: "3306" 
+    db_name: "webprobe" 
+    ssl_mode: ""
+

config/config.go

@@ -1,8 +1,10 @@
 package config
 
 import (
-	"github.com/spf13/viper"
+	"log"
 	"time"
+
+	"github.com/spf13/viper"
 )
 
 type Config struct {
@@ -40,15 +42,23 @@ type Push struct {
 	PushGatewayURL   string `mapstructure:"push_gateway_url"`
 }
 type DBConfig struct {
-	UseDB  bool   `mapstructure:"use_db"`  // default value: true
-	DBPath string `mapstructure:"db_path"` // default value: "url_status.db"
+	UseDB    bool   `mapstructure:"use_db"`   // 是否使用数据库，默认值为 true
+	Dialect  string `mapstructure:"dialect"`  // 数据库方言（例如：mysql、postgres）
+	Username string `mapstructure:"username"` // 数据库用户名
+	Password string `mapstructure:"password"` // 数据库密码
+	Host     string `mapstructure:"host"`     // 数据库主机地址
+	Port     int    `mapstructure:"port"`     // 数据库连接端口号
+	DBName   string `mapstructure:"db_name"`  // 数据库名称
+	SSLMode  string `mapstructure:"ssl_mode"` // SSL 模式（仅适用于某些数据库，如 PostgreSQL）
 }
 
-func LoadConfig() (*Config, error) {
+var ConfigPath string
+
+func LoadConfig(configPath string) (*Config, error) {
+	log.Println(ConfigPath)
 	viper.SetConfigName("config")
 	viper.SetConfigType("yaml")
-	viper.AddConfigPath(".")
-
+	viper.AddConfigPath(configPath)
 	viper.AutomaticEnv() // read from environment variables
 
 	// 设置默认值

go.mod

@@ -6,16 +6,27 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.19
 	github.com/prometheus/client_golang v1.17.0
 	github.com/spf13/viper v1.18.1
-	golang.org/x/net v0.19.0
-	golang.org/x/sync v0.5.0
+	golang.org/x/net v0.21.0
+	golang.org/x/sync v0.7.0
 )
 
 require (
+	filippo.io/edwards25519 v1.1.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-sql-driver/mysql v1.8.1 // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/gookit/color v1.5.4 // indirect
+	github.com/gookit/goutil v0.6.15 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20231201235250-de7065d80cb9 // indirect
+	github.com/jackc/pgx/v5 v5.5.5 // indirect
+	github.com/jackc/puddle/v2 v2.2.1 // indirect
+	github.com/jinzhu/inflection v1.0.0 // indirect
+	github.com/jinzhu/now v1.1.5 // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
@@ -28,14 +39,20 @@ require (
 	github.com/sourcegraph/conc v0.3.0 // indirect
 	github.com/spf13/afero v1.11.0 // indirect
 	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/cobra v1.8.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/crypto v0.22.0 // indirect
 	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.19.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gorm.io/driver/mysql v1.5.6 // indirect
+	gorm.io/driver/postgres v1.5.7 // indirect
+	gorm.io/gorm v1.25.10 // indirect
 )