add dfsclient

Author: sjqzhang

README.md

@@ -10,7 +10,9 @@
 
 ### 大家担心的是这么简单的文件系统，靠不靠谱，可不可以用于生产环境？答案是肯定的，正因为简单所以高效，因为简单所以稳定。如果你担心功能，那就跑单元测试，如果担心性能，那就跑压力测试，项目都自带了，跑一跑更放心^_^。
 
-注意：使用前请认真阅读[使用文档](https://sjqzhang.github.io/go-fastdfs/#character)。
+注意：使用前请认真阅读[使用文档](https://sjqzhang.github.io/go-fastdfs/#character) 或 [视频教程](https://www.bilibili.com/video/av92526484)。
+
+
 
 - 支持curl命令上传
 - 支持浏览器上传
@@ -86,3 +88,9 @@ http://yourserver ip:8080 注意：不要使用127.0.0.1上传
 
 ### [视频教程](https://www.bilibili.com/video/av92526484)
 
+
+#### 如果你觉得本项目不错，请点击项目顶部的 `star` 按钮关注本项目
+
+> QQ交流群：964274270(go-fastdfs技术交流群)
+
+同时要说明的是，该群是一个学习交流群，如果是程序相关问题，请直接提交issues，不接受邮件求助、微信求助和QQ私信求助

doc/benchmark.go

@@ -8,8 +8,7 @@ import (
 	"github.com/sjqzhang/goutil"
 	"io/ioutil"
 	"os"
-	"os/exec"
-	"strings"
+	"path/filepath"
 	"sync"
 	"time"
 )
@@ -30,15 +29,17 @@ func init()  {
 	util=goutil.Common{}
 }
 
-func getDir(dir string) []string  {
-    cmd:=exec.Command("find",dir,"-type","f")
-	var out bytes.Buffer
-	cmd.Stdout = &out
-	if err:=cmd.Run();err!=nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	return strings.Split( out.String(),"\n")
+func getDir(dir string) []string {
+	var (
+		paths []string
+	)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			paths = append(paths, path)
+		}
+		return nil
+	})
+	return paths
 }
 
 func sendFile()  {

doc/dfsclient.go

@@ -0,0 +1,215 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"github.com/astaxie/beego/httplib"
+	"github.com/eventials/go-tus"
+	"github.com/sjqzhang/goutil"
+	"github.com/syndtr/goleveldb/leveldb"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+	"time"
+)
+
+var url *string
+var dir *string
+var worker *int
+var queue chan string
+
+var store tus.Store
+
+//var filesize *int
+//var filecount *int
+//var retry *int
+//var gen *bool
+var scene *string
+var done chan bool = make(chan bool, 1)
+var wg sync.WaitGroup = sync.WaitGroup{}
+var util goutil.Common
+
+type LeveldbStore struct {
+	db *leveldb.DB
+}
+
+func NewLeveldbStore(path string) (tus.Store, error) {
+	db, err := leveldb.OpenFile(path, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	store := &LeveldbStore{db: db}
+	return store, err
+}
+
+func (s *LeveldbStore) Get(fingerprint string) (string, bool) {
+	url, err := s.db.Get([]byte(fingerprint), nil)
+	ok := true
+	if err != nil {
+		ok = false
+	}
+	return string(url), ok
+}
+
+func (s *LeveldbStore) Set(fingerprint, url string) {
+	s.db.Put([]byte(fingerprint), []byte(url), nil)
+}
+
+func (s *LeveldbStore) Delete(fingerprint string) {
+	s.db.Delete([]byte(fingerprint), nil)
+}
+
+func (s *LeveldbStore) Close() {
+	s.Close()
+}
+
+func init() {
+	util = goutil.Common{}
+	defaultTransport := &http.Transport{
+		DisableKeepAlives:   true,
+		Dial:                httplib.TimeoutDialer(time.Second*15, time.Second*300),
+		MaxIdleConns:        100,
+		MaxIdleConnsPerHost: 100,
+	}
+	settins := httplib.BeegoHTTPSettings{
+		UserAgent:        "Go-FastDFS",
+		ConnectTimeout:   15 * time.Second,
+		ReadWriteTimeout: 15 * time.Second,
+		Gzip:             true,
+		DumpBody:         true,
+		Transport:        defaultTransport,
+	}
+	httplib.SetDefaultSetting(settins)
+
+	//store,_=NewLeveldbStore("upload.db")
+
+}
+
+func getDir(dir string) []string {
+	var (
+		paths []string
+	)
+	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if !info.IsDir() {
+			paths = append(paths, path)
+		}
+		return nil
+	})
+	return paths
+}
+
+func sendFile() {
+
+	for {
+		if len(queue) <= 0 {
+			return
+		}
+		filePath := <-queue
+		if strings.Index(*url, "/big/upload") > 0 {
+			bigUpload(filePath)
+		} else {
+			normalUpload(filePath)
+		}
+		wg.Done()
+	}
+
+}
+
+func normalUpload(filePath string) {
+	defer func() {
+		if re := recover(); re != nil {
+		}
+	}()
+	req := httplib.Post(*url)
+	req.PostFile("file", filePath) //注意不是全路径
+	req.Param("output", "text")
+	req.Param("scene", "")
+	path := strings.Replace(filePath, *dir, "", 1)
+	filename := filepath.Base(path)
+	path = strings.Replace(filepath.Dir(path), "\\", "/", -1)
+	req.Param("path", *scene+"/"+strings.TrimLeft(path, "/"))
+	req.Param("filename", filename)
+	req.Retries(-1)
+	if s, err := req.String(); err != nil {
+		fmt.Println(err, filePath)
+	} else {
+		fmt.Println(s, filePath)
+	}
+
+}
+
+func bigUpload(filePath string) {
+	defer func() {
+		if re := recover(); re != nil {
+		}
+	}()
+	f, err := os.Open(filePath)
+	if err != nil {
+
+		panic(err)
+	}
+	defer f.Close()
+	cfg := tus.DefaultConfig()
+	//cfg.Store=store
+	//cfg.Resume=true
+	client, err := tus.NewClient(*url, cfg)
+	if err != nil {
+		fmt.Println(err)
+	}
+	upload, err := tus.NewUploadFromFile(f)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	uploader, err := client.CreateOrResumeUpload(upload)
+	if err != nil {
+		fmt.Println(err)
+		return
+	}
+	url := uploader.Url()
+	err = uploader.Upload()
+	fmt.Println(url, filePath)
+
+}
+
+func startWorker() {
+	defer func() {
+		if re := recover(); re != nil {
+		}
+	}()
+	for i := 0; i < *worker; i++ {
+		go sendFile()
+	}
+}
+
+func main() {
+
+	url = flag.String("url", "http://127.0.0.1:8080/group1/upload", "url")
+	dir = flag.String("dir", "./", "dir to upload")
+	worker = flag.Int("worker", 100, "num of worker")
+	scene = flag.String("scene", "default", "scene")
+	//retry=flag.Int("retry", -1, "retry times when fail")
+	//uploadPath=flag.String("uploadPath", "./", "upload path")
+	//filesize=flag.Int("filesize", 1024*1024, "file of size")
+	//filecount=flag.Int("filecount", 1000000, "file of count")
+	//gen=flag.Bool("gen", false, "gen file")
+	flag.Parse()
+	st := time.Now()
+	//if *gen {
+	//	genFile()
+	//	fmt.Println(time.Since(st))
+	//	os.Exit(0)
+	//}
+	files := getDir(*dir)
+	wg.Add(len(files))
+	queue = make(chan string, len(files))
+	for i := 0; i < len(files); i++ {
+		queue <- files[i]
+	}
+	startWorker()
+	wg.Wait()
+	fmt.Println(time.Since(st))
+}

fileserver.go

@@ -3462,6 +3462,7 @@ func (this *Server) GenGoogleSecret(w http.ResponseWriter, r *http.Request) {
 	if !this.IsPeer(r) {
 		result.Message = this.GetClusterNotPermitMessage(r)
 		w.Write([]byte(this.util.JsonEncodePretty(result)))
+		return
 	}
 	GetSeed := func(length int) string {
 		seeds := "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"