From bbd5f23d4a530e1f264433c22ef6cf27a5ebf6b3 Mon Sep 17 00:00:00 2001 From: haijima Date: Mon, 10 Sep 2018 01:54:15 +0900 Subject: [PATCH 1/5] Create command --- kadai3-2/haijima/.gitignore | 130 ++++++++++++++++++++++++++++++++++ kadai3-2/haijima/main.go | 135 ++++++++++++++++++++++++++++++++++++ 2 files changed, 265 insertions(+) create mode 100644 kadai3-2/haijima/.gitignore create mode 100644 kadai3-2/haijima/main.go diff --git a/kadai3-2/haijima/.gitignore b/kadai3-2/haijima/.gitignore new file mode 100644 index 0000000..8427f66 --- /dev/null +++ b/kadai3-2/haijima/.gitignore @@ -0,0 +1,130 @@ + +# Created by https://www.gitignore.io/api/go,macos,jetbrains+all + +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, build with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +### Go Patch ### +/vendor/ +/Godeps/ + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and WebStorm +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/modules.xml +# .idea/*.iml +# .idea/modules + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +### JetBrains+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + + +# End of https://www.gitignore.io/api/go,macos,jetbrains+all + diff --git a/kadai3-2/haijima/main.go b/kadai3-2/haijima/main.go new file mode 100644 index 0000000..79220f6 --- /dev/null +++ b/kadai3-2/haijima/main.go @@ -0,0 +1,135 @@ +package main + +import ( + "context" + "flag" + "fmt" + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "strconv" +) + +func main() { + var ( + dir = flag.String("o", ".", "output directory") + num = flag.Int("n", 3, "num of worker") + ) + flag.Parse() + + url := flag.Arg(0) + filename := path.Join(*dir, path.Base(url)) + file, err := os.Create(filename) + if err != nil { + fmt.Fprintf(os.Stderr, "error %+v\n", err) + } + defer file.Close() + + err = Exec(url, file, *num) + if err != nil { + fmt.Fprintf(os.Stderr, "error %+v\n", err) + os.Remove(filename) + } +} + +func Exec(url string, w io.Writer, num int) error { + length, ok := acceptsRangeRequest(url) + if num > 1 && ok { + b := length/num + 1 + eg, ctx := errgroup.WithContext(context.Background()) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Range access + tmpFiles := make([]string, num) + for i := 0; i < num; i++ { + i := i + eg.Go(func() error { + tmpFile, err := ioutil.TempFile("", path.Base(url)) + if err != nil { + return errors.WithStack(err) + } + defer tmpFile.Close() + tmpFiles[i] = tmpFile.Name() + header := map[string]string{"Range": fmt.Sprintf("bytes=%v-%v", i*b, (i+1)*b-1)} + return download(url, header, tmpFile) + }) + } + if err := eg.Wait(); err != nil { + cancel() + return errors.WithStack(err) + } + + // Concatenate partial files + return concat(w, tmpFiles) + } else { + return download(url, nil, w) + } +} + +func acceptsRangeRequest(url string) (int, bool) { + req, err := http.NewRequest("HEAD", url, nil) + if err != nil { + return 0, false + } + resp, err := new(http.Client).Do(req) + if err != nil { + return 0, false + } + defer resp.Body.Close() + unit := resp.Header.Get("Accept-Ranges") + length, err := strconv.Atoi(resp.Header.Get("Content-Length")) + if err != nil { + return 0, false + } + return length, unit == "bytes" +} + +func download(url string, h map[string]string, w io.Writer) error { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return errors.WithStack(err) + } + for k, v := range h { + req.Header.Set(k, v) + } + resp, err := new(http.Client).Do(req) + if err != nil { + return errors.WithStack(err) + } + defer resp.Body.Close() + + _, err = io.Copy(w, resp.Body) + if err != nil { + return errors.WithStack(err) + } + return nil +} + +// TODO: io.Reader使いたい +func concat(dst io.Writer, srcs []string) error { + for _, src := range srcs { + err := func() error { + part, err := os.Open(src) + if err != nil { + return errors.WithStack(err) + } + defer os.Remove(src) + defer part.Close() + + _, err = io.Copy(dst, part) + if err != nil { + return errors.WithStack(err) + } + return nil + }() + if err != nil { + return errors.WithStack(err) + } + } + return nil +} From 6880630f9af452266e86f2272f27e9ca97ddeca6 Mon Sep 17 00:00:00 2001 From: haijima Date: Mon, 10 Sep 2018 02:05:19 +0900 Subject: [PATCH 2/5] Add README and Makefile --- kadai3-2/haijima/Makefile | 3 +++ kadai3-2/haijima/README.md | 23 +++++++++++++++++++++++ 2 files changed, 26 insertions(+) create mode 100644 kadai3-2/haijima/Makefile create mode 100644 kadai3-2/haijima/README.md diff --git a/kadai3-2/haijima/Makefile b/kadai3-2/haijima/Makefile new file mode 100644 index 0000000..da6db7d --- /dev/null +++ b/kadai3-2/haijima/Makefile @@ -0,0 +1,3 @@ +build: + go build -o gget main.go + diff --git a/kadai3-2/haijima/README.md b/kadai3-2/haijima/README.md new file mode 100644 index 0000000..70146b9 --- /dev/null +++ b/kadai3-2/haijima/README.md @@ -0,0 +1,23 @@ +# 課題3-2 +## 要求 +分割ダウンロードを行う +* [x] Rangeアクセスを用いる +* [x] いくつかのゴルーチンでダウンロードしてマージする +* [x] エラー処理を工夫する + * golang.org/x/sync/errgourpパッケージなどを使ってみる +* [x] キャンセルが発生した場合の実装を行う + +### できてないこと +* [ ] テスト + + +## How to build +``` +$ make +``` + + +## How to run +``` +$ ./gget [-n num] [-o outputdir] url +``` From ad339d9700d62f3a460a3e9c72f00c724058f6c5 Mon Sep 17 00:00:00 2001 From: haijima Date: Mon, 10 Sep 2018 13:46:44 +0900 Subject: [PATCH 3/5] Refactor --- kadai3-2/haijima/main.go | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/kadai3-2/haijima/main.go b/kadai3-2/haijima/main.go index 79220f6..7f42f38 100644 --- a/kadai3-2/haijima/main.go +++ b/kadai3-2/haijima/main.go @@ -72,11 +72,7 @@ func Exec(url string, w io.Writer, num int) error { } func acceptsRangeRequest(url string) (int, bool) { - req, err := http.NewRequest("HEAD", url, nil) - if err != nil { - return 0, false - } - resp, err := new(http.Client).Do(req) + resp, err := http.Head(url) if err != nil { return 0, false } @@ -97,7 +93,7 @@ func download(url string, h map[string]string, w io.Writer) error { for k, v := range h { req.Header.Set(k, v) } - resp, err := new(http.Client).Do(req) + resp, err := http.DefaultClient.Do(req) if err != nil { return errors.WithStack(err) } From 88d8337b2e164dde67e62b811c7e44dd09509857 Mon Sep 17 00:00:00 2001 From: haijima Date: Mon, 10 Sep 2018 15:01:19 +0900 Subject: [PATCH 4/5] Fix around context --- kadai3-2/haijima/main.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/kadai3-2/haijima/main.go b/kadai3-2/haijima/main.go index 7f42f38..429ac31 100644 --- a/kadai3-2/haijima/main.go +++ b/kadai3-2/haijima/main.go @@ -41,8 +41,6 @@ func Exec(url string, w io.Writer, num int) error { if num > 1 && ok { b := length/num + 1 eg, ctx := errgroup.WithContext(context.Background()) - ctx, cancel := context.WithCancel(ctx) - defer cancel() // Range access tmpFiles := make([]string, num) @@ -56,18 +54,18 @@ func Exec(url string, w io.Writer, num int) error { defer tmpFile.Close() tmpFiles[i] = tmpFile.Name() header := map[string]string{"Range": fmt.Sprintf("bytes=%v-%v", i*b, (i+1)*b-1)} - return download(url, header, tmpFile) + return download(ctx, url, header, tmpFile) }) } if err := eg.Wait(); err != nil { - cancel() return errors.WithStack(err) } // Concatenate partial files return concat(w, tmpFiles) } else { - return download(url, nil, w) + ctx, _ := context.WithCancel(context.Background()) + return download(ctx, url, nil, w) } } @@ -85,7 +83,7 @@ func acceptsRangeRequest(url string) (int, bool) { return length, unit == "bytes" } -func download(url string, h map[string]string, w io.Writer) error { +func download(ctx context.Context, url string, h map[string]string, w io.Writer) error { req, err := http.NewRequest("GET", url, nil) if err != nil { return errors.WithStack(err) @@ -93,7 +91,7 @@ func download(url string, h map[string]string, w io.Writer) error { for k, v := range h { req.Header.Set(k, v) } - resp, err := http.DefaultClient.Do(req) + resp, err := http.DefaultClient.Do(req.WithContext(ctx)) if err != nil { return errors.WithStack(err) } From f1dfaeabc31140007372ea70f10b63a7f76487fb Mon Sep 17 00:00:00 2001 From: haijima Date: Mon, 10 Sep 2018 15:18:18 +0900 Subject: [PATCH 5/5] Fix interface to use io.Reader --- kadai3-2/haijima/main.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/kadai3-2/haijima/main.go b/kadai3-2/haijima/main.go index 429ac31..6f26096 100644 --- a/kadai3-2/haijima/main.go +++ b/kadai3-2/haijima/main.go @@ -43,7 +43,15 @@ func Exec(url string, w io.Writer, num int) error { eg, ctx := errgroup.WithContext(context.Background()) // Range access - tmpFiles := make([]string, num) + tmpFiles := make([]io.Reader, num) + defer func() { + for _, tmpFile := range tmpFiles { + if f, ok := tmpFile.(io.ReadCloser); ok { + f.Close() + } + } + }() + for i := 0; i < num; i++ { i := i eg.Go(func() error { @@ -51,8 +59,7 @@ func Exec(url string, w io.Writer, num int) error { if err != nil { return errors.WithStack(err) } - defer tmpFile.Close() - tmpFiles[i] = tmpFile.Name() + tmpFiles[i] = tmpFile header := map[string]string{"Range": fmt.Sprintf("bytes=%v-%v", i*b, (i+1)*b-1)} return download(ctx, url, header, tmpFile) }) @@ -104,18 +111,10 @@ func download(ctx context.Context, url string, h map[string]string, w io.Writer) return nil } -// TODO: io.Reader使いたい -func concat(dst io.Writer, srcs []string) error { +func concat(dst io.Writer, srcs []io.Reader) error { for _, src := range srcs { err := func() error { - part, err := os.Open(src) - if err != nil { - return errors.WithStack(err) - } - defer os.Remove(src) - defer part.Close() - - _, err = io.Copy(dst, part) + _, err := io.Copy(dst, src) if err != nil { return errors.WithStack(err) }