From 08554dccd0e2eb72af109dc74cefab8c7418b944 Mon Sep 17 00:00:00 2001 From: beerpiss Date: Thu, 26 May 2022 08:34:23 +0700 Subject: [PATCH 1/5] feat(serve): file watch --- cmd/aidoku/cmd/serve.go | 51 +++- go.mod | 1 + go.sum | 3 + internal/watcher/batcher.go | 87 ++++++ internal/watcher/filenotify/filenotify.go | 49 ++++ internal/watcher/filenotify/fsnotify.go | 20 ++ internal/watcher/filenotify/poller.go | 326 ++++++++++++++++++++++ 7 files changed, 536 insertions(+), 1 deletion(-) create mode 100644 internal/watcher/batcher.go create mode 100644 internal/watcher/filenotify/filenotify.go create mode 100644 internal/watcher/filenotify/fsnotify.go create mode 100644 internal/watcher/filenotify/poller.go diff --git a/cmd/aidoku/cmd/serve.go b/cmd/aidoku/cmd/serve.go index 5af589a..4f5b01a 100644 --- a/cmd/aidoku/cmd/serve.go +++ b/cmd/aidoku/cmd/serve.go @@ -5,14 +5,18 @@ import ( "net/http" "os" "os/signal" + "strings" + "sync" "syscall" "time" "github.com/Aidoku/aidoku-cli/internal/build" "github.com/Aidoku/aidoku-cli/internal/common" + "github.com/Aidoku/aidoku-cli/internal/watcher" "github.com/fatih/color" "github.com/felixge/httpsnoop" "github.com/spf13/cobra" + "golang.org/x/exp/maps" ) var ( @@ -42,8 +46,10 @@ var serveCmd = &cobra.Command{ address, _ := cmd.Flags().GetString("address") output, _ := cmd.Flags().GetString("output") port, _ := cmd.Flags().GetString("port") + watch, _ := cmd.Flags().GetBool("watch") - build.BuildWrapper(args, output) + files := common.ProcessGlobs(args) + build.BuildWrapper(files, output) fmt.Println("Listening on these addresses:") if address == "0.0.0.0" { @@ -69,12 +75,55 @@ var serveCmd = &cobra.Command{ fmt.Printf("[%s] \"%s %s\" Error (%s): \"%s\"\n", timestamp, red(method), red(url), red(statusCode), red(http.StatusText(statusCode))) } }) + if watch { + watcher, err := watcher.New(500*time.Millisecond, 500*time.Millisecond, false) + var buildLock sync.Mutex + if err != nil { + color.Red("error: Couldn't create file watcher. Not watching for changes.") + color.Red("details: %s", err) + } else { + defer watcher.Close() + go func() { + for { + select { + case events, ok := <-watcher.Events: + if !ok { + return + } + changed := make(map[string]string) + for _, event := range events { + if _, ok := changed[event.Name]; !ok { + changed[event.Name] = "" + } + } + color.HiBlack("File changed, rebuilding source list: %s", strings.Join(maps.Keys(changed), ", ")) + buildLock.Lock() + build.BuildWrapper(files, output) + buildLock.Unlock() + case err, ok := <-watcher.Errors(): + if !ok { + return + } + color.Red("file watcher error: %s", err) + } + } + }() + for _, file := range files { + err = watcher.Add(file) + if err != nil { + color.Red("error: could not watch %s: %s", file, err) + } + } + } + fmt.Printf("Watching %d file(s) for changes\n", len(files)) + } return http.ListenAndServe(address+":"+port, wrappedHandler) }, } func init() { rootCmd.AddCommand(serveCmd) + serveCmd.Flags().BoolP("watch", "w", false, "Watch files for changes") serveCmd.Flags().StringP("address", "a", "0.0.0.0", "Address to broadcast source list") serveCmd.Flags().StringP("port", "p", "8080", "The port to broadcast the source list on") serveCmd.Flags().StringP("output", "o", "public", "The source list folder") diff --git a/go.mod b/go.mod index 1441294..b0d3f03 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf golang.org/x/text v0.3.7 + github.com/fsnotify/fsnotify v1.5.4 ) require ( diff --git a/go.sum b/go.sum index 961f2d6..b276fa4 100644 --- a/go.sum +++ b/go.sum @@ -21,6 +21,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= @@ -75,6 +77,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20210503060354-a79de5458b56/go.mod h1:tfny5GFUkzUvx4ps4ajbZsCe5lw1metzhBm9T3x7oIY= diff --git a/internal/watcher/batcher.go b/internal/watcher/batcher.go new file mode 100644 index 0000000..9424ccd --- /dev/null +++ b/internal/watcher/batcher.go @@ -0,0 +1,87 @@ +// Copyright 2020 The Hugo Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watcher + +import ( + "time" + + "github.com/Aidoku/aidoku-cli/internal/watcher/filenotify" + "github.com/fsnotify/fsnotify" +) + +// Batcher batches file watch events in a given interval. +type Batcher struct { + filenotify.FileWatcher + interval time.Duration + done chan struct{} + + Events chan []fsnotify.Event // Events are returned on this channel +} + +// New creates and starts a Batcher with the given time interval. +// It will fall back to a poll based watcher if native isn's supported. +// To always use polling, set poll to true. +func New(intervalBatcher, intervalPoll time.Duration, poll bool) (*Batcher, error) { + var err error + var watcher filenotify.FileWatcher + + if poll { + watcher = filenotify.NewPollingWatcher(intervalPoll) + } else { + watcher, err = filenotify.New(intervalPoll) + } + + if err != nil { + return nil, err + } + + batcher := &Batcher{} + batcher.FileWatcher = watcher + batcher.interval = intervalBatcher + batcher.done = make(chan struct{}, 1) + batcher.Events = make(chan []fsnotify.Event, 1) + + if err == nil { + go batcher.run() + } + + return batcher, nil +} + +func (b *Batcher) run() { + tick := time.NewTicker(b.interval) + evs := make([]fsnotify.Event, 0) +OuterLoop: + for { + select { + case ev := <-b.FileWatcher.Events(): + evs = append(evs, ev) + case <-tick.C: + if len(evs) == 0 { + continue + } + b.Events <- evs + evs = make([]fsnotify.Event, 0) + case <-b.done: + break OuterLoop + } + } + close(b.done) +} + +// Close stops the watching of the files. +func (b *Batcher) Close() { + b.done <- struct{}{} + b.FileWatcher.Close() +} diff --git a/internal/watcher/filenotify/filenotify.go b/internal/watcher/filenotify/filenotify.go new file mode 100644 index 0000000..b9d0d2e --- /dev/null +++ b/internal/watcher/filenotify/filenotify.go @@ -0,0 +1,49 @@ +// Package filenotify provides a mechanism for watching file(s) for changes. +// Generally leans on fsnotify, but provides a poll-based notifier which fsnotify does not support. +// These are wrapped up in a common interface so that either can be used interchangeably in your code. +// +// This package is adapted from https://github.com/moby/moby/tree/master/pkg/filenotify, Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package filenotify + +import ( + "time" + + "github.com/fsnotify/fsnotify" +) + +// FileWatcher is an interface for implementing file notification watchers +type FileWatcher interface { + Events() <-chan fsnotify.Event + Errors() <-chan error + Add(name string) error + Remove(name string) error + Close() error +} + +// New tries to use an fs-event watcher, and falls back to the poller if there is an error +func New(interval time.Duration) (FileWatcher, error) { + if watcher, err := NewEventWatcher(); err == nil { + return watcher, nil + } + return NewPollingWatcher(interval), nil +} + +// NewPollingWatcher returns a poll-based file watcher +func NewPollingWatcher(interval time.Duration) FileWatcher { + return &filePoller{ + interval: interval, + done: make(chan struct{}), + events: make(chan fsnotify.Event), + errors: make(chan error), + } +} + +// NewEventWatcher returns an fs-event based file watcher +func NewEventWatcher() (FileWatcher, error) { + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + return &fsNotifyWatcher{watcher}, nil +} diff --git a/internal/watcher/filenotify/fsnotify.go b/internal/watcher/filenotify/fsnotify.go new file mode 100644 index 0000000..1953412 --- /dev/null +++ b/internal/watcher/filenotify/fsnotify.go @@ -0,0 +1,20 @@ +// Package filenotify is adapted from https://github.com/moby/moby/tree/master/pkg/filenotify, Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package filenotify + +import "github.com/fsnotify/fsnotify" + +// fsNotifyWatcher wraps the fsnotify package to satisfy the FileNotifier interface +type fsNotifyWatcher struct { + *fsnotify.Watcher +} + +// Events returns the fsnotify event channel receiver +func (w *fsNotifyWatcher) Events() <-chan fsnotify.Event { + return w.Watcher.Events +} + +// Errors returns the fsnotify error channel receiver +func (w *fsNotifyWatcher) Errors() <-chan error { + return w.Watcher.Errors +} diff --git a/internal/watcher/filenotify/poller.go b/internal/watcher/filenotify/poller.go new file mode 100644 index 0000000..e39f0ce --- /dev/null +++ b/internal/watcher/filenotify/poller.go @@ -0,0 +1,326 @@ +// Package filenotify is adapted from https://github.com/moby/moby/tree/master/pkg/filenotify, Apache-2.0 License. +// Hopefully this can be replaced with an external package sometime in the future, see https://github.com/fsnotify/fsnotify/issues/9 +package filenotify + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +var ( + // errPollerClosed is returned when the poller is closed + errPollerClosed = errors.New("poller is closed") + // errNoSuchWatch is returned when trying to remove a watch that doesn't exist + errNoSuchWatch = errors.New("watch does not exist") +) + +// filePoller is used to poll files for changes, especially in cases where fsnotify +// can't be run (e.g. when inotify handles are exhausted) +// filePoller satisfies the FileWatcher interface +type filePoller struct { + // the duration between polls. + interval time.Duration + // watches is the list of files currently being polled, close the associated channel to stop the watch + watches map[string]struct{} + // Will be closed when done. + done chan struct{} + // events is the channel to listen to for watch events + events chan fsnotify.Event + // errors is the channel to listen to for watch errors + errors chan error + // mu locks the poller for modification + mu sync.Mutex + // closed is used to specify when the poller has already closed + closed bool +} + +// Add adds a filename to the list of watches +// once added the file is polled for changes in a separate goroutine +func (w *filePoller) Add(name string) error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.closed { + return errPollerClosed + } + + item, err := newItemToWatch(name) + if err != nil { + return err + } + if item.left.FileInfo == nil { + return os.ErrNotExist + } + + if w.watches == nil { + w.watches = make(map[string]struct{}) + } + if _, exists := w.watches[name]; exists { + return fmt.Errorf("watch exists") + } + w.watches[name] = struct{}{} + + go w.watch(item) + return nil +} + +// Remove stops and removes watch with the specified name +func (w *filePoller) Remove(name string) error { + w.mu.Lock() + defer w.mu.Unlock() + return w.remove(name) +} + +func (w *filePoller) remove(name string) error { + if w.closed { + return errPollerClosed + } + + _, exists := w.watches[name] + if !exists { + return errNoSuchWatch + } + delete(w.watches, name) + return nil +} + +// Events returns the event channel +// This is used for notifications on events about watched files +func (w *filePoller) Events() <-chan fsnotify.Event { + return w.events +} + +// Errors returns the errors channel +// This is used for notifications about errors on watched files +func (w *filePoller) Errors() <-chan error { + return w.errors +} + +// Close closes the poller +// All watches are stopped, removed, and the poller cannot be added to +func (w *filePoller) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + + if w.closed { + return nil + } + w.closed = true + close(w.done) + for name := range w.watches { + w.remove(name) + } + + return nil +} + +// sendEvent publishes the specified event to the events channel +func (w *filePoller) sendEvent(e fsnotify.Event) error { + select { + case w.events <- e: + case <-w.done: + return fmt.Errorf("closed") + } + return nil +} + +// sendErr publishes the specified error to the errors channel +func (w *filePoller) sendErr(e error) error { + select { + case w.errors <- e: + case <-w.done: + return fmt.Errorf("closed") + } + return nil +} + +// watch watches item for changes until done is closed. +func (w *filePoller) watch(item *itemToWatch) { + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + case <-w.done: + return + } + + evs, err := item.checkForChanges() + if err != nil { + if err := w.sendErr(err); err != nil { + return + } + } + + item.left, item.right = item.right, item.left + + for _, ev := range evs { + if err := w.sendEvent(ev); err != nil { + return + } + } + + } +} + +// recording records the state of a file or a dir. +type recording struct { + os.FileInfo + + // Set if FileInfo is a dir. + entries map[string]os.FileInfo +} + +func (r *recording) clear() { + r.FileInfo = nil + if r.entries != nil { + for k := range r.entries { + delete(r.entries, k) + } + } +} + +func (r *recording) record(filename string) error { + r.clear() + + fi, err := os.Stat(filename) + if err != nil && !os.IsNotExist(err) { + return err + } + + if fi == nil { + return nil + } + + r.FileInfo = fi + + // If fi is a dir, we watch the files inside that directory (not recursively). + // This matches the behaviour of fsnotity. + if fi.IsDir() { + f, err := os.Open(filename) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + defer f.Close() + + fis, err := f.Readdir(-1) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + + for _, fi := range fis { + r.entries[fi.Name()] = fi + } + } + + return nil +} + +// itemToWatch may be a file or a dir. +type itemToWatch struct { + // Full path to the filename. + filename string + + // Snapshots of the stat state of this file or dir. + left *recording + right *recording +} + +func newItemToWatch(filename string) (*itemToWatch, error) { + r := &recording{ + entries: make(map[string]os.FileInfo), + } + err := r.record(filename) + if err != nil { + return nil, err + } + + return &itemToWatch{filename: filename, left: r}, nil + +} + +func (item *itemToWatch) checkForChanges() ([]fsnotify.Event, error) { + if item.right == nil { + item.right = &recording{ + entries: make(map[string]os.FileInfo), + } + } + + err := item.right.record(item.filename) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + + dirOp := checkChange(item.left.FileInfo, item.right.FileInfo) + + if dirOp != 0 { + evs := []fsnotify.Event{{Op: dirOp, Name: item.filename}} + return evs, nil + } + + if item.left.FileInfo == nil || !item.left.IsDir() { + // Done. + return nil, nil + } + + leftIsIn := false + left, right := item.left.entries, item.right.entries + if len(right) > len(left) { + left, right = right, left + leftIsIn = true + } + + var evs []fsnotify.Event + + for name, fi1 := range left { + fi2 := right[name] + fil, fir := fi1, fi2 + if leftIsIn { + fil, fir = fir, fil + } + op := checkChange(fil, fir) + if op != 0 { + evs = append(evs, fsnotify.Event{Op: op, Name: filepath.Join(item.filename, name)}) + } + + } + + return evs, nil + +} + +func checkChange(fi1, fi2 os.FileInfo) fsnotify.Op { + if fi1 == nil && fi2 != nil { + return fsnotify.Create + } + if fi1 != nil && fi2 == nil { + return fsnotify.Remove + } + if fi1 == nil && fi2 == nil { + return 0 + } + if fi1.IsDir() || fi2.IsDir() { + return 0 + } + if fi1.Mode() != fi2.Mode() { + return fsnotify.Chmod + } + if fi1.ModTime() != fi2.ModTime() || fi1.Size() != fi2.Size() { + return fsnotify.Write + } + + return 0 +} From 348493497b5121a386e8a694aa14fa6a5532f4bd Mon Sep 17 00:00:00 2001 From: beerpiss Date: Thu, 26 May 2022 08:35:18 +0700 Subject: [PATCH 2/5] chore(internal/watcher): add license --- internal/watcher/LICENSE | 201 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 201 insertions(+) create mode 100644 internal/watcher/LICENSE diff --git a/internal/watcher/LICENSE b/internal/watcher/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/internal/watcher/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From 4f217471592621b1e418cf979b0b3cffd5b9542e Mon Sep 17 00:00:00 2001 From: beerpiss Date: Thu, 26 May 2022 08:52:53 +0700 Subject: [PATCH 3/5] feat(serve): add poll based watching --- cmd/aidoku/cmd/serve.go | 19 ++++++++++++++----- go.mod | 1 + go.sum | 2 ++ internal/common/common.go | 13 +++++++++++++ 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/cmd/aidoku/cmd/serve.go b/cmd/aidoku/cmd/serve.go index 4f5b01a..2b9f6f5 100644 --- a/cmd/aidoku/cmd/serve.go +++ b/cmd/aidoku/cmd/serve.go @@ -47,6 +47,7 @@ var serveCmd = &cobra.Command{ output, _ := cmd.Flags().GetString("output") port, _ := cmd.Flags().GetString("port") watch, _ := cmd.Flags().GetBool("watch") + poll, _ := cmd.Flags().GetString("poll") files := common.ProcessGlobs(args) build.BuildWrapper(files, output) @@ -75,12 +76,19 @@ var serveCmd = &cobra.Command{ fmt.Printf("[%s] \"%s %s\" Error (%s): \"%s\"\n", timestamp, red(method), red(url), red(statusCode), red(http.StatusText(statusCode))) } }) - if watch { - watcher, err := watcher.New(500*time.Millisecond, 500*time.Millisecond, false) + if watch || len(poll) > 0 { + var pollInterval time.Duration + var err error + if len(poll) > 0 { + pollInterval, err = common.ToDurationE(poll) + if err != nil { + return fmt.Errorf("error: invalid value for --poll: %s", err) + } + } + watcher, err := watcher.New(500*time.Millisecond, pollInterval, len(poll) > 0) var buildLock sync.Mutex if err != nil { - color.Red("error: Couldn't create file watcher. Not watching for changes.") - color.Red("details: %s", err) + color.Red("error: couldn't create file watcher, not watching for changes: %s", err) } else { defer watcher.Close() go func() { @@ -104,7 +112,7 @@ var serveCmd = &cobra.Command{ if !ok { return } - color.Red("file watcher error: %s", err) + color.Red("error: file watcher error: %s", err) } } }() @@ -124,6 +132,7 @@ var serveCmd = &cobra.Command{ func init() { rootCmd.AddCommand(serveCmd) serveCmd.Flags().BoolP("watch", "w", false, "Watch files for changes") + serveCmd.Flags().String("poll", "", "The poll interval") serveCmd.Flags().StringP("address", "a", "0.0.0.0", "Address to broadcast source list") serveCmd.Flags().StringP("port", "p", "8080", "The port to broadcast the source list on") serveCmd.Flags().StringP("output", "o", "public", "The source list folder") diff --git a/go.mod b/go.mod index b0d3f03..b5dc2be 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf golang.org/x/text v0.3.7 github.com/fsnotify/fsnotify v1.5.4 + github.com/spf13/cast v1.5.0 ) require ( diff --git a/go.sum b/go.sum index b276fa4..0292a10 100644 --- a/go.sum +++ b/go.sum @@ -49,6 +49,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= +github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= +github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v1.4.0 h1:y+wJpx64xcgO1V+RcnwW0LEHxTKRi2ZDPSBjWnrg88Q= github.com/spf13/cobra v1.4.0/go.mod h1:Wo4iy3BUC+X2Fybo0PDqwJIv3dNRiZLHQymsfxlB84g= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= diff --git a/internal/common/common.go b/internal/common/common.go index cfd1085..f157ba7 100644 --- a/internal/common/common.go +++ b/internal/common/common.go @@ -9,9 +9,11 @@ import ( "os" "path/filepath" "strings" + "time" "github.com/bmatcuk/doublestar/v4" "github.com/fatih/color" + "github.com/spf13/cast" ) func CopyFileContents(src, dst string) (err error) { @@ -110,3 +112,14 @@ func ProcessGlobs(globs []string) []string { } return fileList } + +func ToDurationE(v any) (time.Duration, error) { + if n := cast.ToInt(v); n > 0 { + return time.Duration(n) * time.Millisecond, nil + } + d, err := time.ParseDuration(cast.ToString(v)) + if err != nil { + return 0, fmt.Errorf("cannot convert %v to time.Duration", v) + } + return d, nil +} From e9719884993900b3f32347df86f60bc7aa24d1a7 Mon Sep 17 00:00:00 2001 From: beerpiss Date: Thu, 26 May 2022 09:01:30 +0700 Subject: [PATCH 4/5] fix(serve/watch): check for writes --- cmd/aidoku/cmd/serve.go | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/aidoku/cmd/serve.go b/cmd/aidoku/cmd/serve.go index 2b9f6f5..917342d 100644 --- a/cmd/aidoku/cmd/serve.go +++ b/cmd/aidoku/cmd/serve.go @@ -15,6 +15,7 @@ import ( "github.com/Aidoku/aidoku-cli/internal/watcher" "github.com/fatih/color" "github.com/felixge/httpsnoop" + "github.com/fsnotify/fsnotify" "github.com/spf13/cobra" "golang.org/x/exp/maps" ) @@ -98,16 +99,19 @@ var serveCmd = &cobra.Command{ if !ok { return } - changed := make(map[string]string) + changed_map := make(map[string]string) for _, event := range events { - if _, ok := changed[event.Name]; !ok { - changed[event.Name] = "" + if _, ok := changed_map[event.Name]; !ok && event.Op&fsnotify.Write == fsnotify.Write { + changed_map[event.Name] = "" } } - color.HiBlack("File changed, rebuilding source list: %s", strings.Join(maps.Keys(changed), ", ")) - buildLock.Lock() - build.BuildWrapper(files, output) - buildLock.Unlock() + changed := maps.Keys(changed_map) + if len(changed) > 0 { + color.HiBlack("File changed, rebuilding source list: %s", strings.Join(changed, ", ")) + buildLock.Lock() + build.BuildWrapper(files, output) + buildLock.Unlock() + } case err, ok := <-watcher.Errors(): if !ok { return From 6aaaacab7cb7475676906aace5d49e6387318c37 Mon Sep 17 00:00:00 2001 From: beerpiss Date: Thu, 26 May 2022 19:59:20 +0700 Subject: [PATCH 5/5] fixes: mostly small tweaks to clean up --- cmd/aidoku/cmd/serve.go | 12 ++++++------ go.mod | 4 ++-- go.sum | 6 ++++++ internal/watcher/batcher.go | 5 +++-- internal/watcher/filenotify/filenotify.go | 1 + internal/watcher/filenotify/poller.go | 6 ++++++ 6 files changed, 24 insertions(+), 10 deletions(-) diff --git a/cmd/aidoku/cmd/serve.go b/cmd/aidoku/cmd/serve.go index 917342d..97cbea9 100644 --- a/cmd/aidoku/cmd/serve.go +++ b/cmd/aidoku/cmd/serve.go @@ -15,7 +15,6 @@ import ( "github.com/Aidoku/aidoku-cli/internal/watcher" "github.com/fatih/color" "github.com/felixge/httpsnoop" - "github.com/fsnotify/fsnotify" "github.com/spf13/cobra" "golang.org/x/exp/maps" ) @@ -87,11 +86,11 @@ var serveCmd = &cobra.Command{ } } watcher, err := watcher.New(500*time.Millisecond, pollInterval, len(poll) > 0) - var buildLock sync.Mutex if err != nil { color.Red("error: couldn't create file watcher, not watching for changes: %s", err) } else { defer watcher.Close() + var buildLock sync.Mutex go func() { for { select { @@ -101,7 +100,7 @@ var serveCmd = &cobra.Command{ } changed_map := make(map[string]string) for _, event := range events { - if _, ok := changed_map[event.Name]; !ok && event.Op&fsnotify.Write == fsnotify.Write { + if _, ok := changed_map[event.Name]; !ok { changed_map[event.Name] = "" } } @@ -127,7 +126,7 @@ var serveCmd = &cobra.Command{ } } } - fmt.Printf("Watching %d file(s) for changes\n", len(files)) + fmt.Printf("Watching %d file(s) for changes\n", len(watcher.WatchList())) } return http.ListenAndServe(address+":"+port, wrappedHandler) }, @@ -135,11 +134,12 @@ var serveCmd = &cobra.Command{ func init() { rootCmd.AddCommand(serveCmd) - serveCmd.Flags().BoolP("watch", "w", false, "Watch files for changes") - serveCmd.Flags().String("poll", "", "The poll interval") serveCmd.Flags().StringP("address", "a", "0.0.0.0", "Address to broadcast source list") serveCmd.Flags().StringP("port", "p", "8080", "The port to broadcast the source list on") serveCmd.Flags().StringP("output", "o", "public", "The source list folder") + serveCmd.Flags().BoolP("watch", "w", false, "Watch for file changes and rebuild source list as needed") + serveCmd.Flags().String("poll", "", "Watch for file changes with a poll-based approach") + serveCmd.Flags().Lookup("poll").NoOptDefVal = "500ms" serveCmd.MarkZshCompPositionalArgumentFile(1, "*.aix") serveCmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { diff --git a/go.mod b/go.mod index b5dc2be..d2f0551 100644 --- a/go.mod +++ b/go.mod @@ -8,15 +8,15 @@ require ( github.com/bmatcuk/doublestar/v4 v4.0.2 github.com/fatih/color v1.13.0 github.com/felixge/httpsnoop v1.0.3 + github.com/fsnotify/fsnotify v1.5.4 github.com/iancoleman/strcase v0.2.0 github.com/segmentio/fasthash v1.0.3 + github.com/spf13/cast v1.5.0 github.com/spf13/cobra v1.4.0 github.com/valyala/fastjson v1.6.3 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/exp v0.0.0-20220518171630-0b5c67f07fdf golang.org/x/text v0.3.7 - github.com/fsnotify/fsnotify v1.5.4 - github.com/spf13/cast v1.5.0 ) require ( diff --git a/go.sum b/go.sum index 0292a10..25a3c6f 100644 --- a/go.sum +++ b/go.sum @@ -21,8 +21,10 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= +github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= @@ -32,6 +34,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= @@ -46,6 +50,7 @@ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyex github.com/nkovacs/streamquote v1.0.0/go.mod h1:BN+NaZ2CmdKqUuTUXUEm9j95B2TRbpOWpxbJYzzgUsc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/segmentio/fasthash v1.0.3 h1:EI9+KE1EwvMLBWwjpRDc+fEM+prwxDYbslddQGtrmhM= github.com/segmentio/fasthash v1.0.3/go.mod h1:waKX8l2N8yckOgmSsXJi7x1ZfdKZ4x7KRMzBtS3oedY= @@ -89,6 +94,7 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/watcher/batcher.go b/internal/watcher/batcher.go index 9424ccd..d653452 100644 --- a/internal/watcher/batcher.go +++ b/internal/watcher/batcher.go @@ -60,14 +60,15 @@ func New(intervalBatcher, intervalPoll time.Duration, poll bool) (*Batcher, erro } func (b *Batcher) run() { - tick := time.NewTicker(b.interval) + //lint:ignore SA1015 it's a looping function + tick := time.Tick(b.interval) evs := make([]fsnotify.Event, 0) OuterLoop: for { select { case ev := <-b.FileWatcher.Events(): evs = append(evs, ev) - case <-tick.C: + case <-tick: if len(evs) == 0 { continue } diff --git a/internal/watcher/filenotify/filenotify.go b/internal/watcher/filenotify/filenotify.go index b9d0d2e..fde5b29 100644 --- a/internal/watcher/filenotify/filenotify.go +++ b/internal/watcher/filenotify/filenotify.go @@ -19,6 +19,7 @@ type FileWatcher interface { Add(name string) error Remove(name string) error Close() error + WatchList() []string } // New tries to use an fs-event watcher, and falls back to the poller if there is an error diff --git a/internal/watcher/filenotify/poller.go b/internal/watcher/filenotify/poller.go index e39f0ce..f926750 100644 --- a/internal/watcher/filenotify/poller.go +++ b/internal/watcher/filenotify/poller.go @@ -11,6 +11,7 @@ import ( "time" "github.com/fsnotify/fsnotify" + "golang.org/x/exp/maps" ) var ( @@ -40,6 +41,11 @@ type filePoller struct { closed bool } +// WatchList returns the directories and files that are being monitored. +func (w *filePoller) WatchList() []string { + return maps.Keys(w.watches) +} + // Add adds a filename to the list of watches // once added the file is polled for changes in a separate goroutine func (w *filePoller) Add(name string) error {