diff --git a/.travis.yml b/.travis.yml index 4fbc886..c1ceb4e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ services: - docker go: - - "1.13" + - "1.15" - "1.14" - tip @@ -16,6 +16,7 @@ install: - go get -u golang.org/x/tools/cmd/goimports script: + - make install-only - make install - make test - make test-race diff --git a/Dockerfile b/Dockerfile index a24198a..6190ea7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ # # The builder-image go-dev can be found in hack/go-dev # Versions can be found on https://hub.docker.com/r/dyweb/go-dev/tags -FROM dyweb/go-dev:1.13.6 as builder +FROM dyweb/go-dev:1.15.3 as builder LABEL maintainer="contact@dongyue.io" @@ -27,4 +27,4 @@ LABEL github="github.com/dyweb/gommon" WORKDIR /usr/bin COPY --from=builder /go/bin/gommon . ENTRYPOINT ["gommon"] -CMD ["help"] \ No newline at end of file +CMD ["help"] diff --git a/Makefile b/Makefile index 5370b00..034b4eb 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,20 @@ # based on https://gist.github.com/azatoth/1030091 +# TODO(at15): it is also possible to generate it automatically using awk etc. define GOMMON_MAKEFILE_HELP_MSG Make commands for gommon help show help -Dev: +Dev +----------------------------------------- install install binaries under ./cmd to $$GOPATH/bin fmt goimports -test unit test +test run unit test generate generate code using gommon loc lines of code (cloc required, brew install cloc) -Build: +Build +----------------------------------------- install install all binaries under ./cmd to $$GOPATH/bin build compile all binary to ./build for current platform build-linux compile all linux binary to ./build with -linux suffix @@ -19,7 +22,8 @@ build-mac compile all mac binary to ./build with -mac suffix build-win compile all windows binary to ./build with -win suffix build-release compile binary for all platforms and generate tarball to ./build -Docker: +Docker +----------------------------------------- docker-build build runner image w/ all binaries using mulitstage build docker-push push runner image to docker registry @@ -31,25 +35,38 @@ export GOMMON_MAKEFILE_HELP_MSG help: @echo "$$GOMMON_MAKEFILE_HELP_MSG" -GO = GO111MODULE=on go +GO = GO111MODULE=on CGO_ENABLED=0 go # -- build vars --- -PKGS =./errors/... ./generator/... ./httpclient/... ./log/... ./noodle/... ./util/... -PKGST =./cmd ./errors ./generator ./httpclient ./log ./noodle ./util +PKGST =./cmd ./dcli ./errors ./generator ./httpclient ./linter ./log ./noodle ./util ./tconfig +PKGS = $(addsuffix ...,$(PKGST)) VERSION = 0.0.13 BUILD_COMMIT := $(shell git rev-parse HEAD) +BUILD_BRANCH := $(shell git rev-parse --abbrev-ref HEAD) BUILD_TIME := $(shell date +%Y-%m-%dT%H:%M:%S%z) CURRENT_USER = $(USER) FLAGS = -X main.version=$(VERSION) -X main.commit=$(BUILD_COMMIT) -X main.buildTime=$(BUILD_TIME) -X main.buildUser=$(CURRENT_USER) DOCKER_REPO = dyweb/gommon +DCLI_PKG = github.com/dyweb/gommon/dcli. +DCLI_LDFLAGS = -X $(DCLI_PKG)buildVersion=$(VERSION) -X $(DCLI_PKG)buildCommit=$(BUILD_COMMIT) -X $(DCLI_PKG)buildBranch=$(BUILD_BRANCH) -X $(DCLI_PKG)buildTime=$(BUILD_TIME) -X $(DCLI_PKG)buildUser=$(CURRENT_USER) # -- build vars --- .PHONY: install -install: fmt test +install: fmt test install-only + +install-only: cd ./cmd/gommon && $(GO) install -ldflags "$(FLAGS)" . mv $(GOPATH)/bin/gommonbin $(GOPATH)/bin/gommon +.PHONY: install2 +install2: + cd ./cmd/gommon2 && $(GO) install -ldflags "$(DCLI_LDFLAGS)" . + .PHONY: fmt fmt: + gommon format -d -l -w $(PKGST) + +# gommon format is a drop in replacement for goimports +deprecated-fmt: goimports -d -l -w $(PKGST) # --- build --- @@ -136,11 +153,6 @@ docker-build: docker-push: docker push $(DOCKER_REPO):$(VERSION) -docker-test: - docker-compose -f hack/docker-compose.yml run --rm golang1.12 -# TODO: not sure why the latest one is not using ... -# docker-compose -f hack/docker-compose.yml run --rm golanglatest - #.PHONY: docker-remove-all-containers #docker-remove-all-containers: # docker rm $(shell docker ps -a -q) diff --git a/ROADMAP.md b/ROADMAP.md index ea44a2b..e061222 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,9 +1,12 @@ # Roadmap +NOTE: it is being moved to [milestones](doc/milestones) + ## Up coming ### 0.0.14 +- [ ] dcli - [ ] wait package, similar to the polling package in k8s - [ ] have retry as alias and provides backoff - [ ] allow use wait for container diff --git a/cmd/gommon/go.mod b/cmd/gommon/go.mod index 4f538cb..a7753ff 100644 --- a/cmd/gommon/go.mod +++ b/cmd/gommon/go.mod @@ -1,12 +1,14 @@ -go 1.13 +go 1.14 require ( github.com/dyweb/gommon v0.0.13 github.com/spf13/cobra v0.0.6 + golang.org/x/tools v0.0.0-20200401192744-099440627f01 ) -replace github.com/dyweb/gommon v0.0.13 => ../.. +replace github.com/dyweb/gommon => ../.. +// TODO: might name is gom // NOTE: rename it to gommonbin to aviod ambiguous import // can't load package: package github.com/dyweb/gommon/cmd/gommon: ambiguous import: found github.com/dyweb/gommon/cmd/gommon in multiple modules: // github.com/dyweb/gommon/cmd/gommon (/home/at15/w/src/github.com/dyweb/gommon/cmd/gommon) diff --git a/cmd/gommon/go.sum b/cmd/gommon/go.sum index 8554d0b..17381e1 100644 --- a/cmd/gommon/go.sum +++ b/cmd/gommon/go.sum @@ -98,6 +98,7 @@ github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGr github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= @@ -105,28 +106,44 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200401192744-099440627f01 h1:ysQJ/fU6laLOZJseIeOqXl6Mo+lw5z6b7QHnmUKjW+k= +golang.org/x/tools v0.0.0-20200401192744-099440627f01/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= diff --git a/cmd/gommon/main.go b/cmd/gommon/main.go index 0a97732..a4d83ee 100644 --- a/cmd/gommon/main.go +++ b/cmd/gommon/main.go @@ -10,6 +10,7 @@ import ( "runtime" "strings" + "github.com/dyweb/gommon/linter" "github.com/spf13/cobra" "github.com/dyweb/gommon/errors" @@ -33,7 +34,6 @@ var ( ) func main() { - // TODO: most code here are copied from go.ice's cli package, dependency management might break if we import go.ice which also import gommon rootCmd := &cobra.Command{ Use: "gommon", Short: "gommon helpers", @@ -73,6 +73,7 @@ func main() { versionCmd, genCmd(), addBuildIgnoreCmd(), + formatCmd(), ) if err := rootCmd.Execute(); err != nil { fmt.Fprintln(os.Stderr, err) @@ -80,6 +81,7 @@ func main() { } } +// triggers generator for logger, go template and embedding asset func genCmd() *cobra.Command { gen := cobra.Command{ Use: "generate", @@ -141,6 +143,7 @@ func genCmd() *cobra.Command { return &gen } +// add // +build ignore to files before moving them to legacy folder func addBuildIgnoreCmd() *cobra.Command { cmd := cobra.Command{ Use: "add-build-ignore", @@ -194,6 +197,59 @@ func addBuildIgnoreCmd() *cobra.Command { return &cmd } +// gommon format +func formatCmd() *cobra.Command { + var flags linter.GoimportFlags + processFile := func(path string, info os.FileInfo, err error) error { + if err == nil && fsutil.IsGoFile(info) { + return linter.CheckAndFormatImportToStdout(path, flags) + } + // Skip directory and stop on walk error + return err + } + + run := func(paths []string) error { + for _, p := range paths { + switch dir, err := os.Stat(p); { + case err != nil: + return err + case dir.IsDir(): + // TODO: walk w/ ignore like generator + if err := filepath.Walk(p, processFile); err != nil { + return err + } + default: + if err := processFile(p, dir, nil); err != nil { + return err + } + } + } + return nil + } + + cmd := cobra.Command{ + Use: "format", + Short: "Format go code like goimports with custom rules", + Run: func(cmd *cobra.Command, args []string) { + paths := args + if len(paths) == 0 { + log.Fatal("format stdin is not implemented") + return + } + if err := run(paths); err != nil { + log.Fatal(err) + } + }, + } + cmd.Flags().BoolVarP(&flags.List, "list", "l", false, "list files whose formatting differs from goimports") + cmd.Flags().BoolVarP(&flags.Write, "write", "w", false, "write result to (source) file instead of stdout, i.e. in place update") + cmd.Flags().BoolVarP(&flags.Diff, "diff", "d", false, "display diffs instead of rewriting files") + cmd.Flags().BoolVarP(&flags.AllErrors, "errors", "e", false, "report all errors (not just the first 10 on different lines)") + cmd.Flags().StringVar(&flags.LocalPrefix, "local", "", "put imports beginning with this string after 3rd-party packages; comma-separated list") + cmd.Flags().BoolVar(&flags.FormatOnly, "format-only", false, "if true, don't fix imports and only format. In this mode, goimports is effectively gofmt, with the addition that imports are grouped into sections.") + return &cmd +} + func init() { dlog.SetHandler(cli.New(os.Stderr, true)) } diff --git a/cmd/gommon2/main.go b/cmd/gommon2/main.go new file mode 100644 index 0000000..248d28f --- /dev/null +++ b/cmd/gommon2/main.go @@ -0,0 +1,23 @@ +// gommon2 is a test binary for using dcli, it will be renamed to gommon once we have most functionality in spf13/cobra +package main + +import ( + "context" + + "github.com/dyweb/gommon/dcli" + dlog "github.com/dyweb/gommon/log" +) + +var logReg = dlog.NewRegistry() +var log = logReg.NewLogger() + +func main() { + root := &dcli.Cmd{ + Name: "gommon2", + Run: func(ctx context.Context) error { + log.Info("gommon2 does nothing") + return nil + }, + } + dcli.RunApplication(root) +} diff --git a/dcli/README.md b/dcli/README.md index f16956b..afb5a85 100644 --- a/dcli/README.md +++ b/dcli/README.md @@ -1,6 +1,12 @@ # dcli -dcli is a light weight cli builder. +dcli is a lightweight cli builder. + +## Usage + +```go + +``` ## Issues @@ -8,7 +14,8 @@ dcli is a light weight cli builder. ## Reference and Alternatives -- [spf13/cobra](https://github.com/spf13/cobra) +- [spf13/cobra](https://github.com/spf13/cobra) Imports etcd and consul ... - [peterbourgon/ff](https://github.com/peterbourgon/ff) Flag first package for configuration - [urfave/cli](https://github.com/urfave/cli) -- [mitchellh/cli](https://github.com/mitchellh/cli) Used by consul, terraform etc. \ No newline at end of file +- [mitchellh/cli](https://github.com/mitchellh/cli) Used by consul, terraform etc. +- [alecthomas/kong](https://github.com/alecthomas/kong) Define command using struct and struct tag diff --git a/dcli/app.go b/dcli/app.go new file mode 100644 index 0000000..2994d7f --- /dev/null +++ b/dcli/app.go @@ -0,0 +1,77 @@ +package dcli + +import ( + "context" + "os" + + "github.com/dyweb/gommon/errors" +) + +// app.go defines application struct, a wrapper for top level command. + +type Application struct { + Build BuildInfo + Root Command // entry command, its Name should be same as Application.Name but it is ignored when execute. +} + +// RunApplication creates a new application and run it directly. +// It logs and exit with 1 if application creation or execution failed. +func RunApplication(cmd Command) { + app, err := NewApplication(cmd) + if err != nil { + log.Fatal(err) + } + app.Run() +} + +const versionCmd = "version" + +// NewApplication validate root command and injects version command if not exists. +func NewApplication(cmd Command) (*Application, error) { + if err := ValidateCommand(cmd); err != nil { + return nil, errors.Wrap(err, "command validation failed") + } + info := DefaultBuildInfo() + // Inject version command if it does not exist + if !hasChildCommand(cmd, versionCmd) { + c, ok := cmd.(MutableCommand) + if ok { + verCmd := &Cmd{ + Name: versionCmd, + Run: func(_ context.Context) error { + PrintBuildInfo(os.Stdout, info) + return nil + }, + } + if err := c.AddChildren(verCmd); err != nil { + return nil, errors.Wrap(err, "error adding version command") + } + } + } + return &Application{ + Build: info, + Root: cmd, + }, nil +} + +// Run calls RunArgs with command line arguments (os.Args[1:]) and exit 1 when there is error. +func (a *Application) Run() { + if err := a.RunArgs(context.Background(), os.Args[1:]); err != nil { + log.Fatal(err) + os.Exit(1) + } +} + +func (a *Application) RunArgs(ctx context.Context, args []string) error { + // TODO: extract both arg and flags + //log.Infof("args %v", args) + // TODO: use special handler for command not found + c, err := FindCommand(a.Root, args) + if err != nil { + return err + } + if err := c.GetRunnable()(ctx); err != nil { + handleCommandError(c, err) + } + return nil +} diff --git a/dcli/build.go b/dcli/build.go new file mode 100644 index 0000000..a7452e7 --- /dev/null +++ b/dcli/build.go @@ -0,0 +1,58 @@ +package dcli + +import ( + "fmt" + "io" + "runtime" +) + +// build.go defines build info that can be set using -ldflags when compiling the binary + +var ( + // set using -ldflags "-X github.com/dyweb/gommon/dcli.buildVersion=0.0.1" + buildVersion string + buildCommit string + buildBranch string + buildTime string + buildUser string +) + +// BuildInfo contains information that should be set at build time. +// e.g. go install ./cmd/myapp -ldflags "-X github.com/dyweb/gommon/dcli.buildVersion=0.0.1" +// You can use DefaultBuildInfo and copy paste its Makefile rules. +type BuildInfo struct { + Version string + Commit string + Branch string + Time string + User string + GoVersion string +} + +// DefaultBuildInfo returns a info based on ld flags sets to github.com/dyweb/gommon/dcli.* +// You can copy the following rules in your Makefile +// +// DCLI_PKG = github.com/dyweb/gommon/dcli. +// DCLI_LDFLAGS = -X $(DCLI_PKG)buildVersion=$(VERSION) -X $(DCLI_PKG)buildCommit=$(BUILD_COMMIT) -X $(DCLI_PKG)buildBranch=$(BUILD_BRANCH) -X $(DCLI_PKG)buildTime=$(BUILD_TIME) -X $(DCLI_PKG)buildUser=$(CURRENT_USER) +// +// install: +// go install -ldflags $(DCLI_LDFLAGS) ./cmd/myapp +func DefaultBuildInfo() BuildInfo { + return BuildInfo{ + Version: buildVersion, + Commit: buildCommit, + Branch: buildBranch, + Time: buildTime, + User: buildUser, + GoVersion: runtime.Version(), + } +} + +func PrintBuildInfo(w io.Writer, i BuildInfo) { + fmt.Fprintf(w, "Version: %s\n", i.Version) + fmt.Fprintf(w, "GitCommit: %s\n", i.Commit) + fmt.Fprintf(w, "GitBranch: %s\n", i.Branch) + fmt.Fprintf(w, "BuildTime: %s\n", i.Time) + fmt.Fprintf(w, "BuildUser: %s\n", i.User) + fmt.Fprintf(w, "GoVersion: %s\n", i.GoVersion) +} diff --git a/dcli/command.go b/dcli/command.go new file mode 100644 index 0000000..79ac7e3 --- /dev/null +++ b/dcli/command.go @@ -0,0 +1,126 @@ +package dcli + +import ( + "context" + + "github.com/dyweb/gommon/errors" +) + +// command.go defines interface and the default implementation Cmd + +type Runnable func(ctx context.Context) error + +type Command interface { + GetName() string + GetRunnable() Runnable + GetChildren() []Command +} + +type MutableCommand interface { + AddChildren(children ...Command) error +} + +var _ Command = (*Cmd)(nil) + +// Cmd is the default implementation of Command interface +type Cmd struct { + Name string + Run Runnable + Children []Command +} + +func (c *Cmd) GetName() string { + return c.Name +} + +func (c *Cmd) GetRunnable() Runnable { + return c.Run +} + +func (c *Cmd) GetChildren() []Command { + return c.Children +} + +func (c *Cmd) AddChildren(children ...Command) error { + // TODO: check name conflict and import cycle + c.Children = append(c.Children, children...) + return nil +} + +// Validate Start +const commandPrefixSep = ">" + +// ValidateCommand checks if a command and its children have set name and runnable properly. +// It also checks if there is cycle TODO: the check is too strict .... +func ValidateCommand(c Command) error { + m := make(map[Command]string) + return validate(c, "", m) +} + +func validate(c Command, prefix string, visited map[Command]string) error { + merr := errors.NewMulti() + name := "unknown" + if c.GetName() == "" { + merr.Append(errors.Errorf("command has no name, prefix: %s", prefix)) + } else { + name = c.GetName() + } + if c.GetRunnable() == nil { + merr.Append(errors.Errorf("command has no runnable, name: %s, prefix: %s", name, prefix)) + } + prefix = prefix + commandPrefixSep + name + // FIXME: this check is too strict, we only want cycle detection ... I think we allow DAG ... + if p, ok := visited[c]; ok { + merr.Append(errors.Errorf("duplicated command, previously used at %s used again at %s", p, prefix)) + return merr.ErrorOrNil() + } + visited[c] = prefix + childNames := make(map[string]bool, len(c.GetChildren())) + for _, child := range c.GetChildren() { + if childNames[child.GetName()] { + merr.Append(errors.Errorf("child defined twice, name: %s, parent: %s", child.GetName(), prefix)) + } + merr.Append(validate(child, prefix, visited)) + } + return merr.ErrorOrNil() +} + +// Validate End + +// Dispatch Start + +func FindCommand(root Command, args []string) (Command, error) { + if len(args) == 0 { + return root, nil + } + // TODO: strip flag + sub := args[0] + for _, cur := range root.GetChildren() { + if cur.GetName() != sub { + continue + } + // Check if there can be more matches + if len(cur.GetChildren()) == 0 || len(args) == 1 { + return cur, nil + } + return FindCommand(cur, args[1:]) + } + // TODO: typed error and suggestion using edit distance + return nil, errors.New("command not found") +} + +// Dispatch End + +// Util Start + +// hasChildCommand checks if command has child with given name. It does NOT check children recursively. +func hasChildCommand(c Command, name string) bool { + for _, child := range c.GetChildren() { + if child.GetName() == name { + return true + } + } + return false +} + +// Util End diff --git a/dcli/command_test.go b/dcli/command_test.go new file mode 100644 index 0000000..772a7a7 --- /dev/null +++ b/dcli/command_test.go @@ -0,0 +1,29 @@ +package dcli_test + +import ( + "testing" + + "github.com/dyweb/gommon/dcli" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFindCommand(t *testing.T) { + // TODO: flag, sub command, auto complete suggestion + r := &dcli.Cmd{ + Name: "bh", + Children: []dcli.Command{ + &dcli.Cmd{ + Name: "user", + Children: []dcli.Command{ + &dcli.Cmd{ + Name: "register", + }, + }, + }, + }, + } + c, err := dcli.FindCommand(r, []string{"user", "register"}) + require.Nil(t, err) + assert.Equal(t, "register", c.GetName()) +} diff --git a/dcli/context.go b/dcli/context.go new file mode 100644 index 0000000..d4d2d32 --- /dev/null +++ b/dcli/context.go @@ -0,0 +1,56 @@ +package dcli + +import ( + "context" + "time" +) + +// Context implements context.Context and provides cli specific helper func. +// A default implementation DefaultContext is provided. +type Context interface { + context.Context +} + +var _ Context = (*DefaultContext)(nil) + +type DefaultContext struct { + stdCtx context.Context +} + +// TODO(generator): those default context wrapper should be generated. It is also used in httpclient package +// Deadline returns Deadline() from underlying context.Context if set +func (c *DefaultContext) Deadline() (deadline time.Time, ok bool) { + if c != nil && c.stdCtx != nil { + return c.stdCtx.Deadline() + } + // NOTE: we are using named return, so empty value will be returned + // learned this from context.Context's emptyCtx implementation + return +} + +// Done returns Done() from underlying context.Context if set +func (c *DefaultContext) Done() <-chan struct{} { + if c != nil && c.stdCtx != nil { + return c.stdCtx.Done() + } + // Done may return nil if this context can never be canceled + return nil +} + +// Err returns Err() from underlying context.Context if set +func (c *DefaultContext) Err() error { + if c != nil && c.stdCtx != nil { + return c.stdCtx.Err() + } + return nil +} + +// Value first checks the map[string]interface{}, +// if not found, it use the underlying context.Context if is set +// if not set, it returns nil +func (c *DefaultContext) Value(key interface{}) interface{} { + if c != nil && c.stdCtx != nil { + return c.stdCtx.Value(key) + } + return nil +} diff --git a/dcli/doc/command.md b/dcli/doc/command.md new file mode 100644 index 0000000..43e7e22 --- /dev/null +++ b/dcli/doc/command.md @@ -0,0 +1,6 @@ +# Gommon dcli Command + +## Overview + +A command runs a function (e.g. `foo run`) or acts as a container for its subcommands (e.g. `server` in `foo server start`). +In rare cases it can be both i.e. there is a default sub command. diff --git a/dcli/doc/design/2020-01-18-init.md b/dcli/doc/log/2020-01-18-init.md similarity index 100% rename from dcli/doc/design/2020-01-18-init.md rename to dcli/doc/log/2020-01-18-init.md diff --git a/dcli/doc/log/2020-03-28-init.md b/dcli/doc/log/2020-03-28-init.md new file mode 100644 index 0000000..4814306 --- /dev/null +++ b/dcli/doc/log/2020-03-28-init.md @@ -0,0 +1,82 @@ +# 2020-03-28 Init + +Init again after [two and half months](2020-01-18-init.md) + +## Goals + +Current + +- support git style sub command +- use interface instead of command, provide a default struct (like spf13/cobra) for simply implementation +- global flags (like spf13/cobra), allow inheriting flags from parent command + +Long term + +- interactive +- completion (including interactive mode) + +## Design + +Examples + +```text +gommon -h +gommon generate -v --ignore=*.proto +gommon generate noodle -v --ignore=node_modules +``` + +When defining command line application and flags, use struct instead of adhoc `flags.String, flags.StringP`. +Rust's [structopt](https://github.com/TeXitoi/structopt) can be an example. +Though go does not have macro, so we may need to use comment and code generator. + +The cli interface definition should be more declarative. + +```text +// in spf13/cobra +sub1 := Cmd{xxx} +subsub1 := Cmd{xxx} +sub1.AddCommand(subsub1) // cmd.commands is not exported, and AddCommand does some extra calculation +sub1.Flags().BoolVarP(&verbose, "", ) + +// a more straightforward approach is +// cmd is the default command implementation +sub := Cmd{ + Commands: []{ + Cmd{ + Name: + Flags: A flag definition struct. // TODO: how to handle persistent flags + Run: + } + } +} +sub.Validate() // check if spec is correct and init some internal states +``` + +Components + +- `Application` entry point, a thin wrapper for the top level command +- `Command` an interface that has name, runnable and children + - a default implementation +- `Argument` position argument +- `Flag` flag + - [ ] should we distinguish flag and position argument? +- `Help` + - help only command e.g. `git` itself only prints help and exit with `1` + - auto generated help message + - custom help message +- `Suggestion` + - command not found for typo + - common flag values e.g. `waitfor --protocol tcp|http` +- `Shell` + - bash auto completion without invoking the cli, i.e. only based on argument and flags + - completion by running the cli + +## Implementation + +TODO: break up by components + +- build the command tree using `Command` interface and the default `Cmd` struct +- validate the command tree + - [ ] we should allow DAG, dfs + back edge? +- find the sub command sequence from args + - there can be ambiguity, e.g. `gommon generate noodle`, if there is no `noodle` sub command, then `noodle` is position argument. diff --git a/dcli/doc/survey/clap.md b/dcli/doc/survey/clap.md index 149f6d5..b40d49c 100644 --- a/dcli/doc/survey/clap.md +++ b/dcli/doc/survey/clap.md @@ -1,5 +1,25 @@ # Clap +- [Command line apps in Rust](https://rust-cli.github.io/book/index.html) - TBH, compared with cobra it's hard to use ... requires to run the dispatch logic by doing pattern matching by yourself - maybe I not using it in the right way +## StructOpt + +- https://github.com/TeXitoi/structopt +- https://clap.rs/2019/03/08/clap-v3-update-structopt/ + +```rust +use structopt::StructOpt; + +/// Search for a pattern in a file and display the lines that contain it. +#[derive(StructOpt)] +struct Cli { + /// The pattern to look for + pattern: String, + /// The path to the file to read + #[structopt(parse(from_os_str))] + path: std::path::PathBuf, +} + +``` \ No newline at end of file diff --git a/dcli/doc/survey/cobra.md b/dcli/doc/survey/cobra.md index 9b1cf78..b99ff20 100644 --- a/dcli/doc/survey/cobra.md +++ b/dcli/doc/survey/cobra.md @@ -3,4 +3,95 @@ https://github.com/spf13/cobra - support persistent flag, i.e. flag that can be applied to sub commands `gommon gen --foo bar --verbose` -- https://github.com/at15/code-i-read/tree/master/go/spf13/cobra has much more detail \ No newline at end of file +- https://github.com/at15/code-i-read/tree/master/go/spf13/cobra has much more detail + +## Subcommand + +If you defined a subcommand like `git clone` and you do `git glone`, +it will error out instead of passing `clone` as argument to the `git` command. +However if you just use `git`, it will call the `git` command without argument. + +```text +// pseudo code for execute logic in cobra +func ExecuteC() { + commands = stripFlags(args) + nextCommand = commands[0] + var cmd + for _, c := range c.commands { + if c.Name() == nextCommand { + cmd = c + break + } + } + cmd.execute() +} +``` + +```go +// https://github.com/spf13/cobra/blob/master/command.go#L605 + +// Find the target command given the args and command tree +// Meant to be run on the highest node. Only searches down. +func (c *Command) Find(args []string) (*Command, []string, error) { + var innerfind func(*Command, []string) (*Command, []string) + + innerfind = func(c *Command, innerArgs []string) (*Command, []string) { + argsWOflags := stripFlags(innerArgs, c) + if len(argsWOflags) == 0 { + return c, innerArgs + } + nextSubCmd := argsWOflags[0] + + cmd := c.findNext(nextSubCmd) + if cmd != nil { + return innerfind(cmd, argsMinusFirstX(innerArgs, nextSubCmd)) + } + return c, innerArgs + } + + commandFound, a := innerfind(c, args) + if commandFound.Args == nil { + return commandFound, a, legacyArgs(commandFound, stripFlags(a, commandFound)) + } + return commandFound, a, nil +} + +// https://github.com/spf13/cobra/blob/6607e6b8603f56adb027298ee6695e06ffb3a819/command.go#L546 +func stripFlags(args []string, c *Command) []string { + if len(args) == 0 { + return args + } + c.mergePersistentFlags() + + commands := []string{} + flags := c.Flags() + +Loop: + for len(args) > 0 { + s := args[0] + args = args[1:] + switch { + case s == "--": + // "--" terminates the flags + break Loop + case strings.HasPrefix(s, "--") && !strings.Contains(s, "=") && !hasNoOptDefVal(s[2:], flags): + // If '--flag arg' then + // delete arg from args. + fallthrough // (do the same as below) + case strings.HasPrefix(s, "-") && !strings.Contains(s, "=") && len(s) == 2 && !shortHasNoOptDefVal(s[1:], flags): + // If '-f arg' then + // delete 'arg' from args or break the loop if len(args) <= 1. + if len(args) <= 1 { + break Loop + } else { + args = args[1:] + continue + } + case s != "" && !strings.HasPrefix(s, "-"): + commands = append(commands, s) + } + } + + return commands +} +``` \ No newline at end of file diff --git a/dcli/doc/survey/mitchellh.md b/dcli/doc/survey/mitchellh.md index 5b774b0..faae054 100644 --- a/dcli/doc/survey/mitchellh.md +++ b/dcli/doc/survey/mitchellh.md @@ -16,4 +16,23 @@ If you use a CLI with nested subcommands, some semantics change due to ambiguiti > Any parent commands that don't exist are automatically created as no-op commands that just show help for other subcommands. For example, - if you only register "foo bar", then "foo" is automatically created. \ No newline at end of file + if you only register "foo bar", then "foo" is automatically created. + + +```go +package main + +func main() { + c := cli.NewCLI("app", "1.0.0") + c.Args = os.Args[1:] + c.Commands = map[string]cli.CommandFactory{ + "foo": fooCommandFactory, + "bar": barCommandFactory, + } + + exitStatus, err := c.Run() + if err != nil { + log.Println(err) + } +} +``` \ No newline at end of file diff --git a/dcli/error.go b/dcli/error.go new file mode 100644 index 0000000..1926e85 --- /dev/null +++ b/dcli/error.go @@ -0,0 +1,37 @@ +package dcli + +import ( + "fmt" + "reflect" +) + +// error.go Defines special errors and handlers. + +// ErrHelpOnlyCommand means the command does not contain execution logic and simply print usage info for sub commands. +type ErrHelpOnlyCommand struct { + Command string +} + +func NewErrHelpOnly(cmd string) *ErrHelpOnlyCommand { + return &ErrHelpOnlyCommand{Command: cmd} +} + +func (e *ErrHelpOnlyCommand) Error() string { + return "command only prints help and has no execution logic: " + e.Command +} + +func commandNotFound(root Command, args []string) { + +} + +// handles error from a specific command +// TODO: specify print output location for testing +func handleCommandError(cmd Command, err error) { + switch x := err.(type) { + case *ErrHelpOnlyCommand: + fmt.Println("TODO: should print help for command " + cmd.GetName() + x.Command) + // TODO: unwrap error? or by default simply print it ... + default: + fmt.Printf("TODO: unhandled error of type %s %s\n", reflect.TypeOf(err).String(), err) + } +} diff --git a/dcli/pkg.go b/dcli/pkg.go new file mode 100644 index 0000000..bc9937e --- /dev/null +++ b/dcli/pkg.go @@ -0,0 +1,10 @@ +// Package dcli is a commandline application builder. +// It supports git style sub command and is modeled after spf13/cobra. +package dcli + +import ( + dlog "github.com/dyweb/gommon/log" +) + +var logReg = dlog.NewRegistry() +var log = logReg.NewLogger() diff --git a/doc/README.md b/doc/README.md index a11d6bd..0858255 100644 --- a/doc/README.md +++ b/doc/README.md @@ -1,7 +1,16 @@ # Gommon Documentation -- Style - - [General](style.md) - - [Application using Gommon](style-application.md) - - [Library using Gommon](style-library.md) - - [Writing Gommon](style-gommon.md) +Style + +- [General](style.md) +- [Application using Gommon](style-application.md) +- [Library using Gommon](style-library.md) +- [Writing Gommon](style-gommon.md) + +## Milestones + +See [milestones](milestones) for detail milestones. + +## Components + +TODO: link to different packages. \ No newline at end of file diff --git a/doc/attribution.md b/doc/attribution.md index e3230ca..0294824 100644 --- a/doc/attribution.md +++ b/doc/attribution.md @@ -1,14 +1,13 @@ # Attribution & Comparison -Gommon is inspired by many awesome libraries. -However, we chose to reinvent the wheel for most of them. -Doing so allow us to shrink codebase, introduce break changes frequently, unify error handling and logging. +Gommon is inspired by many awesome libraries. However, we chose to reinvent the wheel for most functionalities. +Doing so allow us to introduce break changes frequently ... ## errors -- [pkg/errors](https://github.com/pkg/errors) it can not introduce breaking change, but `WithMessage` and `WithStack` is annoying - - see [#54](https://github.com/dyweb/gommon/issues/54) and [errors/doc](errors/doc) about other error packages - - https://github.com/pkg/errors/pull/122 for check existing stack before attach new one +- [pkg/errors](https://github.com/pkg/errors) it cannot introduce breaking change, but `WithMessage` and `WithStack` is annoying + - see [#54](https://github.com/dyweb/gommon/issues/54) and [errors/doc](../errors/doc) about other error packages + - https://github.com/pkg/errors/pull/122 implemented checking existing stack before attach new one - [uber-go/multierr#21]( https://github.com/uber-go/multierr/issues/21) for return bool after append - [hashicorp/go-multierror](https://github.com/hashicorp/go-multierror) for `ErrorOrNil` @@ -46,5 +45,5 @@ Doing so allow us to shrink codebase, introduce break changes frequently, unify - [benbjohnson/tmpl](https://github.com/benbjohnson/tmpl) for go template based generator - first saw it in [influxdata/influxdb](https://github.com/influxdata/influxdb/blob/master/tsdb/engine/tsm1/encoding.gen.go.tmpl) - we put template data in `gommon.yml`, so we don't need to pass data as json via cli. - Using YAML instead of cli is inspired by [docker-compose](https://github.com/docker/compose) + - Using YAML instead of flags based on [docker-compose](https://github.com/docker/compose) diff --git a/doc/components.md b/doc/components.md new file mode 100644 index 0000000..7c15610 --- /dev/null +++ b/doc/components.md @@ -0,0 +1,19 @@ +# Gommon Components + +NOTE: Unlike other doc, most gommon components has their own dock folder. This file only serves as index and TODO list. + +- [dcli](../dcli) Command line builder +- [dlog](../log) Logging library +- [errors](../errors) Error wrapping and multi error +- [ ] test(x?) + - benchmark? + - color output + - coverage merge + - condition + - golden + - visualization +- [generator](../generator) +- [linter](../linter) + - format + - check import +- [httpclient](../httpclient) diff --git a/doc/log/README.md b/doc/log/README.md new file mode 100644 index 0000000..4703f14 --- /dev/null +++ b/doc/log/README.md @@ -0,0 +1,3 @@ +# Gommon Developer Log + +- [@at15](at15) \ No newline at end of file diff --git a/doc/log/at15/2020-08-06-pending-issues.md b/doc/log/at15/2020-08-06-pending-issues.md new file mode 100644 index 0000000..94bf6e8 --- /dev/null +++ b/doc/log/at15/2020-08-06-pending-issues.md @@ -0,0 +1,85 @@ +# 2020-08-06 Pending Issues + +## Background + +I decided to follow the project management in [BenchHub](https://github.com/benchhub/benchhub). +i.e. moving project planning from github issue to markdown file in milestones and components. +There are many pending issues across a long time span and I need to summarize them before drafting new plan. + +The [issues](https://github.com/dyweb/gommon/issues?page=1&q=is%3Aissue+is%3Aopen) can be divided into the following categories: + +- new package +- new features on existing packages +- bug + +Some issues are so old the original packages are removed (config, requests etc.). + +## Issues + +### New package + +Large + +- [dcli](https://github.com/dyweb/gommon/issues/117) a cli builder to replace cobra +- [tail](https://github.com/dyweb/gommon/issues/95) like `tail -f` and might even parse log and send metrics to tsdb like [mtail](https://github.com/google/mtail) +- [testx](https://github.com/dyweb/gommon/issues/101) test and benchmark result dashboard + +Medium + +- [goimport that checks and replaces specific import](https://github.com/dyweb/gommon/issues/118) + +Small + +- [human](https://github.com/dyweb/gommon/issues/10) +- [retry](https://github.com/dyweb/gommon/issues/126) +- [mathutil](https://github.com/dyweb/gommon/issues/123) `mathuitl.MaxInt(64)` +- [netutil](https://github.com/dyweb/gommon/issues/122) port wait for it + +### New feature + +- error + - [a more complex error interface](https://github.com/dyweb/gommon/issues/76) + - [human readable suggestion for possible solutions](https://github.com/dyweb/gommon/issues/73) + - [fmt.Formatter](https://github.com/dyweb/gommon/issues/62) I think the new go error package has abandoned this +- log + - [rename log to dlog](https://github.com/dyweb/gommon/issues/120) + - [parse generated log](https://github.com/dyweb/gommon/issues/89) + - [generate file and line number to avoid calling runtime](https://github.com/dyweb/gommon/issues/43) + - [http API to control log level at runtime](https://github.com/dyweb/gommon/issues/23) + - [support grep log and web UI](https://github.com/dyweb/gommon/issues/9) +- noodle + - [set modification time for generated file](https://github.com/dyweb/gommon/issues/128) + - [interface around http.FileSystem](https://github.com/dyweb/gommon/issues/84) +- generator + - [deepcopy](https://github.com/dyweb/gommon/issues/102) +- testutil + - [Only run test in IDE](https://github.com/dyweb/gommon/issues/91) +- requests (the package is in legacy already and replaced by httpclient) + - [oauth2 client with access token](https://github.com/dyweb/gommon/issues/70) +- config + - [validate config struct using tag](https://github.com/dyweb/gommon/issues/19) + +### Bug + +- [go vet error in example](https://github.com/dyweb/gommon/issues/107) + +## Priority + +It's impossible to fix all the gommon issues at once, and there are several active projects using gommon (benchhub, gce4-go, pm). +Some features are nice to have e.g. adjust log level using http API while some features are essential e.g. dcli. + +- dcli + - used by all projects that requires a cli, benchhub, gce4-go, pm, ayi +- log + - used by all projects, the API is hard to use, and it's hard to read the log +- error + - used by all projects, but most time I am just wrapping w/o analysing i.e. no unwrap +- generator + - used by projects that uses protobuf, benchhub +- test + - helps develop all the go packages + - used by benchhub for gobench and gotest framework +- util + - mathutil, stringutil, maputil etc. +- noodle + - I rarely use it because I read from local fs, it will be useful for projects that distribute binary w/ UI \ No newline at end of file diff --git a/doc/log/at15/2020-08-11-format-import.md b/doc/log/at15/2020-08-11-format-import.md new file mode 100644 index 0000000..4054f15 --- /dev/null +++ b/doc/log/at15/2020-08-11-format-import.md @@ -0,0 +1,25 @@ +# 2020-08-11 Format Import + +## TODO + +- [ ] format w/o considering comment +- [ ] format w/ comment + +## Background + +For [linter/import](../../../linter/doc/import.md). Most time is spent on reading goimport source. +The hard part is how to rearrange the AST and still generates a correct output w/ comment. +Go generates modified file by printing AST, and it also requires the modified `FileSet`. +It seems not all positions are adjusted properly, and they does not match the actual output in the end. +Removing blank line is easy (and used frequently). `fset.File(s.Pos()).MergeLine(fset.Position(p).Line)` +Adding blank line is harder, it is added after the code is already formatted using `addImportSpaces` + +## Implementation + +### Group + +The grouping logic is same as `importGroup` + +- define a set of rules, each rules gives a group number +- sort spec by group number like `byImportSpec` +- fix the comments and position like `sortSpecs` \ No newline at end of file diff --git a/doc/log/at15/2020-08-27-tconfig.md b/doc/log/at15/2020-08-27-tconfig.md new file mode 100644 index 0000000..547c97a --- /dev/null +++ b/doc/log/at15/2020-08-27-tconfig.md @@ -0,0 +1,65 @@ +# 2020-08-27 tconfig + +## TODO + +- [ ] design interface for `Value` and `Mutator` +- [ ] allow extending data type, we only support `bool`, `int`, `string` out of box +- [ ] there are five sources of config, default, env, config file, flag, user specified + +## Background + +There was a config package `gommon/config` is the very beginning, it is modelled after `spf13/cobra`. +Which essentially load everything as a `map[string]interface{}` and access using `a.b.c.d`. +It supports multiple config source, e.g. merge configs from different config file `~/.foo/default.yml`, `$PWD/foo.yml`. +It's very flexible and dynamic, however the convenience comes with cost. It's hard to modify the config. +If you add/remove config value, the code can break randomly, especially when people build key path on the fly +e.g. `"foo." + var1 + ".bar"`. instead of `foo.svc1.bar`. There is no compiler error telling you something is wrong. + +So I changed to another approach, write config struct and decode from structured format like `json`, `yaml`. +This approach works fine when the config file is the only source and all the values are required in the config file. +However, once we started to consider config value override (e.g. default, multiple config sources) things become complex. +We lose the track of origin of the config value, it can be the default, a command line flag, a field in JSON, +an override in code or even a user provided input that happens to be the default. + +There is a concept called [data lineage](https://en.wikipedia.org/wiki/Data_lineage) which I first learned from @palvaro's +LDFI paper. Just like you can trace data change, micro services etc. You can also track your config change (even within a process). +Like all the tracing methods, you need to use wrapped functions (that calls trace logic underneath) +or inject trace logic in original functions. + +btw: I think it's also a bit similar to Redux and even Raft. Honestly, I feel all the applying log to get state approaches +looks similar in some extent. + +The name `tconfig` got the `t` from tracing/traceable. + +## Design + +There are two main use cases for config value, a single value or a set of value in a struct. +We ignore use cases like `[]Value` and `map[string]Value` because using a struct is a more type safe alternative. + +A value contains a current state and a list of mutations, if the list is append only, the state is equal to +application of all the mutations in their insertion order. However, the list of mutations might get truncated +for performance reason, though in that case, chances are the package is being misused as a database. + +An alternative approach is using a linked list, where each mutation points to its ancestor. +However, I think keep a slice makes traverse and print etc. faster and easier w/ native `range` support in go. + +``` +type Value interface { + Eval() interface{} +} + +type IntValue interface { + EvalInt() int +} + +type Mutator interface { + MutateBool() + MutateInt() + MutateString() +} + +type MutableValue interace { + Mutate(m Mutator) + Mutations() []Mutation +} +``` diff --git a/doc/log/at15/README.md b/doc/log/at15/README.md new file mode 100644 index 0000000..d7b66d5 --- /dev/null +++ b/doc/log/at15/README.md @@ -0,0 +1 @@ +# Gommon at15 Developer Log diff --git a/doc/milestones/README.md b/doc/milestones/README.md new file mode 100644 index 0000000..d2d38ec --- /dev/null +++ b/doc/milestones/README.md @@ -0,0 +1,6 @@ +# gommon Milestones + +- [v0.1.0 MVP](v0.1.0-mvp) + - [v0.0.14 dcli](v0.0.14-dcli) + - [v0.0.15 dlog](v0.0.15-dlog) + - [v0.0.16 static analysis](v0.0.16-linter) \ No newline at end of file diff --git a/doc/milestones/v0.0.14-dcli/README.md b/doc/milestones/v0.0.14-dcli/README.md new file mode 100644 index 0000000..ae95cbd --- /dev/null +++ b/doc/milestones/v0.0.14-dcli/README.md @@ -0,0 +1,70 @@ +# Gommon v0.0.14 dcli + +## TODO + +- [ ] merge w/ existing design doc in [dcli/doc/design](../../../dcli/doc/design) +- [ ] split up features +- [ ] list implementation order + +## Overview + +A commandline application builder `dcli` that replaces [spf13/cobra](https://github.com/spf13/cobra). +Minor fix to update small util packages e.g. `mathutil`, `stringutil`, `envutil`. + +## Motivation + +`dcli` + +- less dependencies +- more customization +- type safe +- learn from existing command line builders in different languages, e.g. clap (w/ structopt) + +## Implementation + +- [x] define `Command` interface +- [x] define a common `Command` implementation so user don't need to create new struct most of the time + - [x] `Cmd` is the struct +- [x] look up sub command `binary foo bar boar` + - assume there is no flag, e.g. no `binary foo --bla bar` + - assume there is no position argument, e.g. no `binary foo arg1 arg2` +- [ ] show help message + - [x] define an error for command whose sole purpose is showing help, e.g. `git` + - [ ] a global error handler (or per command) + - [ ] show formatted error message + - [ ] dump error message as HTML (there is no flag support, how to do that now, env?) +- [ ] rename `gommon` binary package to `gom` or whatever way to make it's easier to install the binary w/ `go get` +- [ ] remove `cobra` from `gommon` binary dependency + +## Specs + +- support git style flag and subcommand +- use `dcli` for `gommon` command + +## Features + +### Sub command + +Description + +Support subcommand like `git clone`, and show help message for available sub command. +For simplicity in this feature we don't consider flag and position argument. + +Components + +- `help` + - list sub command in help messages, and provide short description + - print a html page if there are too many things for terminal and user prefer clicking around + - print a text file so user can open it in vim and search it + +### Flag + +Description + +Flag can show up in multiple places + +- after binary name `foo --verbose=true` +- after sub command `foo bar --target==127.0.0.1:3530` + +We should follow spf13/cobra where you can define a flag in parent command and shared by all child commands. +i.e. `PersistentFlag` \ No newline at end of file diff --git a/doc/milestones/v0.0.15-dlog/README.md b/doc/milestones/v0.0.15-dlog/README.md new file mode 100644 index 0000000..3adf2bc --- /dev/null +++ b/doc/milestones/v0.0.15-dlog/README.md @@ -0,0 +1,21 @@ +# v0.0.15 dlog + +## TODO + +- [ ] list things I want to do to dlog + +## Overview + +Rename `log` to `dlog` and provider better API for both write and read. + +## Motivation + +`dyweb/log` spent a lot of time on write performance instead of usability. +It has a structured logging API, but I find it hard to use for both cli and server apps. + +## Implementation + +- [ ] rename `log` package to `dlog` +- [ ] add the ability to read the log it generates +- [ ] reduce size of interface + - [ ] remove `PanicX` simply call `ErrorX` and `panic` diff --git a/doc/milestones/v0.0.16-linter/README.md b/doc/milestones/v0.0.16-linter/README.md new file mode 100644 index 0000000..de9ba99 --- /dev/null +++ b/doc/milestones/v0.0.16-linter/README.md @@ -0,0 +1,13 @@ +# v0.0.16 Linter + +## TODO + +- [ ] list spec etc. but import implementation is already half-way through ... + +## Overview + +A static analyser to enforce customized coding style. e.g. more grouping and naming rules compared w/ `goimports`. + +## Implementation + +- [ ] add a flag to error out if there are differences after formatting code, i.e. the code is not formatted, useful for CI \ No newline at end of file diff --git a/doc/milestones/v0.1.0-mvp/README.md b/doc/milestones/v0.1.0-mvp/README.md new file mode 100644 index 0000000..a3f8145 --- /dev/null +++ b/doc/milestones/v0.1.0-mvp/README.md @@ -0,0 +1,12 @@ +# Gommon v0.1.0 MVP + +## Overview + +A relative stable API for linter, log, errors, dcli, generator, etc. + +## Related + +- Children: + - [v0.0.14 dcli](../v0.0.14-dcli) + - [v0.0.15 dlog](../v0.0.15-dlog) + - [v0.0.16 linter](../v0.0.16-linter) \ No newline at end of file diff --git a/errors/errortype/pkg.go b/errors/errortype/pkg.go index ba36963..5c28d5d 100644 --- a/errors/errortype/pkg.go +++ b/errors/errortype/pkg.go @@ -1,3 +1,4 @@ // Package errortype defines helper for inspect common error types generated in standard library, // so you don't need to import tons of packages in your file for sentinel error and custom error type. +// TODO: rename to errortypes? package errortype diff --git a/errors/multi.go b/errors/multi.go index ae223bb..0380661 100644 --- a/errors/multi.go +++ b/errors/multi.go @@ -26,6 +26,8 @@ type MultiErr interface { // It returns true if the appended error is not nil, inspired by https://github.com/uber-go/multierr/issues/21 Append(error) bool // Errors returns errors stored, if no error + // TODO: multiErr returns internal []error while multiErrSafe returns a copy + // but even for thread safe error, Errors() normally get called in one go routine. Errors() []error // ErrorOrNil returns itself or nil if there are no errors, inspired by https://github.com/hashicorp/go-multierror ErrorOrNil() error @@ -36,16 +38,31 @@ type MultiErr interface { } // NewMultiErr returns a non thread safe implementation +// Deprecated: use NewMulti instead, this is an error package and *Err is redundant. func NewMultiErr() MultiErr { return &multiErr{} } +// NewMulti returns a non thread safe MultiErr implementation. +// Use NewMultiSafe if you need one protected with sync.Mutex. +// Or you can use your own locking for access from multiple goroutines. +func NewMulti() MultiErr { + return &multiErr{} +} + // NewMultiErrSafe returns a thread safe implementation which protects the underlying slice using mutex. // It returns a copy of slice when Errors is called func NewMultiErrSafe() MultiErr { return &multiErrSafe{} } +// NewMultiSafe returns a MultiErr protected with sync.Mutex. +// Use NewMulti if you use multi error in one go routine or have your own locking. +// It returns a copy of slice when Errors is called. TODO(at15): consider change this behavior or update interface doc. +func NewMultiSafe() MultiErr { + return &multiErrSafe{} +} + var ( _ MultiErr = (*multiErr)(nil) _ MultiErr = (*multiErrSafe)(nil) diff --git a/errors/pkg.go b/errors/pkg.go index 305c80d..744dcc6 100644 --- a/errors/pkg.go +++ b/errors/pkg.go @@ -34,7 +34,7 @@ func Ignore(_ error) { // do nothing } -// Ignore2 allow you to ignore return value and error, it is useful for io.Writer like functions +// Ignore2 ignores return value and error, it is useful for functions like Write(b []byte) (int64, error) // It is also inspired by dgraph x/error.go func Ignore2(_ interface{}, _ error) { // do nothing diff --git a/generator/gotmpl.go b/generator/gotmpl.go index 700739c..bd6b5a7 100644 --- a/generator/gotmpl.go +++ b/generator/gotmpl.go @@ -2,16 +2,21 @@ package generator import ( "bytes" - "go/format" + "io" "io/ioutil" "text/template" - "unicode" + + "github.com/dyweb/gommon/util/stringutil" + "golang.org/x/tools/imports" "github.com/dyweb/gommon/errors" "github.com/dyweb/gommon/util/fsutil" "github.com/dyweb/gommon/util/genutil" ) +// GoTemplateConfig maps to gomtpls config in gommon.yml. +// It reads go template source from Src and writes to Dst using Data as data. +// The Go flag determines if the rendered template is formatted as go code. type GoTemplateConfig struct { Src string `yaml:"src"` Dst string `yaml:"dst"` @@ -20,48 +25,103 @@ type GoTemplateConfig struct { } func (c *GoTemplateConfig) Render(root string) error { - var ( - b []byte - buf bytes.Buffer - err error - t *template.Template - ) + var buf bytes.Buffer //log.Infof("data is %v", c.Data) - if b, err = ioutil.ReadFile(join(root, c.Src)); err != nil { + src := join(root, c.Src) + b, err := ioutil.ReadFile(src) + if err != nil { return errors.Wrap(err, "can't read template file") } - if t, err = template.New(c.Src). - Funcs(template.FuncMap{ - "UcFirst": UcFirst, - }). - Parse(string(b)); err != nil { - return errors.Wrap(err, "can't parse template") - } buf.WriteString(genutil.DefaultHeader(join(root, c.Src))) - if err = t.Execute(&buf, c.Data); err != nil { - return errors.Wrap(err, "can't render template") + tmpl := GoCodeTemplate{ + Name: src, + Content: string(b), + Data: c.Data, + NoFormat: !c.Go, + Funcs: genutil.TemplateFuncMap(), } - if c.Go { - if b, err = format.Source(buf.Bytes()); err != nil { - return errors.Wrap(err, "can't format as go code") - } - } else { - b = buf.Bytes() + if err := RenderGoCodeTo(&buf, tmpl); err != nil { + return err } - if err = fsutil.WriteFile(join(root, c.Dst), b); err != nil { + dst := join(root, c.Dst) + if err = fsutil.WriteFile(dst, buf.Bytes()); err != nil { return err } - log.Debugf("rendered go tmpl %s to %s", join(root, c.Src), join(root, c.Dst)) + log.Debugf("rendered go tmpl %s to %s", src, dst) return nil } -// UcFirst change first character to upper case. -// It is based on https://github.com/99designs/gqlgen/blob/master/codegen/templates/templates.go#L205 -func UcFirst(s string) string { - if s == "" { - return "" +// ---------------------------------------------------------------------------- +// GoCodeTemplate + +type GoCodeTemplate struct { + Name string // name used in error message e.g. generic-btree + Content string // the actual template content + Data interface{} // template data + NoFormat bool // disable calling goimports + Funcs template.FuncMap // additional template function map +} + +func RenderGoCode(tmpl GoCodeTemplate) ([]byte, error) { + // Sanity check + if len(tmpl.Name) > len(tmpl.Content) { + return nil, errors.Errorf("template name is longer than content, wrong order? shorter one is %s", + stringutil.Shorter(tmpl.Name, tmpl.Content)) + } + parsed, err := template.New(tmpl.Name).Parse(tmpl.Content) + if err != nil { + return nil, err + } + var buf bytes.Buffer + if err := parsed.Execute(&buf, tmpl.Data); err != nil { + return nil, errors.Wrapf(err, "error render template %s", tmpl.Name) + } + if tmpl.NoFormat { + return buf.Bytes(), nil + } + formatted, err := FormatGo(buf.Bytes()) + if err != nil { + return nil, errors.Wrap(err, "error format generated code") + } + return formatted, nil +} + +func RenderGoCodeTo(dst io.Writer, tmpl GoCodeTemplate) error { + // Sanity check + if dst == nil { + return errors.New("nil writer for RenderGoCode, forgot to check error when create file?") + } + b, err := RenderGoCode(tmpl) + if err != nil { + return err + } + _, err = dst.Write(b) + return err +} + +// ---------------------------------------------------------------------------- +// Go Util + +type GoStructDef struct { + Name string + Fields []GoFieldDef +} + +type GoFieldDef struct { + Name string + Type string + Tag string +} + +// FormatGo formats go code using goimports without out fixing missing imports. +func FormatGo(src []byte) ([]byte, error) { + opt := &imports.Options{ + Fragment: false, + AllErrors: true, + Comments: true, + TabIndent: true, + TabWidth: 8, + FormatOnly: true, } - r := []rune(s) - r[0] = unicode.ToUpper(r[0]) - return string(r) + return imports.Process("", src, opt) } diff --git a/generator/gotmpl_test.go b/generator/gotmpl_test.go index fa90e23..6e2e01d 100644 --- a/generator/gotmpl_test.go +++ b/generator/gotmpl_test.go @@ -5,13 +5,13 @@ import ( "testing" "text/template" - "github.com/dyweb/gommon/generator" + "github.com/dyweb/gommon/util/genutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestUcFirst(t *testing.T) { - tmpl, err := template.New("test_ucfirst").Funcs(template.FuncMap{"UcFirst": generator.UcFirst}).Parse(`{{ .foo | UcFirst }}`) + tmpl, err := template.New("test_ucfirst").Funcs(template.FuncMap{"UcFirst": genutil.UcFirst}).Parse(`{{ .foo | UcFirst }}`) require.Nil(t, err) var buf bytes.Buffer err = tmpl.Execute(&buf, map[string]string{ diff --git a/generator/pkg.go b/generator/pkg.go index 048bae8..42eb19b 100644 --- a/generator/pkg.go +++ b/generator/pkg.go @@ -12,8 +12,10 @@ const ( DefaultGeneratedFile = "gommon_generated.go" ) -var logReg = dlog.NewRegistry() -var log = logReg.Logger() +var ( + logReg = dlog.NewRegistry() + log = logReg.Logger() +) type ConfigFile struct { // Loggers is helper methods on struct for gommon/log to build a tree for logger, this is subject to change diff --git a/generator/util.go b/generator/util.go index 536393d..7428285 100644 --- a/generator/util.go +++ b/generator/util.go @@ -20,6 +20,7 @@ func DefaultIgnores() *fsutil.Ignores { ) } +// join is alias for filepath.Join func join(s ...string) string { return filepath.Join(s...) } diff --git a/go.mod b/go.mod index cd9b478..d7603af 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module github.com/dyweb/gommon +go 1.14 + require ( github.com/davecgh/go-spew v1.1.1 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 github.com/stretchr/testify v1.4.0 + golang.org/x/tools v0.0.0-20200401192744-099440627f01 gopkg.in/yaml.v2 v2.2.7 ) - -go 1.13 diff --git a/go.sum b/go.sum index 3f1a60b..5742e4c 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,26 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200401192744-099440627f01 h1:ysQJ/fU6laLOZJseIeOqXl6Mo+lw5z6b7QHnmUKjW+k= +golang.org/x/tools v0.0.0-20200401192744-099440627f01/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= diff --git a/hack/README.md b/hack/README.md new file mode 100644 index 0000000..d79f1f6 --- /dev/null +++ b/hack/README.md @@ -0,0 +1,5 @@ +# Hack + +Hack contains script and manifests for setting up develop environment. + +- [Go Docker Builder Image](go-dev) \ No newline at end of file diff --git a/hack/go-dev/Dockerfile b/hack/go-dev/Dockerfile index 1226c69..d811e83 100644 --- a/hack/go-dev/Dockerfile +++ b/hack/go-dev/Dockerfile @@ -36,11 +36,11 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" ARG BUILD_GO_VERSION=1.11.2 -# glide no longer have release, just hard code it to latest version +# TODO: Deprecate glide and dep, we are all go mod now + ENV GO_VERSION=$BUILD_GO_VERSION \ GLIDE_VERSION=v0.13.2 -# TODO: might put glide under GOPATH/bin or just remove it entirely, not sure if anyone is still using it RUN \ curl -L https://dl.google.com/go/go$GO_VERSION.linux-amd64.tar.gz | tar -C /usr/local -xz \ && curl -sSL https://github.com/Masterminds/glide/releases/download/$GLIDE_VERSION/glide-$GLIDE_VERSION-linux-amd64.tar.gz \ diff --git a/hack/go-dev/Makefile b/hack/go-dev/Makefile index ead872a..2994b57 100644 --- a/hack/go-dev/Makefile +++ b/hack/go-dev/Makefile @@ -1,5 +1,5 @@ DOCKER_REPO = dyweb/go-dev -GO_VERSIONS = 1.14 +GO_VERSIONS = 1.15.3 BUILDS = $(addprefix build-, $(GO_VERSIONS)) PUSHS = $(addprefix push-, $(GO_VERSIONS)) @@ -16,4 +16,4 @@ build: $(BUILDS) push: $(PUSHS) run: - docker run --rm -it --entrypoint /bin/bash $(DOCKER_REPO):1.11.4 \ No newline at end of file + docker run --rm -it --entrypoint /bin/bash $(DOCKER_REPO):1.15.3 \ No newline at end of file diff --git a/httpclient/README.md b/httpclient/README.md index 0e3e9f5..ec6f478 100644 --- a/httpclient/README.md +++ b/httpclient/README.md @@ -1,2 +1,15 @@ # HTTPClient +This package loosely models after [exp-httpclient](https://github.com/bradfitz/exp-httpclient). +Though it wraps `net/http` instead of reverse. + +## TODO + +- [ ] list features supported by this package +- [ ] create an example +- [ ] unit test + +## Ref + +- https://github.com/bradfitz/exp-httpclient +- [net/http: new HTTP client package](https://github.com/golang/go/issues/23707) \ No newline at end of file diff --git a/httpclient/client.go b/httpclient/client.go index 352fa38..4e775be 100644 --- a/httpclient/client.go +++ b/httpclient/client.go @@ -69,7 +69,7 @@ func (c *Client) SetHeader(k, v string) *Client { } // GetHeaders a copy of headers set on the client, -// it will return a empty but non nil map even if no header is set +// it will return a empty but non nil map even if no header is set. func (c *Client) GetHeaders() map[string]string { if c.headers == nil { return make(map[string]string) diff --git a/httpclient/context.go b/httpclient/context.go index 72a36ea..6dc76f6 100644 --- a/httpclient/context.go +++ b/httpclient/context.go @@ -7,10 +7,8 @@ import ( var _ context.Context = (*Context)(nil) -// Context -// -// It is lazy initialized, only call `make` when they are actually write to, -// so all the maps are EMPTY even when using factory func. +// Context implements context.Context and provides HTTP request specific helpers. +// It is lazy initialized, only call `make` when there is write to internal maps. // User (including this package itself) should use setter when set value. type Context struct { // base overrides base path set in client if it is not empty @@ -23,9 +21,7 @@ type Context struct { // if it has non nil value errHandler ErrorHandler - // values improve performance by set value in place - // TODO: do I really need this map? - values map[string]interface{} + // wraps a standard context implementation for value and deadline stdCtx context.Context } @@ -57,6 +53,8 @@ func ConvertContext(ctx context.Context) *Context { return NewContext(ctx) } +// SetBase allow a single request to override client level request base. +// This is useful when most request is /api/bla and suddenly there is a /bla/api. func (c *Context) SetBase(s string) *Context { c.base = s return c @@ -114,15 +112,6 @@ func (c *Context) Err() error { // if not found, it use the underlying context.Context if is set // if not set, it returns nil func (c *Context) Value(key interface{}) interface{} { - if c != nil && c.values != nil { - k, ok := key.(string) - if ok { - v, ok := c.values[k] - if ok { - return v - } - } - } if c != nil && c.stdCtx != nil { return c.stdCtx.Value(key) } diff --git a/httpclient/method.go b/httpclient/method.go index 0efedf6..0bf245f 100644 --- a/httpclient/method.go +++ b/httpclient/method.go @@ -6,7 +6,7 @@ import ( "github.com/dyweb/gommon/util/httputil" ) -// method.go contains wrapper for common http verbs, GET, POST, PATCH, DELETE +// method.go contains wrapper for common http verbs, GET, POST, PUT, PATCH, DELETE // GET @@ -36,6 +36,20 @@ func (c *Client) PostIgnoreRes(ctx *Context, path string, reqBody interface{}) e return c.FetchToNull(ctx, httputil.Post, path, reqBody) } +// PUT + +func (c *Client) Put(ctx *Context, path string, reqBody interface{}, resBody interface{}) error { + return c.FetchTo(ctx, httputil.Put, path, reqBody, resBody) +} + +func (c *Client) PutRaw(ctx *Context, path string, reqBody interface{}) (*http.Response, error) { + return c.Do(ctx, httputil.Put, path, reqBody) +} + +func (c *Client) PutIgnoreRes(ctx *Context, path string, reqBody interface{}) error { + return c.FetchToNull(ctx, httputil.Put, path, reqBody) +} + // PATCH func (c *Client) Patch(ctx *Context, path string, reqBody interface{}, resBody interface{}) error { diff --git a/httpclient/option.go b/httpclient/option.go index 7d64001..bcb80c9 100644 --- a/httpclient/option.go +++ b/httpclient/option.go @@ -29,6 +29,12 @@ func WithErrorHandlerFunc(f ErrorHandlerFunc) Option { return nil } } +func WithClient(h *http.Client) Option { + return func(c *Client) error { + c.h = h + return nil + } +} func WithTransport(tr *http.Transport) Option { return func(c *Client) error { diff --git a/httpclient/pkg.go b/httpclient/pkg.go index f79cbba..6bc6747 100644 --- a/httpclient/pkg.go +++ b/httpclient/pkg.go @@ -1,8 +1,8 @@ // Package httpclient is a high level wrapper around net/http with more types and easier to use interface -// TODO: ref https://github.com/bradfitz/exp-httpclient +// It is loosely modeled after https://github.com/bradfitz/exp-httpclient package httpclient -// UnixBasePath is used as a placeholder for unix domain socket client, -// only the protocol http is needed, host is ignored because dialer use socket path -// TODO: what about tls over unix domain socket? +// UnixBasePath is used as a placeholder for unix domain socket client. +// Only the protocol http is needed, host can be anything because dialer use socket path +// TODO: what about https over unix domain socket? const UnixBasePath = "http://localhost" diff --git a/linter/README.md b/linter/README.md new file mode 100644 index 0000000..b663a88 --- /dev/null +++ b/linter/README.md @@ -0,0 +1,28 @@ +# Gommon Linter + +## Overview + +Package `linter` is a code formatter and linter. It extends `goimports` with custom grouping rules. + +## Install + +- [ ] FIXME: rename the binary package to `gom` so we can download using go get + +```bash +mkdir /tmp/gommon && cd /tmp/gommon && go get github.com/dyweb/gommon/cmd/gommon +``` + +## Usage + +Format + +- `gommon format` flags are compatible with `goimports` + +```bash +# print diff, list file and update file in place for go files under folder ./server ./client (recursively) +gommon format -d -l -w ./server ./client +``` + +## Internal + +- [ ] TODO: talks about how the format works (maybe link to the blog post) diff --git a/linter/doc/import.md b/linter/doc/import.md new file mode 100644 index 0000000..5d7088d --- /dev/null +++ b/linter/doc/import.md @@ -0,0 +1,52 @@ +# Gommon Import Linter and Formatter + +## TODO + +- [ ] `LocalPrefix` kind of does the extra grouping +- [ ] Check forbidden import +- [ ] More groups for import + +## Overview + +`goimports` with custom grouping rules, package black list and alias naming check. + +## Motivation + +`goimports` improves `gofmt` by doing additional import grouping. However it is missing the following features: + +- customize group rules, e.g. put all proto import at the bottom, split import from current project with external libs +- black list specific packages, sometimes IDE are too smart and introduced unknown import on the fly +- validation on import rename, e.g. rename lengthy proto package to `foopb` + +## Design + +`goimports` binary is a thin cli that calls `x/tools/imports` which calls `x/tools/internal/imports.Process`. +After merging, sorting and grouping import, it use `go/printer` to dump the ast and call `go/format.Source`. +(Essentially the code is pared twice, not sure why format.format is not exported). + +It is possible to duplicate the format functionality in `x/tools/internal/imports`. However `goimports` can fix missing import, +and that logic is actually much larger than format and result in 17k lines of code for `imports` package. +So we take the easy way and run `goimport` before running `gommon/linter/import`. +One major drawback is the code is parsed and printed several times (in memory), so it should work for medium/small projects. + +The overall flow is like following: + +``` +walkDir { + src = readFile(p) + b1 = imports.Process(p, src) // run goimports + ast = parse(b1) + err = lint.CheckImport(ast) + b2 = lint.FormatImport(ast) + diff(src, b2) +} +``` + +## Implementation + +See [import.go](../import.go) + +- `walkDir` and `diff` can copy from `goimports` binary until we have a better `fsutil.WalkWithIgnore` implementation +- `CheckImport` should use a default set of rules + - so user can provide their own rules by writing their own binary +- `FormatImport` should use a default set of rules for grouping \ No newline at end of file diff --git a/linter/import.go b/linter/import.go new file mode 100644 index 0000000..6e5a144 --- /dev/null +++ b/linter/import.go @@ -0,0 +1,363 @@ +package linter + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "io" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/dyweb/gommon/errors" + "github.com/dyweb/gommon/util/fsutil" + "golang.org/x/tools/imports" +) + +// import.go checks if there are deprecated import and sort import by group + +// ---------------------------------------------------------------------------- +// goimports shim + +const TabWidth = 8 + +// GoimportFlags is a struct whose fields map to goimports command flags. +type GoimportFlags struct { + List bool + Write bool + Diff bool + // TODO: srcdir, single file as if in dir xxx + AllErrors bool + LocalPrefix string + FormatOnly bool +} + +// CheckAndFormatImportToStdout calls CheckAndFormatImport and update file and/or print diff +func CheckAndFormatImportToStdout(p string, flags GoimportFlags) error { + res, err := CheckAndFormatImport(p, flags) + if err != nil { + return err + } + return printImportDiff(os.Stdout, flags, p, res.Src, res.Formatted) +} + +func CheckAndFormatImport(p string, flags GoimportFlags) (*FormatResult, error) { + log.Debugf("check and format %s", p) + src, err := ioutil.ReadFile(p) + if err != nil { + return nil, err + } + + // goimports + opt := imports.Options{ + TabWidth: TabWidth, + TabIndent: true, + Comments: true, + Fragment: false, // Set it to false because we are reading a full go file + // NOTE: we don't have Env because it is in internal/imports and relies on default env. + AllErrors: flags.AllErrors, + FormatOnly: flags.FormatOnly, + } + // TODO: we can't set LocalPrefix in env, and the package var seems to be left for this purpose + if flags.LocalPrefix != "" { + imports.LocalPrefix = flags.LocalPrefix + } + goimportRes, err := imports.Process(p, src, &opt) + if err != nil { + return nil, errors.Wrap(err, "error calling goimports") + } + + // Parse again ... + fset := token.NewFileSet() + // NOTE: we can't use parser.ImportOnly because we need all the AST to print the file back. + mode := parser.ParseComments + if flags.AllErrors { + mode |= parser.AllErrors + } + f, err := parser.ParseFile(fset, p, goimportRes, mode) + if err != nil { + return nil, errors.Wrap(err, "error parse goimport formatted code") + } + + // Check before format + // TODO: pass in lint rule + if err := CheckImport(f, nil); err != nil { + return nil, err + } + + // Format again with custom rules + // TODO: pass in format config from outside + fmtCfg := ImportFormatConfig{} + res, err := FormatImport(f, fset, fmtCfg) + if err != nil { + return nil, err + } + + return &FormatResult{ + Src: src, + Formatted: res, + }, nil +} + +// ---------------------------------------------------------------------------- +// Import Context + +// ImportContext contains information from ImportSpec and current file. +type ImportContext struct { + Name string // import alias or dot import + Path string // import path e.g. github.com/foo/bar/bla + File string // file name only, e.g. foo.go + Folder string // folder, e.g. github.com/gommon/errors TODO: this depends on where the command runs + Package string // package of current file +} + +func importSpecToContext(spec *ast.ImportSpec, pkg string) ImportContext { + return ImportContext{ + Name: nameIfNotNil(spec.Name), + Path: spec.Path.Value, + // TODO: file etc. + Package: pkg, + } +} + +// ---------------------------------------------------------------------------- +// Check Import + +// TODO: convert rules from config? +type ImportCheckConfig struct { +} + +type ImportCheckRule interface { + RuleName() string + Check(ctx ImportContext) error +} + +type ImportDeprecated struct { + Path string // import path of deprecated package e.g. github.com/foo/bar + Suggested string // suggested new package e.g. github.com/bar/foo + Reason string // why, link to issues etc. +} + +func CheckImport(f *ast.File, rules []ImportCheckRule) error { + pkg := nameIfNotNil(f.Name) + imps, _ := extractImports(f) + merr := errors.NewMulti() + for _, imp := range imps { + for _, r := range rules { + // TODO: need to attach more context and maybe have extra grouping + merr.Append(r.Check(importSpecToContext(imp, pkg))) + } + } + return merr.ErrorOrNil() +} + +// ---------------------------------------------------------------------------- +// Format Import + +// FormatResult avoids remembering order of src and dst when returning two bytes. +type FormatResult struct { + Src []byte + Formatted []byte +} + +// TODO: missing other rules +type ImportFormatConfig struct { + // Path of current project, imports from current project are grouped together + ProjectPath string + + GroupRules []ImportGroupRule +} + +// ImportGroupRule checks if an import belongs to a group. +// TODO: might change to interface +type ImportGroupRule struct { + // Name is a short string for enabling/disabling rules. + Name string + // Match determines if the import spec matches this group. + Match func(ctx ImportContext) bool +} + +var importStd = ImportGroupRule{ + Name: "std", + Match: func(ctx ImportContext) bool { + // NOTE: it is based on goimports, if the path has no domain name, it is standard library. + if !strings.Contains(ctx.Path, ".") { + return true + } + return false + }, +} + +var importCurrentPackage = ImportGroupRule{ + Name: "currentPackage", + Match: func(ctx ImportContext) bool { + // TODO: this does not work if we don't run gommon format at project root + if strings.HasSuffix(ctx.Folder, ctx.Path) { + return true + } + // FIXME: the package name may not match the folder name foo/bar, package boar + return false + }, +} + +func DefaultImportGroupRules() []ImportGroupRule { + return []ImportGroupRule{ + importCurrentPackage, + importStd, + } +} + +// TODO: split the API, one use cfg, one using rules (i.e. full customization) +func FormatImport(f *ast.File, fset *token.FileSet, cfg ImportFormatConfig) ([]byte, error) { + rules := cfg.GroupRules + if len(rules) == 0 { + // TODO: need to pass project dir, and make it easy for user to define rules + rules = DefaultImportGroupRules() + } + // Change import order and do the grouping. + if err := adjustImport(f, fset, rules); err != nil { + return nil, err + } + + // TODO: apply the rules + // TODO: the hard part is how to keep the position valid after modifying AST + + // Print AST + pCfg := printer.Config{ + Mode: printer.UseSpaces | printer.TabIndent, + Tabwidth: TabWidth, + Indent: 0, + } + var buf bytes.Buffer + if err := pCfg.Fprint(&buf, fset, f); err != nil { + return nil, errors.Wrap(err, "error print ast") + } + return buf.Bytes(), nil +} + +func adjustImport(f *ast.File, fset *token.FileSet, rules []ImportGroupRule) error { + specs, nBlocks := extractImports(f) + if nBlocks > 1 { + return errors.Errorf("goimports not called, expect import declaration merged into one block, but got %d", nBlocks) + } + // TODO: linter says decl.Specs can be nil ... writing a linter w/ linter error + for _, imp := range specs { + log.Debugf("adjustImport: spec %s %s", nameIfNotNil(imp.Name), imp.Path.Value) + } + return nil +} + +func extractImports(f *ast.File) ([]*ast.ImportSpec, int) { + var ( + imps []*ast.ImportSpec + nBlocks int + ) + for _, d := range f.Decls { + d, ok := d.(*ast.GenDecl) + if !ok || d.Tok != token.IMPORT { + break + } + nBlocks++ + for _, spec := range d.Specs { + imps = append(imps, spec.(*ast.ImportSpec)) + } + } + return imps, nBlocks +} + +func nameIfNotNil(id *ast.Ident) string { + if id == nil { + return "" + } + return id.Name +} + +// ---------------------------------------------------------------------------- +// Diff formatted go file + +// NOTE: Copied from processFile in goimports +// If there is diff after format, try update file in place and print diff. +func printImportDiff(out io.Writer, flags GoimportFlags, p string, src []byte, formatted []byte) error { + if bytes.Equal(src, formatted) { + return nil + } + + // Print file name + if flags.List { + fmt.Fprintln(out, p) + } + // Update file directly + if flags.Write { + // TODO: why goimports use 0 for file permission? + if err := fsutil.WriteFile(p, formatted); err != nil { + return err + } + } + // Shell out to diff + if flags.Diff { + // TODO(upstream): goimports is using Printf instead Fprintf + fmt.Fprintf(out, "diff -u %s %s\n", filepath.ToSlash(p+".orig"), filepath.ToSlash(p)) + diff, err := diffBytes(src, formatted, p) + if err != nil { + log.Warnf("diff failed %s", err) + } else { + out.Write(diff) + } + } + + // No flags, dump formatted (may or may not changed) to stdout + if !flags.List && !flags.Write && !flags.Diff { + if _, err := out.Write(formatted); err != nil { + return err + } + } + return nil +} + +// Writes bytes to two temp files and shell out to diff. +// Replaces file name in output +// TODO: we can use diffBytes in other packages, might move it to a util package, bytesutil? +func diffBytes(a []byte, b []byte, filename string) ([]byte, error) { + files, err := fsutil.WriteTempFiles("", "gomfmt", a, b) + if err != nil { + return nil, err + } + defer fsutil.RemoveFiles(files) + + diff, err := fsutil.Diff(files[0], files[1]) + if err != nil { + return nil, err + } + // No diff + if len(diff) == 0 { + return nil, nil + } + + // Replace temp file name with original file path + // NOTE: it is based on replaceTempFilename in goimports + segs := bytes.SplitN(diff, []byte{'\n'}, 3) + // NOTE: we ignore invalid diff output and returns whatever the output is. + // This is different from goimports which stops and return nil. + if len(segs) < 3 { + return diff, nil + } + + // COPY: Copied from goimports replaceTempFilename + // Preserve timestamps. + var t0, t1 []byte + if i := bytes.LastIndexByte(segs[0], '\t'); i != -1 { + t0 = segs[0][i:] + } + if i := bytes.LastIndexByte(segs[1], '\t'); i != -1 { + t1 = segs[1][i:] + } + // Always print filepath with slash separator. + f := filepath.ToSlash(filename) + segs[0] = []byte(fmt.Sprintf("--- %s%s", f+".orig", t0)) + segs[1] = []byte(fmt.Sprintf("+++ %s%s", f, t1)) + return bytes.Join(segs, []byte{'\n'}), nil +} diff --git a/linter/pkg.go b/linter/pkg.go new file mode 100644 index 0000000..1548ff3 --- /dev/null +++ b/linter/pkg.go @@ -0,0 +1,22 @@ +// Package linter use static analysis to enforce customized coding style and fix(format) go code. +package linter + +import ( + dlog "github.com/dyweb/gommon/log" +) + +var ( + logReg = dlog.NewRegistry() + log = logReg.Logger() +) + +// Level describes lint error level +// TODO: sync w/ existing lint tools +type Level int + +const ( + UnknownLevel Level = iota + Deprecated + Warn + Error +) diff --git a/log/doc/survey/README.md b/log/doc/survey/README.md index a7f7794..dd8e09a 100644 --- a/log/doc/survey/README.md +++ b/log/doc/survey/README.md @@ -1,4 +1,4 @@ -# Survey +# Go Log Library Survey https://github.com/avelino/awesome-go#logging @@ -22,11 +22,15 @@ Structured Java(ish) -- [solr](solr.md) the last straw that drive us to log v2, gives you [a tree graph to control log level of ALL the packages](solr-log-admin.png), including dependencies -- [seelog](seelog.md) javaish, fine grained control log filtering (by func, file etc.) +- [solr](solr.md) the last straw that drives us to log v2, gives you [a tree graph to control log level of ALL the packages](solr-log-admin.png), including dependencies +- [seelog](seelog.md) javaish, fine grained control log filtering (by func, file etc.) at log site - [log4j](log4j.md) java logger - [ ] TODO: might check open tracing as well, instrument like code should be put into other package Logging library used by popular go projects -- k8s, [CockroachDB](https://github.com/cockroachdb/cockroach/tree/master/pkg/util/log) glog \ No newline at end of file +- k8s, [CockroachDB](https://github.com/cockroachdb/cockroach/tree/master/pkg/util/log) glog + +Rotate + +- [ ] https://github.com/lestrrat-go/file-rotatelogs \ No newline at end of file diff --git a/noodle/README.md b/noodle/README.md index f93043f..d540617 100644 --- a/noodle/README.md +++ b/noodle/README.md @@ -63,5 +63,6 @@ func main() { ## References and Alternatives +- [Go Embed Draft](https://go.googlesource.com/proposal/+/master/design/draft-embed.md) - [Proposal to add it to cmd/go](https://github.com/golang/go/issues/35950) - [Feature request to go.rice back in 2016](https://github.com/GeertJohan/go.rice/issues/83) \ No newline at end of file diff --git a/noodle/_examples/embed/gen/noodle.go b/noodle/_examples/embed/gen/noodle.go index d7a76e2..8338786 100644 --- a/noodle/_examples/embed/gen/noodle.go +++ b/noodle/_examples/embed/gen/noodle.go @@ -14,129 +14,129 @@ func GetNoodleAssets() (noodle.EmbedBowel, error) { dirs := map[string]noodle.EmbedDir{"": { FileInfo: noodle.FileInfo{ FileName: "assets", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 256, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, Entries: []noodle.FileInfo{{ FileName: "404", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 96, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, { FileName: "idx", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 256, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, { FileName: "noidx", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 160, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, { FileName: ".noodleignore", FileSize: 147, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }, { FileName: "index.html", FileSize: 105, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }}, }, "/404": { FileInfo: noodle.FileInfo{ FileName: "404", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 96, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, Entries: []noodle.FileInfo{}, }, "/idx": { FileInfo: noodle.FileInfo{ FileName: "idx", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 256, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, Entries: []noodle.FileInfo{{ FileName: "sub", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 96, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, { FileName: "index.html", FileSize: 180, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }, { FileName: "main.css", FileSize: 38, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }, { FileName: "main.js", FileSize: 51, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }}, }, "/idx/sub": { FileInfo: noodle.FileInfo{ FileName: "sub", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 96, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, Entries: []noodle.FileInfo{{ FileName: "index.html", FileSize: 115, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }}, }, "/noidx": { FileInfo: noodle.FileInfo{ FileName: "noidx", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 160, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, Entries: []noodle.FileInfo{{ FileName: "main.css", FileSize: 37, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }, { FileName: "main.js", FileSize: 50, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }, { FileName: "noindex.html", FileSize: 192, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }}, }} - data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x54, 0xcb, 0xb1, 0xaa, 0x85, 0x30, 0x10, 0x84, 0xe1, 0x7e, 0x9f, 0x62, 0xc0, 0x2e, 0x85, 0xde, 0xc2, 0xf7, 0xb9, 0x4, 0x77, 0x34, 0x81, 0x4d, 0x2, 0xc9, 0x86, 0xe3, 0xe3, 0x1f, 0xac, 0xe4, 0x94, 0xc3, 0x37, 0xff, 0x2, 0x4f, 0x79, 0x20, 0xe2, 0x68, 0xa5, 0xb0, 0xba, 0xd4, 0xa6, 0xfc, 0x2f, 0x4d, 0xa7, 0x71, 0x60, 0x41, 0xbe, 0x6a, 0xeb, 0x44, 0x8d, 0x85, 0x22, 0x61, 0x4d, 0xf3, 0xe2, 0x99, 0x8d, 0xbf, 0x82, 0x4f, 0x36, 0x3d, 0x62, 0x57, 0x91, 0xfd, 0x6f, 0xdf, 0xc2, 0xab, 0xd1, 0xc, 0x9e, 0x88, 0xa7, 0x19, 0x98, 0x55, 0xd9, 0x11, 0x71, 0x36, 0x53, 0x76, 0xc9, 0x7a, 0x6f, 0x61, 0xf5, 0xdb, 0xdf, 0xff, 0xb3, 0xbe, 0x1, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0x28, 0xca, 0xcf, 0x2f, 0xb1, 0xd1, 0x87, 0x8, 0x70, 0xd9, 0xe8, 0x43, 0x14, 0xd9, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0xb4, 0x18, 0x22, 0x14, 0xa9, 0x17, 0x2b, 0x64, 0xe6, 0xa5, 0xa4, 0x56, 0xe8, 0x81, 0x8c, 0xb3, 0xd1, 0xcf, 0x30, 0x4, 0x29, 0x87, 0xaa, 0xd3, 0x7, 0x5b, 0x1, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x64, 0xce, 0x31, 0xae, 0x3, 0x21, 0xc, 0x4, 0xd0, 0x9e, 0x53, 0x58, 0x1c, 0x60, 0xd1, 0xf6, 0x5e, 0xf7, 0xff, 0x18, 0x7c, 0x70, 0x4, 0x89, 0x21, 0x11, 0x76, 0x91, 0xbd, 0x7d, 0x44, 0xd8, 0x2e, 0xed, 0x68, 0xfc, 0x3c, 0x58, 0xac, 0x9, 0x39, 0x2c, 0x1c, 0x33, 0x39, 0x0, 0x0, 0xb4, 0x6a, 0xc2, 0xf4, 0x7, 0xb1, 0x41, 0xed, 0x99, 0xdf, 0xdb, 0xac, 0x60, 0x58, 0xf1, 0xaa, 0x48, 0xed, 0xf, 0x18, 0x2c, 0x87, 0x57, 0x3b, 0x85, 0xb5, 0x30, 0x9b, 0x87, 0x32, 0xf8, 0x76, 0xf8, 0x16, 0x6b, 0xdf, 0x92, 0xaa, 0x27, 0x87, 0x61, 0xb1, 0xf8, 0xff, 0xcc, 0xe7, 0x7c, 0xb2, 0xff, 0xb2, 0x65, 0x27, 0x87, 0x9a, 0x46, 0x7d, 0x19, 0xe8, 0x48, 0xd7, 0xfd, 0x5d, 0x3d, 0x61, 0x58, 0xf1, 0x74, 0x2e, 0x20, 0x7c, 0xd7, 0x7e, 0x2, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0xb7, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0x57, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x12, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0xc8, 0xcc, 0x4b, 0x49, 0xad, 0xd0, 0x3, 0x29, 0xb1, 0xd1, 0x87, 0x8, 0x73, 0xd9, 0xe8, 0x43, 0x94, 0xda, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0x34, 0x1a, 0xa2, 0x2b, 0x55, 0xc8, 0xcc, 0x2b, 0xce, 0x4c, 0x49, 0x55, 0x28, 0x2e, 0x4d, 0xb2, 0xd1, 0xcf, 0x30, 0x4, 0x69, 0x81, 0xaa, 0xd5, 0x7, 0x5b, 0x6, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0xb3, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0x53, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x6c, 0xce, 0x31, 0xe, 0xc3, 0x20, 0xc, 0x40, 0xd1, 0x9d, 0x53, 0x58, 0x1c, 0x20, 0x28, 0xbb, 0xe3, 0xbd, 0xc7, 0xa0, 0xe0, 0xca, 0xb4, 0x40, 0x2a, 0xec, 0xa1, 0xb9, 0x7d, 0x95, 0x92, 0xb1, 0xab, 0xfd, 0xfd, 0x64, 0x14, 0x6b, 0x95, 0x1c, 0xa, 0xc7, 0x4c, 0xe, 0x0, 0x0, 0xad, 0x58, 0x65, 0xba, 0x41, 0x6c, 0xd0, 0x77, 0x83, 0xd2, 0x33, 0x7f, 0x96, 0x33, 0xc3, 0x30, 0x57, 0x33, 0xab, 0xa5, 0xbf, 0x60, 0x70, 0xdd, 0xbc, 0xda, 0x51, 0x59, 0x85, 0xd9, 0x3c, 0xc8, 0xe0, 0xc7, 0xe6, 0x5b, 0x2c, 0x7d, 0x49, 0xaa, 0x9e, 0x1c, 0x86, 0x49, 0xe3, 0x7d, 0xcf, 0xc7, 0x75, 0x2a, 0xeb, 0x7f, 0x5e, 0x56, 0x72, 0xa8, 0x69, 0x94, 0xb7, 0x81, 0x8e, 0x74, 0x39, 0x4f, 0xf5, 0x84, 0x61, 0x8e, 0x4f, 0x6f, 0x42, 0x18, 0x7e, 0x9f, 0x7f, 0x3, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xb1, 0x0, 0x0, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x43, 0x1, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x4, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x73, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x12, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xee, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x8c, 0x3, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0xfa, 0x3, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x74, 0x4, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x9, 0x0, 0x9, 0x0, 0x64, 0x2, 0x0, 0x0, 0x39, 0x5, 0x0, 0x0, 0x0, 0x0} + data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x54, 0xcb, 0xb1, 0xaa, 0x85, 0x30, 0x10, 0x84, 0xe1, 0x7e, 0x9f, 0x62, 0xc0, 0x2e, 0x85, 0xde, 0xc2, 0xf7, 0xb9, 0x4, 0x77, 0x34, 0x81, 0x4d, 0x2, 0xc9, 0x86, 0xe3, 0xe3, 0x1f, 0xac, 0xe4, 0x94, 0xc3, 0x37, 0xff, 0x2, 0x4f, 0x79, 0x20, 0xe2, 0x68, 0xa5, 0xb0, 0xba, 0xd4, 0xa6, 0xfc, 0x2f, 0x4d, 0xa7, 0x71, 0x60, 0x41, 0xbe, 0x6a, 0xeb, 0x44, 0x8d, 0x85, 0x22, 0x61, 0x4d, 0xf3, 0xe2, 0x99, 0x8d, 0xbf, 0x82, 0x4f, 0x36, 0x3d, 0x62, 0x57, 0x91, 0xfd, 0x6f, 0xdf, 0xc2, 0xab, 0xd1, 0xc, 0x9e, 0x88, 0xa7, 0x19, 0x98, 0x55, 0xd9, 0x11, 0x71, 0x36, 0x53, 0x76, 0xc9, 0x7a, 0x6f, 0x61, 0xf5, 0xdb, 0xdf, 0xff, 0xb3, 0xbe, 0x1, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0x28, 0xca, 0xcf, 0x2f, 0xb1, 0xd1, 0x87, 0x8, 0x70, 0xd9, 0xe8, 0x43, 0x14, 0xd9, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0xb4, 0x18, 0x22, 0x14, 0xa9, 0x17, 0x2b, 0x64, 0xe6, 0xa5, 0xa4, 0x56, 0xe8, 0x81, 0x8c, 0xb3, 0xd1, 0xcf, 0x30, 0x4, 0x29, 0x87, 0xaa, 0xd3, 0x7, 0x5b, 0x1, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x64, 0xce, 0x31, 0xae, 0x3, 0x21, 0xc, 0x4, 0xd0, 0x9e, 0x53, 0x58, 0x1c, 0x60, 0xd1, 0xf6, 0x5e, 0xf7, 0xff, 0x18, 0x7c, 0x70, 0x4, 0x89, 0x21, 0x11, 0x76, 0x91, 0xbd, 0x7d, 0x44, 0xd8, 0x2e, 0xed, 0x68, 0xfc, 0x3c, 0x58, 0xac, 0x9, 0x39, 0x2c, 0x1c, 0x33, 0x39, 0x0, 0x0, 0xb4, 0x6a, 0xc2, 0xf4, 0x7, 0xb1, 0x41, 0xed, 0x99, 0xdf, 0xdb, 0xac, 0x60, 0x58, 0xf1, 0xaa, 0x48, 0xed, 0xf, 0x18, 0x2c, 0x87, 0x57, 0x3b, 0x85, 0xb5, 0x30, 0x9b, 0x87, 0x32, 0xf8, 0x76, 0xf8, 0x16, 0x6b, 0xdf, 0x92, 0xaa, 0x27, 0x87, 0x61, 0xb1, 0xf8, 0xff, 0xcc, 0xe7, 0x7c, 0xb2, 0xff, 0xb2, 0x65, 0x27, 0x87, 0x9a, 0x46, 0x7d, 0x19, 0xe8, 0x48, 0xd7, 0xfd, 0x5d, 0x3d, 0x61, 0x58, 0xf1, 0x74, 0x2e, 0x20, 0x7c, 0xd7, 0x7e, 0x2, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0xb7, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0xa8, 0x4c, 0xcd, 0xc9, 0xc9, 0x2f, 0x57, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x12, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0xb2, 0xc9, 0x28, 0xc9, 0xcd, 0xb1, 0xe3, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xe3, 0x52, 0x50, 0x50, 0x50, 0xb0, 0x29, 0xc9, 0x2c, 0xc9, 0x49, 0xb5, 0xf3, 0x54, 0x48, 0xcc, 0x55, 0xc8, 0xcc, 0x4b, 0x49, 0xad, 0xd0, 0x3, 0x29, 0xb1, 0xd1, 0x87, 0x8, 0x73, 0xd9, 0xe8, 0x43, 0x94, 0xda, 0x24, 0xe5, 0xa7, 0x54, 0x82, 0x34, 0x1a, 0xa2, 0x2b, 0x55, 0xc8, 0xcc, 0x2b, 0xce, 0x4c, 0x49, 0x55, 0x28, 0x2e, 0x4d, 0xb2, 0xd1, 0xcf, 0x30, 0x4, 0x69, 0x81, 0xaa, 0xd5, 0x7, 0x5b, 0x6, 0x8, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x4a, 0xca, 0x4f, 0xa9, 0x54, 0xa8, 0xe6, 0x52, 0x50, 0x50, 0x50, 0x48, 0x4a, 0x4c, 0xce, 0x4e, 0x2f, 0xca, 0x2f, 0xcd, 0x4b, 0xd1, 0x4d, 0xce, 0xcf, 0xc9, 0x2f, 0xb2, 0x52, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0xb3, 0xe6, 0xaa, 0x5, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x52, 0x2f, 0x2d, 0x4e, 0x55, 0x28, 0x2e, 0x29, 0xca, 0x4c, 0x2e, 0x51, 0xb7, 0xe6, 0xe2, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0x2f, 0xc9, 0xc8, 0x2c, 0x56, 0xc8, 0x2c, 0x56, 0x48, 0x2f, 0x4a, 0x4d, 0xcd, 0x53, 0x28, 0x48, 0x4c, 0x4f, 0x55, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x9, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x6c, 0xce, 0x31, 0xe, 0xc3, 0x20, 0xc, 0x40, 0xd1, 0x9d, 0x53, 0x58, 0x1c, 0x20, 0x28, 0xbb, 0xe3, 0xbd, 0xc7, 0xa0, 0xe0, 0xca, 0xb4, 0x40, 0x2a, 0xec, 0xa1, 0xb9, 0x7d, 0x95, 0x92, 0xb1, 0xab, 0xfd, 0xfd, 0x64, 0x14, 0x6b, 0x95, 0x1c, 0xa, 0xc7, 0x4c, 0xe, 0x0, 0x0, 0xad, 0x58, 0x65, 0xba, 0x41, 0x6c, 0xd0, 0x77, 0x83, 0xd2, 0x33, 0x7f, 0x96, 0x33, 0xc3, 0x30, 0x57, 0x33, 0xab, 0xa5, 0xbf, 0x60, 0x70, 0xdd, 0xbc, 0xda, 0x51, 0x59, 0x85, 0xd9, 0x3c, 0xc8, 0xe0, 0xc7, 0xe6, 0x5b, 0x2c, 0x7d, 0x49, 0xaa, 0x9e, 0x1c, 0x86, 0x49, 0xe3, 0x7d, 0xcf, 0xc7, 0x75, 0x2a, 0xeb, 0x7f, 0x5e, 0x56, 0x72, 0xa8, 0x69, 0x94, 0xb7, 0x81, 0x8e, 0x74, 0x39, 0x4f, 0xf5, 0x84, 0x61, 0x8e, 0x4f, 0x6f, 0x42, 0x18, 0x7e, 0x9f, 0x7f, 0x3, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x75, 0x72, 0x72, 0x7f, 0x6d, 0x0, 0x0, 0x0, 0x93, 0x0, 0x0, 0x0, 0xd, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x2e, 0x6e, 0x6f, 0x6f, 0x64, 0x6c, 0x65, 0x69, 0x67, 0x6e, 0x6f, 0x72, 0x65, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x3e, 0xe1, 0xdb, 0x8, 0x51, 0x0, 0x0, 0x0, 0x69, 0x0, 0x0, 0x0, 0xa, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0xb1, 0x0, 0x0, 0x0, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0xfe, 0x33, 0x1f, 0xd9, 0x7c, 0x0, 0x0, 0x0, 0xb4, 0x0, 0x0, 0x0, 0xe, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0x43, 0x1, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x48, 0x46, 0x3, 0xbf, 0x2c, 0x0, 0x0, 0x0, 0x26, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0x4, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x39, 0xd, 0x75, 0x6d, 0x39, 0x0, 0x0, 0x0, 0x33, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0x73, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0xdc, 0xc9, 0x85, 0x5d, 0x55, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x12, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0xee, 0x2, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x73, 0x75, 0x62, 0x2f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x64, 0x6c, 0x1a, 0xf4, 0x2b, 0x0, 0x0, 0x0, 0x25, 0x0, 0x0, 0x0, 0xc, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0x8c, 0x3, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x63, 0x73, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x2c, 0x13, 0x1c, 0x99, 0x38, 0x0, 0x0, 0x0, 0x32, 0x0, 0x0, 0x0, 0xb, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0xfa, 0x3, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x8c, 0x98, 0x6f, 0x99, 0x7e, 0x0, 0x0, 0x0, 0xc0, 0x0, 0x0, 0x0, 0x10, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0x74, 0x4, 0x0, 0x0, 0x69, 0x64, 0x78, 0x2f, 0x6e, 0x6f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x2e, 0x68, 0x74, 0x6d, 0x6c, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x9, 0x0, 0x9, 0x0, 0x64, 0x2, 0x0, 0x0, 0x39, 0x5, 0x0, 0x0, 0x0, 0x0} bowl := noodle.EmbedBowel{ Dirs: dirs, Data: data, @@ -153,21 +153,21 @@ func GetNoodleThirdParty() (noodle.EmbedBowel, error) { dirs := map[string]noodle.EmbedDir{"": { FileInfo: noodle.FileInfo{ FileName: "third_party", - FileSize: 4096, - FileMode: 020000000775, - FileModTime: time.Unix(1575356526, 0), + FileSize: 96, + FileMode: 020000000755, + FileModTime: time.Unix(1585298396, 0), FileIsDir: true, }, Entries: []noodle.FileInfo{{ FileName: "huge.js", FileSize: 47, - FileMode: 0664, - FileModTime: time.Unix(1575356526, 0), + FileMode: 0644, + FileModTime: time.Unix(1585298396, 0), FileIsDir: false, }}, }} - data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0xf7, 0x54, 0x48, 0xcc, 0x55, 0x48, 0x54, 0xc8, 0x28, 0x4d, 0x4f, 0x55, 0x28, 0xc9, 0xc8, 0x2c, 0x4a, 0x51, 0x28, 0x48, 0x2c, 0x2a, 0xa9, 0x54, 0xc8, 0xc9, 0x4c, 0x2a, 0x4a, 0x2c, 0xaa, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0x43, 0x38, 0x83, 0x4f, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0x6e, 0x8, 0xe6, 0x5d, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x3e, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x0, 0x0} + data := []byte{0x50, 0x4b, 0x3, 0x4, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x4a, 0xce, 0xcf, 0x2b, 0xce, 0xcf, 0x49, 0xd5, 0xcb, 0xc9, 0x4f, 0xd7, 0x50, 0xf7, 0x54, 0x48, 0xcc, 0x55, 0x48, 0x54, 0xc8, 0x28, 0x4d, 0x4f, 0x55, 0x28, 0xc9, 0xc8, 0x2c, 0x4a, 0x51, 0x28, 0x48, 0x2c, 0x2a, 0xa9, 0x54, 0xc8, 0xc9, 0x4c, 0x2a, 0x4a, 0x2c, 0xaa, 0x54, 0xd7, 0xb4, 0x6, 0x4, 0x0, 0x0, 0xff, 0xff, 0x50, 0x4b, 0x7, 0x8, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x50, 0x4b, 0x1, 0x2, 0x14, 0x3, 0x14, 0x0, 0x8, 0x0, 0x8, 0x0, 0xfc, 0x44, 0x7b, 0x50, 0x71, 0x67, 0x15, 0xf0, 0x35, 0x0, 0x0, 0x0, 0x2f, 0x0, 0x0, 0x0, 0x7, 0x0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xa4, 0x81, 0x0, 0x0, 0x0, 0x0, 0x68, 0x75, 0x67, 0x65, 0x2e, 0x6a, 0x73, 0x55, 0x54, 0x5, 0x0, 0x1, 0xdc, 0xbb, 0x7d, 0x5e, 0x50, 0x4b, 0x5, 0x6, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x1, 0x0, 0x3e, 0x0, 0x0, 0x0, 0x73, 0x0, 0x0, 0x0, 0x0, 0x0} bowl := noodle.EmbedBowel{ Dirs: dirs, Data: data, diff --git a/playground/README.md b/playground/README.md index c597ccb..a7d4dfe 100644 --- a/playground/README.md +++ b/playground/README.md @@ -1,4 +1,10 @@ # Playground For testing language semantics and small benchmarks. Prototype and library examples are also put here. -It also keeps some minimal code for reproduce/solve issues. \ No newline at end of file +It also keeps some minimal code for reproduce/solve issues. + +- [Standard Library](stdlib) + +Issues + +- [#50](issue_noodle_50) should not append pointer of for range temp variable to slice \ No newline at end of file diff --git a/playground/linter/import_test.go b/playground/linter/import_test.go new file mode 100644 index 0000000..7f9b12a --- /dev/null +++ b/playground/linter/import_test.go @@ -0,0 +1,38 @@ +package linter + +import ( + "github.com/dyweb/gommon/linter" + "github.com/dyweb/gommon/util/fsutil" + "github.com/stretchr/testify/require" + "golang.org/x/tools/imports" + "testing" +) + +func TestImport(t *testing.T) { + // for jump into x/tools/imports and x/tools/internal/imports + opt := imports.Options{ + TabWidth: 8, + TabIndent: true, + Comments: true, + AllErrors: true, + FormatOnly: true, + } + // This allow extra grouping + imports.LocalPrefix = "golang.org" + defer func() { + imports.LocalPrefix = "" + }() + b, err := imports.Process("unordered_import.go", nil, &opt) + require.Nil(t, err) + fsutil.WriteFile("unordered_import_goimport.txt", b) +} + +func TestGommonImport(t *testing.T) { + res, err := linter.CheckAndFormatImport("unordered_import.go", linter.GoimportFlags{ + List: true, + Diff: true, + FormatOnly: true, + }) + require.Nil(t, err) + fsutil.WriteFile("unordered_import_gommon.txt", res.Formatted) +} diff --git a/playground/linter/unordered_import.go b/playground/linter/unordered_import.go new file mode 100644 index 0000000..0e06baf --- /dev/null +++ b/playground/linter/unordered_import.go @@ -0,0 +1,21 @@ +package linter + +import ( + "github.com/dyweb/gommon/errors" + "github.com/dyweb/gommon/util/fsutil" + "golang.org/x/tools/imports" + "strings" + "unicode" + + "bytes" +) + +func foo() error { + var _ = unicode.ToUpper('a') + var msg = strings.Join([]string{"foo", "error"}, " ") + _, err := imports.Process("", nil, nil) + _, err = fsutil.CreateFileAndPath("foo", "boar.txt") + var buf = bytes.Buffer{} + buf.Write([]byte(msg)) + return errors.Wrap(err, msg) +} diff --git a/playground/linter/unordered_import_goimport.txt b/playground/linter/unordered_import_goimport.txt new file mode 100644 index 0000000..997194c --- /dev/null +++ b/playground/linter/unordered_import_goimport.txt @@ -0,0 +1,23 @@ +package linter + +import ( + "strings" + "unicode" + + "github.com/dyweb/gommon/errors" + "github.com/dyweb/gommon/util/fsutil" + + "golang.org/x/tools/imports" + + "bytes" +) + +func foo() error { + var _ = unicode.ToUpper('a') + var msg = strings.Join([]string{"foo", "error"}, " ") + _, err := imports.Process("", nil, nil) + _, err = fsutil.CreateFileAndPath("foo", "boar.txt") + var buf = bytes.Buffer{} + buf.Write([]byte(msg)) + return errors.Wrap(err, msg) +} diff --git a/playground/linter/unordered_import_gommon.txt b/playground/linter/unordered_import_gommon.txt new file mode 100644 index 0000000..ec43df7 --- /dev/null +++ b/playground/linter/unordered_import_gommon.txt @@ -0,0 +1,18 @@ +package linter + +import ( + "strings" + "unicode" + + "github.com/dyweb/gommon/errors" + "github.com/dyweb/gommon/util/fsutil" + "golang.org/x/tools/imports" +) + +func foo() error { + var _ = unicode.ToUpper('a') + var msg = strings.Join([]string{"foo", "error"}, " ") + _, err := imports.Process("", nil, nil) + _, err = fsutil.CreateFileAndPath("foo", "boar.txt") + return errors.Wrap(err, msg) +} diff --git a/playground/pkg.go b/playground/pkg.go deleted file mode 100644 index 19c4c27..0000000 --- a/playground/pkg.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package playground is used for testing out issues and language features -package playground diff --git a/tconfig/README.md b/tconfig/README.md new file mode 100644 index 0000000..6f3c71c --- /dev/null +++ b/tconfig/README.md @@ -0,0 +1,6 @@ +# Gommon tconfig + +## TODO + +- [ ] usage example +- [ ] link to bkg and motivation in log \ No newline at end of file diff --git a/tconfig/env.go b/tconfig/env.go new file mode 100644 index 0000000..3664a9a --- /dev/null +++ b/tconfig/env.go @@ -0,0 +1,133 @@ +package tconfig + +import ( + "fmt" + "os" + "reflect" + "strconv" + "strings" + "unicode" +) + +// env.go defines environment variable related operations + +// EnvReader reads environment variable by key and returns empty string if it does not exists +type EnvReader func(key string) (value string) + +// EvalBool converts a string value to boolean value. +type EvalBool func(s string) bool + +type EnvToStructConfig struct { + Prefix string + Env EnvReader // Env determines how to get environment variable, os.Getenv is used when set to nil + Bool EvalBool // Bool determines how to evaluate a string to bool, DefaultEvalBool is used when set to nil +} + +// a private default so other packages can't change it +var defaultEnvToStructConfig = DefaultEnvToStructConfig() + +func DefaultEnvToStructConfig() EnvToStructConfig { + return EnvToStructConfig{ + Env: os.Getenv, + Bool: DefaultEvalBool(), + } +} + +// a private default so other packages can't change it +var defaultEvalBool = DefaultEvalBool() + +// DefaultEvalBool treats empty string, 0, false, FALSE as false. +func DefaultEvalBool() EvalBool { + return func(s string) bool { + if s == "" || s == "0" || s == "false" || s == "FALSE" { + return false + } + return true + } +} + +// TODO: it should works both normal struct and traceable struct config +func (c *EnvToStructConfig) To(v interface{}) error { + envReader := c.Env + if envReader == nil { + envReader = os.Getenv + } + evalBool := c.Bool + if evalBool == nil { + evalBool = defaultEvalBool + } + + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.IsNil() { + return fmt.Errorf("v must be a non nil pointer") + } + rv = rv.Elem() + if rv.Kind() != reflect.Struct { + return fmt.Errorf("can only decode to a struct got %s", rv.Kind()) + } + rt := rv.Type() + nFields := rv.NumField() + for i := 0; i < nFields; i++ { + fv := rv.Field(i) + ft := rt.Field(i) + name := ft.Name + key := c.Prefix + fieldNameToEnvKey(name) + val := envReader(key) + switch fv.Kind() { + case reflect.Int: + if val != "" { + i, err := strconv.Atoi(val) + if err != nil { + return fmt.Errorf("invalid int key %s val %s for field %s: %w", key, val, name, err) + } + fv.SetInt(int64(i)) + } + case reflect.Bool: + fv.SetBool(evalBool(val)) + case reflect.String: + if val != "" { + fv.SetString(val) + } + default: + return fmt.Errorf("only int, bool and string fields are supported got %s", fv.Kind()) + } + } + return nil +} + +// EnvToStruct decodes environment variable to struct using DefaultEnvToStructConfig +func EnvToStruct(v interface{}) error { + return defaultEnvToStructConfig.To(v) +} + +func fieldNameToEnvKey(name string) string { + var b strings.Builder + for i, r := range name { + if unicode.IsUpper(r) && i != 0 { + b.WriteRune('_') + } + b.WriteRune(unicode.ToUpper(r)) + } + return b.String() +} + +// ---------------------------------------------------------------------------- +// Env Util, they are not that related to tconfig but save some typing in adhoc code + +// EnvInt returns decoded int or 0 if the key does not exists or contains invalid value. +func EnvInt(key string) int { + return EnvIntDefault(key, 0) +} + +// EnvIntDefault returns decoded int or defaultValue if the key does not exists or contains invalid value. +func EnvIntDefault(key string, defaultValue int) int { + v := os.Getenv(key) + if v == "" { + return defaultValue + } + i, err := strconv.Atoi(v) + if err != nil { + return defaultValue + } + return i +} diff --git a/tconfig/pkg.go b/tconfig/pkg.go new file mode 100644 index 0000000..eb73b00 --- /dev/null +++ b/tconfig/pkg.go @@ -0,0 +1,19 @@ +// Package tconfig is a traceable config package. It allows you to keep track of the source and mutation +// of single or a set of config values from different sources, e.g. environment variable, config file, cli flags. +package tconfig + +type Var interface { + Eval() interface{} +} + +type BoolVar interface { + EvalBool() bool +} + +type IntVar interface { + EvalInt() int +} + +type StringVar interface { + EvalString() string +} diff --git a/util/fsutil/file.go b/util/fsutil/file.go index 0257831..b8148a5 100644 --- a/util/fsutil/file.go +++ b/util/fsutil/file.go @@ -3,6 +3,9 @@ package fsutil import ( "io/ioutil" "os" + "os/exec" + "path/filepath" + "strings" "github.com/dyweb/gommon/errors" ) @@ -12,6 +15,12 @@ const ( DefaultDirPerm = 0775 ) +func IsGoFile(info os.FileInfo) bool { + name := info.Name() + // not a folder & not hidden & .go + return !info.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".go") +} + // WriteFile use 0664 as permission and wrap standard error func WriteFile(path string, data []byte) error { if err := ioutil.WriteFile(path, data, DefaultFilePerm); err != nil { @@ -32,3 +41,68 @@ func MkdirIfNotExists(path string) error { } return os.MkdirAll(path, DefaultDirPerm) } + +// CreateFileAndPath creates the folder if it does not exists and create a new file using os.Create. +func CreateFileAndPath(path, file string) (*os.File, error) { + if err := MkdirIfNotExists(path); err != nil { + return nil, err + } + return os.Create(filepath.Join(path, file)) +} + +// WriteTempFile creates a temporary file and writes data to it. +// Dir can be empty string. It returns path of the created file. +// NOTE: It is based on goimports command. +func WriteTempFile(dir, prefix string, data []byte) (string, error) { + f, err := ioutil.TempFile(dir, prefix) + if err != nil { + return "", err + } + _, err = f.Write(data) + if errClose := f.Close(); err == nil { + err = errClose + } + if err != nil { + os.Remove(f.Name()) + return "", err + } + return f.Name(), nil +} + +// WriteTempFiles creates multiple files under same dir w/ same prefix. +// It stops if there are any error and always returns created files. +func WriteTempFiles(dir, prefix string, dataList ...[]byte) ([]string, error) { + var names []string + for _, data := range dataList { + n, err := WriteTempFile(dir, prefix, data) + if err != nil { + return names, err + } + names = append(names, n) + } + return names, nil +} + +// RemoveFiles removes multiple files. It keeps track of error for each removal. +// But it does NOT stop when there is error. +// Returned non nil error must be a dyweb/gommon/errors.MultiErr. +func RemoveFiles(names []string) error { + merr := errors.NewMultiErr() + for _, name := range names { + merr.Append(os.Remove(name)) + } + return merr.ErrorOrNil() +} + +// Diff compares two files by shelling out to system diff binary. +// NOTE: It is based on goimports command. +// TODO: there are pure go diff package, and there is also diff w/ syntax highlight written in rust +// TODO: allow force color output, default is auto and I guess it detects tty +func Diff(p1 string, p2 string) ([]byte, error) { + b, err := exec.Command("diff", "-u", p1, p2).CombinedOutput() + // NOTE: diff returns 1 when there are diff, so we ignore the error as long as there is valid output. + if len(b) != 0 { + return b, nil + } + return b, errors.Wrapf(err, "error shell out to diff -u %s %s", p1, p2) +} diff --git a/util/fsutil/ignore.go b/util/fsutil/ignore.go index d84e00b..c34bfa4 100644 --- a/util/fsutil/ignore.go +++ b/util/fsutil/ignore.go @@ -10,6 +10,8 @@ import ( "github.com/dyweb/gommon/errors" ) +var AcceptAll = NewIgnores(nil, nil) + // ReadIgnoreFile reads a .ignore file and parse the patterns. func ReadIgnoreFile(path string) (*Ignores, error) { f, err := os.Open(path) diff --git a/util/fsutil/pkg.go b/util/fsutil/pkg.go index 60cc250..067d4c6 100644 --- a/util/fsutil/pkg.go +++ b/util/fsutil/pkg.go @@ -1,9 +1,11 @@ // Package fsutil adds ignore support for walk -package fsutil // import "github.com/dyweb/gommon/util/fsutil" +package fsutil import ( dlog "github.com/dyweb/gommon/log" ) -var logReg = dlog.NewRegistry() -var log = logReg.Logger() +var ( + logReg = dlog.NewRegistry() + log = logReg.Logger() +) diff --git a/util/fsutil/walk.go b/util/fsutil/walk.go index a7a68c6..63f00de 100644 --- a/util/fsutil/walk.go +++ b/util/fsutil/walk.go @@ -13,6 +13,9 @@ type WalkFunc func(path string, info os.FileInfo) // Walk traverse the directory with ignore patterns in Pre-Order DFS func Walk(root string, ignores *Ignores, walkFunc WalkFunc) error { // TODO: validate ignores or assign a default accept all + if ignores == nil { + ignores = AcceptAll + } files, err := ioutil.ReadDir(root) if err != nil { return errors.Wrapf(err, "can't read dir %s", root) diff --git a/util/genutil/go.go b/util/genutil/go.go new file mode 100644 index 0000000..06cca0e --- /dev/null +++ b/util/genutil/go.go @@ -0,0 +1,12 @@ +package genutil + +import "golang.org/x/tools/imports" + +// go.go defines helper when generating go code + +// Format calls imports.Process so it behaves like goimports i.e. sort and merge imports. +// Deprecated: call generator.FormatGo +func Format(src []byte) ([]byte, error) { + // TODO: might need to disable finding import and chose format only + return imports.Process("", src, nil) +} diff --git a/util/genutil/pkg.go b/util/genutil/pkg.go index 2de13f7..f3c9660 100644 --- a/util/genutil/pkg.go +++ b/util/genutil/pkg.go @@ -1,15 +1,16 @@ // Package genutil contains helper when generating files, // it is used to break dependency cycle between generator package // and packages that contain generator logic like log, noodle etc. +// TODO: move it to generator and use registry to avoid dependency cycle package genutil -// DefaultHeader calls Header and set generator to gommon +// DefaultHeader calls Header and set generator to gommon. func DefaultHeader(templateSrc string) string { return "// Code generated by gommon from " + templateSrc + " DO NOT EDIT.\n\n" } // Header returns the standard go header for generated files with two trailing \n, -// the second \n is to avoid this header becomes documentation of the package +// the second \n is to avoid this header becomes documentation of the package. func Header(generator string, templateSrc string) string { return "// Code generated by " + generator + " from " + templateSrc + " DO NOT EDIT.\n\n" } diff --git a/util/genutil/template_funcs.go b/util/genutil/template_funcs.go new file mode 100644 index 0000000..72dd2d5 --- /dev/null +++ b/util/genutil/template_funcs.go @@ -0,0 +1,74 @@ +package genutil + +import ( + htmltemplate "html/template" + texttemplate "text/template" + "unicode" + + "github.com/dyweb/gommon/util/stringutil" +) + +// template_funcs.go defines common template functions used in go template + +// TemplateFuncMap returns a new func map that includes all template func in this page. +func TemplateFuncMap() map[string]interface{} { + return map[string]interface{}{ + "UcFirst": stringutil.UcFirst, + "LcFirst": stringutil.LcFirst, + } +} + +// TextTemplateFuncMap returns func map for text/template +func TextTemplateFuncMap() texttemplate.FuncMap { + return TemplateFuncMap() +} + +// HTMLTemplateFuncMap returns func map for html/template +// TODO: maybe we should have some extra html specific helpers +func HTMLTemplateFuncMap() htmltemplate.FuncMap { + return TemplateFuncMap() +} + +// UcFirst changes first character to upper case. +// Deprecated: use stringuitl.UcFirst +func UcFirst(s string) string { + if s == "" { + return "" + } + r := []rune(s) + r[0] = unicode.ToUpper(r[0]) + return string(r) +} + +// SnakeToCamel converts snake_case to CamelCase. +// Deprecated: use stringutil.SnakeToCamel +func SnakeToCamel(s string) string { + src := []rune(s) + var dst []rune + toUpper := true + for _, r := range src { + if r == '_' { + toUpper = true + continue + } + + r2 := r + if toUpper { + r2 = unicode.ToUpper(r) + toUpper = false + } + dst = append(dst, r2) + } + return string(dst) +} + +// LcFirst changes first character to lower case. +// Deprecated: use stringutil.SnakeToCamel +func LcFirst(s string) string { + if s == "" { + return "" + } + r := []rune(s) + r[0] = unicode.ToLower(r[0]) + return string(r) +} diff --git a/util/hashutil/fnv64.go b/util/hashutil/fnv.go similarity index 74% rename from util/hashutil/fnv64.go rename to util/hashutil/fnv.go index b45ee53..2b5c819 100644 --- a/util/hashutil/fnv64.go +++ b/util/hashutil/fnv.go @@ -1,19 +1,27 @@ package hashutil const ( - prime64 = 1099511628211 + offset32 = 2166136261 offset64 = 14695981039346656037 + prime32 = 16777619 + prime64 = 1099511628211 ) // InlineFNV64a is a alloc-free version of https://golang.org/pkg/hash/fnv/ // copied from Xephon-K, which is copied from influxdb/models https://github.com/influxdata/influxdb/blob/master/models/inline_fnv.go type InlineFNV64a uint64 +type InlineFNV32a uint32 + // NewInlineFNV64a returns a new instance of InlineFNV64a. func NewInlineFNV64a() InlineFNV64a { return offset64 } +func NewInlineFNV32a() InlineFNV32a { + return offset32 +} + // Write adds data to the running hash. func (s *InlineFNV64a) Write(data []byte) (int, error) { hash := uint64(*s) @@ -25,6 +33,16 @@ func (s *InlineFNV64a) Write(data []byte) (int, error) { return len(data), nil } +func (s *InlineFNV32a) Write(data []byte) (int, error) { + hash := uint32(*s) + for _, c := range data { + hash ^= uint32(c) + hash *= prime32 + } + *s = InlineFNV32a(hash) + return len(data), nil +} + // WriteString avoids a []byte(str) conversion BUT yield different result when string contains non ASCII characters func (s *InlineFNV64a) WriteString(str string) (int, error) { hash := uint64(*s) @@ -40,3 +58,7 @@ func (s *InlineFNV64a) WriteString(str string) (int, error) { func (s *InlineFNV64a) Sum64() uint64 { return uint64(*s) } + +func (s *InlineFNV32a) Sum32() uint32 { + return uint32(*s) +} diff --git a/util/hashutil/fnv64_test.go b/util/hashutil/fnv_test.go similarity index 73% rename from util/hashutil/fnv64_test.go rename to util/hashutil/fnv_test.go index a417b08..b88b80b 100644 --- a/util/hashutil/fnv64_test.go +++ b/util/hashutil/fnv_test.go @@ -2,6 +2,7 @@ package hashutil import ( "fmt" + "hash/fnv" "testing" asst "github.com/stretchr/testify/assert" @@ -20,6 +21,24 @@ func TestNewInlineFNV64a(t *testing.T) { assert.Equal(r1, r2) } +func TestInlineFNV32a_Write(t *testing.T) { + data := []string{ + "8KBF520", + "BJUB", + "AMD YES!", + } + for _, d := range data { + b := []byte(d) + stdfnv := fnv.New32a() + myfnv := NewInlineFNV32a() + stdfnv.Write(b) + myfnv.Write(b) + if stdfnv.Sum32() != myfnv.Sum32() { + t.Errorf("Mismatch for %s std %d my %d", d, stdfnv.Sum32(), myfnv.Sum32()) + } + } +} + // for ascii, Write and WriteString has same result, for non-ascii, NO func TestInlineFNV64a_Write(t *testing.T) { assert := asst.New(t) @@ -52,4 +71,4 @@ func TestInlineFNV64a_WriteString(t *testing.T) { } } -// TODO: benchmark byte alloc when using Write([]byte(str)) and WriteString() +// TODO: benchmark InlineFNV64a with the hash/fnv diff --git a/util/hashutil/pkg.go b/util/hashutil/pkg.go index 0603a6f..c14db38 100644 --- a/util/hashutil/pkg.go +++ b/util/hashutil/pkg.go @@ -1,4 +1,8 @@ -// Package hashutil provides alloc free alternatives for pkg/hash +// Package hashutil provides alloc free alternatives for pkg/hash. +// Currently only fnv64a and fnv32a are supported. +// TODO: +// https://segment.com/blog/allocation-efficiency-in-high-performance-go-services/ +// https://github.com/segmentio/fasthash package hashutil func HashStringFnv64a(str string) uint64 { @@ -17,6 +21,10 @@ func HashFnv64a(b []byte) uint64 { return h.Sum64() } -// TODO: -// https://segment.com/blog/allocation-efficiency-in-high-performance-go-services/ -// https://github.com/segmentio/fasthash +func HashFnv32a(b []byte) uint32 { + h := NewInlineFNV32a() + if _, err := h.Write(b); err != nil { + panic(err) + } + return h.Sum32() +} diff --git a/util/httputil/http.go b/util/httputil/http.go index ee7f126..a968b19 100644 --- a/util/httputil/http.go +++ b/util/httputil/http.go @@ -5,6 +5,7 @@ import ( "io/ioutil" "net" "net/http" + "runtime" "time" ) @@ -20,20 +21,20 @@ func NewUnPooledTransport() *http.Transport { return tr } -// NewPooledTransport is same as DefaultTransport in net/http, but it is not shared and won't be alerted by other library +// NewPooledTransport is same as DefaultTransport in net/http. +// But it is not shared and won't be alerted by other library func NewPooledTransport() *http.Transport { return &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ Timeout: 30 * time.Second, KeepAlive: 30 * time.Second, - DualStack: true, }).DialContext, MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, - // TODO: set MaxIdleConnsPerHost like https://github.com/hashicorp/go-cleanhttp/blob/master/cleanhttp.go#L35 runtime.GOMAXPROCS(0) + 1 ? + MaxConnsPerHost: runtime.GOMAXPROCS(0) + 1, // https://github.com/hashicorp/go-cleanhttp/blob/master/cleanhttp.go#L35 } } @@ -45,7 +46,7 @@ func NewClient(tr *http.Transport) *http.Client { panic("transport is nil") } if tr == http.DefaultTransport { - panic("stop using default transport") + panic("stop using http.DefaultTransport") } return &http.Client{ Transport: tr, diff --git a/util/httputil/pkg.go b/util/httputil/pkg.go index 322e002..a75dfa2 100644 --- a/util/httputil/pkg.go +++ b/util/httputil/pkg.go @@ -21,6 +21,21 @@ const ( Trace Method = http.MethodTrace ) +// AllMethods returns all the http methods (defined in this package) +func AllMethods() []Method { + return []Method{ + Get, + Head, + Post, + Put, + Patch, + Delete, + Connect, + Options, + Trace, + } +} + // UserAgent data are from https://techblog.willshouse.com/2012/01/03/most-common-user-agents/ // For UserAgent spec, see MDN https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/User-Agent type UserAgent string @@ -33,5 +48,5 @@ const ( UAChromeWin UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36" UAChromeLinux UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/67.0.3396.62 Safari/537.36" UAChromeMac UserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.181 Safari/537.36" - // TODO: add UA for mobile device (so you can see mobile optimized page in terminal? .... + // TODO: add UA for mobile device ) diff --git a/util/httputil/writer.go b/util/httputil/writer.go index f27527a..20e2b83 100644 --- a/util/httputil/writer.go +++ b/util/httputil/writer.go @@ -32,7 +32,7 @@ type TrackedWriter struct { // NewTrackedWriter set the underlying writer based on argument, // It returns a value instead of pointer so it can be allocated on stack. -// TODO: add benchmark to prove it ... +// TODO: add benchmark to prove it ... might change it to pointer ... func NewTrackedWriter(w http.ResponseWriter) TrackedWriter { return TrackedWriter{w: w, status: 200} } diff --git a/util/maputil/interface.go b/util/maputil/interface.go new file mode 100644 index 0000000..f25b767 --- /dev/null +++ b/util/maputil/interface.go @@ -0,0 +1,16 @@ +package maputil + +// interface.go provides helper related to map[string]interface{} + +// MergeStringInterface creates a new map[string]interface{} that contains values from two maps. +// If there are duplicated keys, values from second map is preserved. +func MergeStringInterface(a, b map[string]interface{}) map[string]interface{} { + c := make(map[string]interface{}) + for k, v := range a { + c[k] = v + } + for k, v := range b { + c[k] = v + } + return c +} diff --git a/util/stringutil/algo.go b/util/stringutil/algo.go new file mode 100644 index 0000000..e7ba9e9 --- /dev/null +++ b/util/stringutil/algo.go @@ -0,0 +1,35 @@ +package stringutil + +// algo.go implements common string algorithms like EditDistance. + +// CopySlice makes a deep copy of string slice. +// NOTE: it assumes the underlying string is immutable, i.e. the are not created using unsafe. +func CopySlice(src []string) []string { + cp := make([]string, len(src)) + for i, s := range src { + cp[i] = s + } + return cp +} + +func EditDistance(word string, candidates []string, maxEdit int) { + // TODO: return results group by distances [][]string should work when maxEdit is small + // TODO: sort the response so it is stable? + panic("not implemented") +} + +// Shorter returns the shorter string or a if the length equals +func Shorter(a, b string) string { + if len(a) <= len(b) { + return a + } + return b +} + +// Longer returns the longer string or b if the string equals +func Longer(a, b string) string { + if len(a) >= len(b) { + return a + } + return b +} diff --git a/util/stringutil/collection.go b/util/stringutil/collection.go new file mode 100644 index 0000000..6f39dd3 --- /dev/null +++ b/util/stringutil/collection.go @@ -0,0 +1,48 @@ +package stringutil + +import "sort" + +// collection.go contains set etc. + +type Set struct { + m map[string]struct{} + insertion []string +} + +func NewSet() *Set { + return &Set{ + m: make(map[string]struct{}), + } +} + +func (s *Set) Add(str string) { + _, ok := s.m[str] + if ok { + return + } + s.m[str] = struct{}{} + s.insertion = append(s.insertion, str) +} + +func (s *Set) AddMulti(ss ...string) { + for _, str := range ss { + s.Add(str) + } +} + +// Inserted returns a copy of unique strings in their insertion order. +func (s *Set) Inserted() []string { + return CopySlice(s.insertion) +} + +// Sorted returns sorted unique strings in ascending order. e.g. [a, b, c] +func (s *Set) Sorted() []string { + cp := CopySlice(s.insertion) + sort.Strings(cp) + return cp +} + +// TODO: impl +//func (s *Set) SortedDesc() []string { +// +//} diff --git a/util/stringutil/convert.go b/util/stringutil/convert.go new file mode 100644 index 0000000..072de39 --- /dev/null +++ b/util/stringutil/convert.go @@ -0,0 +1,58 @@ +package stringutil + +import "unicode" + +// convert.go converts string and strings. + +// UcFirst changes first character to upper case. +// It is based on https://github.com/99designs/gqlgen/blob/master/codegen/templates/templates.go#L205 +func UcFirst(s string) string { + if s == "" { + return "" + } + r := []rune(s) + r[0] = unicode.ToUpper(r[0]) + return string(r) +} + +// LcFirst changes first character to lower case. +func LcFirst(s string) string { + if s == "" { + return "" + } + r := []rune(s) + r[0] = unicode.ToLower(r[0]) + return string(r) +} + +// SnakeToCamel converts snake_case to CamelCase. +func SnakeToCamel(s string) string { + src := []rune(s) + var dst []rune + toUpper := true + for _, r := range src { + if r == '_' { + toUpper = true + continue + } + + r2 := r + if toUpper { + r2 = unicode.ToUpper(r) + toUpper = false + } + dst = append(dst, r2) + } + return string(dst) +} + +// RemoveEmpty removes empty string within the slice +func RemoveEmpty(src []string) []string { + var d []string + for _, s := range src { + if s != "" { + d = append(d, s) + } + } + return d +} diff --git a/util/stringutil/convert_test.go b/util/stringutil/convert_test.go new file mode 100644 index 0000000..24d20b4 --- /dev/null +++ b/util/stringutil/convert_test.go @@ -0,0 +1,25 @@ +package stringutil_test + +import ( + "testing" + + "github.com/dyweb/gommon/util/stringutil" + "github.com/stretchr/testify/assert" +) + +// TODO: fuzz test +func TestSnakeToCamel(t *testing.T) { + cases := []struct { + s string + c string + }{ + {"snake", "Snake"}, + {"snake_", "Snake"}, + {"snake_case", "SnakeCase"}, + {"snake_case_case", "SnakeCaseCase"}, + {"snake__case", "SnakeCase"}, + } + for _, tc := range cases { + assert.Equal(t, tc.c, stringutil.SnakeToCamel(tc.s)) + } +} diff --git a/util/stringutil/pkg.go b/util/stringutil/pkg.go new file mode 100644 index 0000000..13e5066 --- /dev/null +++ b/util/stringutil/pkg.go @@ -0,0 +1,2 @@ +// Package stringutil provides string conversion (e.g. UcFirst) and algorithms (e.g. EditDistance). +package stringutil diff --git a/util/testutil/condition.go b/util/testutil/condition.go index 1a50d23..2f38ee2 100644 --- a/util/testutil/condition.go +++ b/util/testutil/condition.go @@ -190,6 +190,22 @@ func BinaryExist(name string) Condition { }} } +// CommandSuccess returns a test condition that executes a command and wait its completion. +// It evaluates to true if the process exist with 0. +// TODO: maybe have a default timeout? +func CommandSuccess(cmd string, args ...string) Condition { + return &con{stmt: func() (res bool, msg string, err error) { + cmd := exec.Command(cmd, args...) + full := fmt.Sprintf("%s %s", cmd, strings.Join(args, " ")) + b, err := cmd.CombinedOutput() + if err == nil { + return true, "command success: " + full, nil + } + // FIXME: we return nil error to avoid failing the test in RunIf, might need to change the design + return false, "command failed: " + full + string(b), nil + }} +} + // wrapper for common conditions, NOTE: some are defined in other files like golden.go // IsTravis check if env TRAVIS is true @@ -197,10 +213,10 @@ func IsTravis() Condition { return EnvTrue("TRAVIS") } -// HasDocker returns a test condition that checks if docker client exists. -// NOTE: it does not check if the docker daemon is running or the current user has the privilege to execute docker cli. +// HasDocker returns a test condition that checks if docker client exists and the daemon is up and running. +// TODO: cache the result of docker version? func HasDocker() Condition { - return BinaryExist("docker") + return And(BinaryExist("docker"), CommandSuccess("docker", "version")) } // Dump check if env DUMP or GOMMON_DUMP is set, so print detail or use go-spew to dump structs etc. diff --git a/util/testutil/docker.go b/util/testutil/docker.go index 98636aa..83448e3 100644 --- a/util/testutil/docker.go +++ b/util/testutil/docker.go @@ -138,3 +138,14 @@ func (c *Container) Stop() error { } return nil } + +// IsDockerRunning returns true if docker version exit with 0. +// Meaning there is docker cli and the server it points to is up and running +func IsDockerRunning() bool { + cmd := exec.Command("docker", "version") + _, err := cmd.CombinedOutput() + if err != nil { + return false + } + return true +} diff --git a/util/testutil/docker_test.go b/util/testutil/docker_test.go index d02c72f..1170ab0 100644 --- a/util/testutil/docker_test.go +++ b/util/testutil/docker_test.go @@ -32,6 +32,9 @@ func TestContainer_DockerRunArgs(t *testing.T) { } func TestContainer_Stop(t *testing.T) { + t.Skip("always skip docker test") + + // TODO: need more rules, the test always run for machine that has active docker running RunIf(t, HasDocker()) port, err := netutil.AvailablePortBySystem() diff --git a/util/testutil/golden.go b/util/testutil/golden.go index 9c65f67..7c947b7 100644 --- a/util/testutil/golden.go +++ b/util/testutil/golden.go @@ -14,12 +14,12 @@ var ( // GenGolden check if env GOLDEN or GEN_GOLDEN is set, sometimes you need to generate test fixture in test func GenGolden() Condition { - return Or(EnvHas("GOLDEN"), EnvHas("GEN_GOLDEN")) + return Or(EnvTrue("GOLDEN"), EnvTrue("GEN_GOLDEN")) } // GenGoldenT check if current test is manually set to generate golden file func GenGoldenT(t *testing.T) Condition { - return Or(Or(EnvHas("GOLDEN"), EnvHas("GEN_GOLDEN")), &con{ + return Or(GenGolden(), &con{ stmt: func() (res bool, msg string, err error) { enabledGoldenMu.RLock() enabled := enabledGolden[t] @@ -49,7 +49,7 @@ func WriteOrCompare(t *testing.T, file string, data []byte) { WriteFixture(t, file, data) } else { b := ReadFixture(t, file) - assert.Equal(t, b, data) + assert.Equal(t, b, data, file) } }