Complete code refactoring

- `LOGGING` environment variable `json` or `human`
- `NODEID` environment variable (integer)
- Much cleaner go code
- Listener for exit of program to do cleanup
- All code is in packages except main.go
- Custom logger package added
- Connectivity checks reworked
- Healthcheck server on localhost only, so not exposed to outside world
- Updated `go.mod` and `go.sum`

.dockerignore

@@ -1,7 +1,9 @@
-.gitignore
-.travis.yml
-.git/
+.git
 *.exe
-README.md
+.vscode
+.travis.yml
 docker-compose.yml
-readme/
+LICENSE
+*.md
+readme
+.gitignore

.gitignore

@@ -1,14 +1,2 @@
-# Binaries for programs and plugins
 *.exe
-*.exe~
-*.dll
-*.so
-*.dylib
-
-# Test binary, build with `go test -c`
-*.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
-*.out
-
 .vscode

Dockerfile

@@ -1,6 +1,17 @@
 ARG ALPINE_VERSION=3.9
 ARG GO_VERSION=1.12.4
 
+FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS builder
+RUN apk --update add git build-base upx
+RUN go get -u -v golang.org/x/vgo
+WORKDIR /tmp/gobuild
+COPY go.mod go.sum ./
+RUN go mod download
+COPY pkg/ ./pkg/
+COPY main.go .
+#RUN go test -v
+RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags="-s -w" -o app .
+
 FROM alpine:${ALPINE_VERSION} AS final
 ARG BUILD_DATE
 ARG VCS_REF
@@ -16,7 +27,7 @@ LABEL org.label-schema.schema-version="1.0.0-rc1" \
       org.label-schema.docker.cmd.devel="docker run -it --rm -p 8000:8000/tcp -e RECORD1=example.com,@,namecheap,provider,0e4512a9c45a4fe88313bcc2234bf547 qmcgaw/ddns-updater" \
       org.label-schema.docker.params="See readme" \
       org.label-schema.version="" \
-      image-size="19.3MB" \
+      image-size="21.4MB" \
       ram-usage="13MB" \
       cpu-usage="Very Low"
 RUN apk add --update sqlite ca-certificates && \
@@ -32,17 +43,11 @@ EXPOSE 8000
 HEALTHCHECK --interval=300s --timeout=5s --start-period=5s --retries=1 CMD ["/updater/app", "healthcheck"]
 USER 1000
 ENTRYPOINT ["/updater/app"]
-
-FROM golang:${GO_VERSION}-alpine${ALPINE_VERSION} AS builder
-RUN apk --update add git build-base upx
-RUN go get -u -v golang.org/x/vgo
-WORKDIR /tmp/gobuild
-COPY updater/go.mod updater/go.sum ./
-RUN go mod download
-COPY updater/*.go ./
-#RUN go test -v
-RUN CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags="-s -w" -o app .
-
-FROM final
+ENV DELAY= \
+    ROOTURL= \
+    LISTENINGPORT= \
+    RECORD0= \
+    LOGGING= \
+    NODEID=
 COPY --from=builder --chown=1000 /tmp/gobuild/app /updater/app
-COPY --chown=1000 updater/ui/* /updater/ui/
+COPY --chown=1000 ui/* /updater/ui/

README.md

@@ -20,7 +20,7 @@
 
 | Image size | RAM usage | CPU usage |
 | --- | --- | --- |
-| 19.3MB | 13MB | Very low |
+| 21.4MB | 13MB | Very low |
 
 ## Features
 
@@ -63,7 +63,9 @@ chmod 700 data/
 | `DELAY` | `300` | Delay between updates in seconds |
 | `ROOTURL` | `/` | URL path to append to all paths (i.e. `/ddns` for accessing `https://example.com/ddns`) |
 | `LISTENINGPORT` | `8000` | Internal TCP listening port for the web UI |
-| `RECORDi` | | A record to update in the form `domain_name,host,provider,ip_method,password` |
+| `RECORDi` | | A record `i` to update in the form `domain_name,host,provider,ip_method,password` |
+| `LOGGING` | `json` | Format of logging, `json` or `human` |
+| `NODEID` | `0` | Node ID (for distributed systems), can be any integer |
 
 - The environement variables `RECORD1`, `RECORD2`, etc. are domains to update the IP address for
     - The program reads them, starting at `RECORD1` and will stop as soon as `RECORDn` is not set
@@ -158,6 +160,8 @@ In this example, the key is `dLP4WKz5PdkS_GuUDNigHcLQFpw4CWNwAQ5` and the secret
 
 ## TODOs
 
+- [ ] ARM travis builds
+- [ ] Break update function (pkg/update/update.go)
 - [ ] Read parameters from JSON file
 - [ ] Unit tests
 - [ ] Finish readme

docker-compose.yml

@@ -13,6 +13,8 @@ services:
       - DELAY=300
       - ROOTURL=
       - LISTENINGPORT=8000
+      - LOGGING=human
+      - NODEID=0
       - RECORD1=example.com,@,namecheap,provider,0e4512a9c45a4fe88313bcc2234bf547
       - RECORD2=example.info,@,namecheap,duckduckgo,157fd2a9c45a4fe88313bcc2234bfd58
       - RECORD3=example.io,www,namecheap,opendns,0e4512a9c45a4fe88313bcc2234bf547      

go.mod

@@ -0,0 +1,15 @@
+module ddns-updater
+
+go 1.12
+
+require (
+	github.com/BurntSushi/toml v0.3.1 // indirect
+	github.com/fatih/color v1.7.0
+	github.com/google/uuid v1.1.1
+	github.com/julienschmidt/httprouter v1.2.0
+	github.com/kyokomi/emoji v2.1.0+incompatible
+	github.com/mattn/go-colorable v0.1.1 // indirect
+	github.com/mattn/go-isatty v0.0.7 // indirect
+	github.com/mattn/go-sqlite3 v1.10.0
+	github.com/spf13/viper v1.3.2
+)

go.sum

@@ -0,0 +1,60 @@
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kyokomi/emoji v2.1.0+incompatible h1:+DYU2RgpI6OHG4oQkM5KlqD3Wd3UPEsX8jamTo1Mp6o=
+github.com/kyokomi/emoji v2.1.0+incompatible/go.mod h1:mZ6aGCD7yk8j6QY6KICwnZ2pxoszVseX1DNoGtU2tBA=
+github.com/magiconair/properties v1.8.0 h1:LLgXmsheXeRoUOBOjtwPQCWIYqM/LU1ayDtDePerRcY=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg=
+github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
+github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc=
+github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o=
+github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/viper v1.3.2 h1:VUFqw5KcqRf7i70GOzW7N+Q7+gxVBkSSqiXB12+JQ4M=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a h1:1n5lsVfiQW3yfsRGu98756EH1YthsFqr/5mxHduZW2A=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 h1:DH4skfRX4EBpamg7iV4ZlCpblAHI6s6TDM39bFZumv8=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -0,0 +1,94 @@
+package main
+
+import (
+	_ "github.com/mattn/go-sqlite3"
+
+	"fmt"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"ddns-updater/pkg/database"
+
+	"ddns-updater/pkg/healthcheck"
+	"ddns-updater/pkg/logging"
+	"ddns-updater/pkg/params"
+	"ddns-updater/pkg/update"
+	"ddns-updater/pkg/network"
+	"ddns-updater/pkg/server"
+
+	"github.com/kyokomi/emoji"
+)
+
+func main() {
+	if healthcheck.Mode() {
+		healthcheck.Query()
+	}
+	fmt.Println("#################################")
+	fmt.Println("##### DDNS Universal Updater ####")
+	fmt.Println("######## by Quentin McGaw #######")
+	fmt.Println("######## Give some " + emoji.Sprint(":heart:") + "at #########")
+	fmt.Println("# github.com/qdm12/ddns-updater #")
+	fmt.Print("#################################\n\n")
+	go waitForExit()
+	logging.SetGlobalLoggerLevel(logging.InfoLevel)
+	loggerMode := params.GetLoggerMode()
+	logging.SetGlobalLoggerMode(loggerMode)	
+	nodeID := params.GetNodeID()
+	logging.SetGlobalLoggerNodeID(nodeID)
+	httpClient := &http.Client{Timeout: 10 * time.Second}
+	dir := params.GetDir()
+	listeningPort := params.GetListeningPort()
+	rootURL := params.GetRootURL()
+	delay := params.GetDelay()
+	recordsConfigs := params.GetRecordConfigs()
+	logging.Info("Found %d records to update", len(recordsConfigs))
+	go healthcheck.Serve(recordsConfigs)
+	dataDir := params.GetDataDir(dir)
+	errs := network.ConnectivityChecks(httpClient, []string{"google.com"})
+	for _, err := range errs {
+		logging.Warn("%s", err)
+	}
+	sqlDb, err := database.NewDb(dataDir)
+	if err != nil {
+		logging.Fatal("%s", err)
+	}
+	for i := range recordsConfigs {
+		domain := recordsConfigs[i].Settings.Domain
+		host := recordsConfigs[i].Settings.Host
+		logging.Info("Reading history for domain %s and host %s", domain, host)
+		ips, tSuccess, err := sqlDb.GetIps(domain, host)
+		if err != nil {
+			logging.Fatal("%s", err)
+		}
+		recordsConfigs[i].Lock()
+		recordsConfigs[i].History.IPs = ips
+		recordsConfigs[i].History.TSuccess = tSuccess
+		recordsConfigs[i].Unlock()
+	}
+	forceCh := make(chan struct{})
+	quitCh := make(chan struct{})
+	go update.TriggerServer(delay, forceCh, quitCh, recordsConfigs, httpClient, sqlDb)
+	forceCh <- struct{}{}
+	router := server.CreateRouter(rootURL, dir, forceCh, recordsConfigs)
+	logging.Info("Web UI listening on 0.0.0.0:%s", listeningPort)
+	err = http.ListenAndServe("0.0.0.0:"+listeningPort, router)
+	if err != nil {
+		logging.Fatal("%s", err)
+	}
+}
+
+func waitForExit() {
+	signals := make(chan os.Signal)
+	signal.Notify(signals,
+		syscall.SIGINT,
+		syscall.SIGTERM,
+		syscall.SIGKILL,
+		os.Interrupt,
+	)
+	signal := <-signals
+	logging.Warn("Caught OS signal: %s", signal)
+	os.Exit(0)
+}

pkg/database/init.go

@@ -0,0 +1,36 @@
+package database
+
+import (
+	"database/sql"
+	"strings"
+)
+
+// A sqlite database is used to store previous IPs, when re launching the program.
+
+// DB contains the database connection pool pointer.
+// It is used so that methods are declared on it, in order
+// to mock the database easily, through the help of the Datastore interface
+// WARNING: Use in one single go routine, it is not thread safe !
+type DB struct {
+	*sql.DB
+}
+
+// NewDb opens or creates the database if necessary.
+func NewDb(dataDir string) (*DB, error) {
+	dataDir = strings.TrimSuffix(dataDir, "/")
+	db, err := sql.Open("sqlite3", dataDir+"/updates.db")
+	if err != nil {
+		return nil, err
+	}
+	_, err = db.Exec(
+		`CREATE TABLE IF NOT EXISTS updates_ips (
+		domain TEXT NOT NULL,
+		host TEXT NOT NULL,
+		ip TEXT NOT NULL,
+		t_new DATETIME NOT NULL,
+		t_last DATETIME NOT NULL,
+		current INTEGER DEFAULT 1 NOT NULL,
+		PRIMARY KEY(domain, host, ip, t_new)
+		);`)
+	return &DB{db}, err
+}