Playing Around with Go

I've previously described my temperature monitoring solution, written in Python, and I've also described my various attempts at optimizing this solution, using NodeRED and Apache Camel, but all of these attempts have been focused on the server side, while the client has been mostly left to itself.

The client runs on an old Raspberry Pi B+, with a total of 256MB RAM. The RPi also runs a surveillance camera, via the RPi camera module, which requires a memory split of 128 MB. Previously this has been more than enough. The python client ate up 18-25 MB ram, and the surveillance images were streamed to my NAS, which then did the motion detection.
Recently I started experimenting with letting the RPi run motion detection on its own, and only stream video when motion is detected (more about that in a later post), and RAM started to be a bit scarce.

Ever since Go was released, I've had it on my todo list to learn how to program in it. Not because I have a particular need for yet another language, but because I'm a geek :-)
After seeing examples like the code below, I was fairly sure that Go would be a good match.

package main

import (  
  "fmt"
  "net/http"
)

func main() {  
  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    fmt.Fprintf(w, "Hello World!")
  })
  http.ListenAndServe(":8080", nil)
}

Example web server in Go, using only the standard packages.

So after a short Internet search, to see if the required functionality for writing the surveillance client was available for Go, I threw myself at it.

Go turned out to be remarkably easy to learn. I started out by "browsing" the excellent, free An Introduction to Programming in Go. book.

My original fear that Go would turn out like another Java were put to shame. Implementing the surveillance client in Go, I was able to reimplement the same functionality with only 10% extra code compared to the python version. I don't consider that a bad trade off for gaining a statically typed language.

Here's what I ended up with.

package main

import (  
    "bufio"
    "encoding/json"
    "flag"
    "fmt"
    MQTT "github.com/eclipse/paho.mqtt.golang"
    "github.com/jasonlvhit/gocron"
    "github.com/op/go-logging"
    "os"
    "path"
    "path/filepath"
    "strconv"
    "strings"
    "time"
)

var (  
    flags    SurveillanceFlags
    mqclient *MQTT.Client
    log      = logging.MustGetLogger("surveillance-client")
    format   = logging.MustStringFormatter(`%{color}%{time:15:04:05.000} %{shortfunc:12.12s} ▶ %{level:.5s} %{color:reset} %{message}`)
    sensors  []string
)

type SurveillanceFlags struct {  
    host     string
    hostname string
    port     int
    topic    string
    verbose  bool
    logfile  string
}

type TemperatureReading struct {  
    sensor  string
    reading float64
}

func InitFlags() (flags SurveillanceFlags) {  
    flag.StringVar(&flags.hostname, "h", "localhost", "MQTT Hostname to connect to, defaults to localhost")
    flag.IntVar(&flags.port, "p", 1883, "MQTT Port, defaults to 1883")
    flag.StringVar(&flags.host, "H", "undefined", "Hostname to report to server")
    flag.StringVar(&flags.topic, "t", "/surveillance/temperature/", "MQTT Topic to publish to")
    flag.StringVar(&flags.logfile, "l", "stderr", "Enable logging to file")
    flag.BoolVar(&flags.verbose, "v", false, "Enable verbose logging")
    flag.Parse()
    return flags
}

func init() {  
    flags = InitFlags()
    var outfile []*os.File
    outfile = append(outfile, os.Stderr)

    if flags.logfile != "stderr" {
        f, err := os.OpenFile(flags.logfile, os.O_RDWR|os.O_CREATE|os.O_APPEND, os.ModePerm)
        if err != nil {
            log.Error(err)
        } else {
            outfile = append(outfile, f)
        }
    }

    var backends []logging.Backend

    for _, of := range outfile {
        backend1 := logging.NewLogBackend(of, "", 0)
        backend1Formatter := logging.NewBackendFormatter(backend1, format)
        backend1Leveled := logging.AddModuleLevel(backend1Formatter)
        if flags.verbose == false {
            backend1Leveled.SetLevel(logging.INFO, "")
        } else {
            log.Info("Enabling verbose logging")
            backend1Leveled.SetLevel(logging.DEBUG, "")
        }
        backends = append(backends, backend1Leveled)
    }
    logging.SetBackend(backends...)
    mqclient = SetupMQTT(flags.hostname, flags.port)
}

func ConnectionLost(client *MQTT.Client, err error) {  
    log.Error("Connection Lost:", err)
    token := client.Connect()
    token.Wait()
}

func SetupMQTT(host string, port int) *MQTT.Client {  
    url := fmt.Sprintf("tcp://%s:%d", host, port)
    opts := MQTT.NewClientOptions().AddBroker(url)
    opts.SetAutoReconnect(true)
    opts.SetConnectionLostHandler(ConnectionLost)
    dur, _ := time.ParseDuration("1m")
    opts.SetKeepAlive(dur)
    log.Info("Connecting to ", url)
    c := MQTT.NewClient(opts)
    if token := c.Connect(); token.Wait() && token.Error() != nil {
        panic(token.Error())
    }
    log.Info("Done setting up MQTT")
    return c
}

func readTempRaw(filename string) []string {  
    var ret []string
    file, err := os.Open(filename)
    if err != nil {
        log.Fatal(err)
    }
    defer file.Close()

    scanner := bufio.NewScanner(file)
    for scanner.Scan() {
        ret = append(ret, scanner.Text())
    }

    if err := scanner.Err(); err != nil {
        log.Fatal(err)
    }
    return ret
}

func ReadTemp() (readings []TemperatureReading) {

    files := make([]string, len(sensors))
    for idx, file := range sensors {
        files[idx] = filepath.Join(file, "w1_slave")
        log.Debug("Adding sensor ", files[idx])
    }
    readings = make([]TemperatureReading, len(files))
    for idx, file := range files {
        lines := readTempRaw(file)
        retryCt := 0
        for {
            l := strings.TrimSpace(lines[0])
            slice := l[len(l)-3 : len(l)]
            if slice != "YES" {
                log.Debug("slice is ", slice, ", rereading file")
                time.Sleep(200 * time.Millisecond)
                retryCt++
                if retryCt < 10 {
                    lines = readTempRaw(file)
                } else {
                    log.Error("Giving up!, Max retries reached reading file ", file)
                    break
                }
            } else {
                break
            }
        }
        equals_pos := strings.Index(lines[1], "t=")
        if equals_pos != -1 {
            temp_string := lines[1][equals_pos+2:]
            temp_c, err := strconv.ParseFloat(temp_string, 64)
            if err != nil {
                log.Error(err)
                continue
            }
            temp_c = temp_c / 1000.0
            //temp_f := temp_c * 9.0 / 5.0 + 32.0
            readings[idx].reading = temp_c
            readings[idx].sensor = path.Base(sensors[idx])
        } else {
            log.Debug("equals sign not found in \"" + lines[1] + "\"")
        }
    }
    return readings
}

func ReadAndPublish() {  
    go func() {
        log.Debug("Reading temperatures")
        readings := ReadTemp()
        for _, reading := range readings {
            out := make(map[string]map[string]interface{}, 1)
            out["reading"] = make(map[string]interface{}, 1)
            out["reading"]["host"] = flags.host
            out["reading"]["timestamp"] = time.Now().Format("2006-01-02 15:04:05.000000")
            out["reading"]["sensor"] = reading.sensor
            out["reading"]["reading"] = reading.reading

            output, err := json.Marshal(out)
            if err != nil {
                log.Error(err)
                continue
            }
            log.Debug(string(output))
            go func() {
                token := mqclient.Publish(flags.topic+flags.host+"/"+reading.sensor, 1, false, output)
                token.Wait()
            }()
        }
        _, time := gocron.NextRun()
        log.Debug("Next update at ", time)
    }()
}

func RegisterSensors() {  
    var err error
    log.Info("Registering Sensors")
    sensors, err = filepath.Glob("/sys/bus/w1/devices/28*")
    if err != nil {
        panic(err)
    }
    for _, sensor := range sensors {
        out := make(map[string]map[string]interface{}, 1)
        out["register_sensor"] = make(map[string]interface{}, 1)
        out["register_sensor"]["host"] = flags.host
        out["register_sensor"]["sensor"] = path.Base(sensor)

        output, err := json.Marshal(out)
        if err != nil {
            log.Error(err)
            continue
        }
        log.Debug(string(output))
        go func() {
            token := mqclient.Publish(flags.topic+flags.host+"/"+path.Base(sensor), 1, false, output)
            token.Wait()
        }()
    }
}

func main() {  
    RegisterSensors()
    gocron.Every(1).Minute().Do(ReadAndPublish)
    ReadAndPublish() //Perform reading when starting up
    <-gocron.Start()
}

Initial testing looks promising. There isn't much to gain in the performance department, all the client does is read a couple of files every minute, something Python is more than fast enough at, but memory wise it's a completely different thing:

The old client:

13456 nobody      20   0 50732 18136  8312 S  0.5  4.8  3:06.45 python3  
13457 nobody      20   0 50732 18136  8312 S  0.9  4.8  1:46.40 python3  
13458 nobody      20   0 50732 18136  8312 S  0.0  4.8  1:45.76 python3  
13448 nobody      20   0 50732 18136  8312 S  1.4  4.8  6:44.31 python3  

The new client:

18967 nobody 20   0  774M  5320  4400 S  0.0  1.4  0:00.04 /usr/local/bin//surveillance_client  
18968 nobody 20   0  774M  5320  4400 S  0.0  1.4  0:00.02 /usr/local/bin//surveillance_client  
18973 nobody 20   0  774M  5320  4400 S  0.0  1.4  0:00.03 /usr/local/bin//surveillance_client  
18974 nobody 20   0  774M  5320  4400 S  0.0  1.4  0:00.03 /usr/local/bin//surveillance_client  
18964 nobody 20   0  774M  5320  4400 S  0.0  1.4  0:00.26 /usr/local/bin//surveillance_client  

4.5 MB vs 18MB. Normally not something to even make it worth the rewrite, but on a memory constrained platform it makes a huge difference.

I'm still playing around with Go, experimenting with Goroutines and channels, and at the same time implementing some more CPU intensive things.

Expect more stories about Go, as I attempt to wrestle more power from my Raspberry Pi servers :-

Jimmy Selgen Nielsen

Read more posts by this author.