Detail how to hot restart the golang server

Server-side code often needs to be upgraded. For online system upgrading, a common practice is to ensure that at least one service is available through front-end load balancing (such as nginx), and upgrade successively (gray scale).

A more convenient approach is to do a hot restart on the application and upgrade the application without stopping.

The principle of

The principle of a hot restart is simple, but involves a few system calls and the passing of file handles between parent and child processes.
The process is divided into the following steps:

Listening signal (USR2) Upon receipt of the signal, the fork child process (using the same start command) passes the socket file descriptor that the service is listening to to the child process The child listens to the parent's socket, at which point both the parent and child can receive requests After the child starts successfully, the parent process stops receiving new connections and waits for the processing of the old one to complete (or time out) The parent process exits and the upgrade is complete

details

The parent can pass the socket file descriptor to the child via the command line, environment variables, and so on The child starts with the same command line as the parent, and for golang overrides the old program with the newer executable server. Shutdown() elegant closing method is a new feature of go1.8 The server. Serve(l) method returns immediately on Shutdown, and the Shutdown method blocks until context completes, so the Shutdown method is written in the main goroutine

code

package main
import (
  "context"
  "errors"
  "flag"
  "log"
  "net"
  "net/http"
  "os"
  "os/exec"
  "os/signal"
  "syscall"
  "time"
)
 
var (
  server  *http.Server
  listener net.Listener
  graceful = flag.Bool("graceful", false, "listen on fd open 3 (internal use only)")
)
 
func handler(w http.ResponseWriter, r *http.Request) {
  time.Sleep(20 * time.Second)
  w.Write([]byte("hello world233333!!!!"))
}
 
func main() {
  flag.Parse()
 
  http.HandleFunc("/hello", handler)
  server = &http.Server{Addr: ":9999"}
 
  var err error
  if *graceful {
    log.Print("main: Listening to existing file descriptor 3.")
    // cmd.ExtraFiles: If non-nil, entry i becomes file descriptor 3+i.
    // when we put socket FD at the first entry, it will always be 3(0+3)
    f := os.NewFile(3, "")
    listener, err = net.FileListener(f)
  } else {
    log.Print("main: Listening on a new file descriptor.")
    listener, err = net.Listen("tcp", server.Addr)
  }
 
  if err != nil {
    log.Fatalf("listener error: %v", err)
  }
 
  go func() {
    // server.Shutdown() stops Serve() immediately, thus server.Serve() should not be in main goroutine
    err = server.Serve(listener)
    log.Printf("server.Serve err: %v\n", err)
  }()
  signalHandler()
  log.Printf("signal end")
}
 
func reload() error {
  tl, ok := listener.(*net.TCPListener)
  if !ok {
    return errors.New("listener is not tcp listener")
  }
 
  f, err := tl.File()
  if err != nil {
    return err
  }
 
  args := []string{"-graceful"}
  cmd := exec.Command(os.Args[0], args...)
  cmd.Stdout = os.Stdout
  cmd.Stderr = os.Stderr
  // put socket FD at the first entry
  cmd.ExtraFiles = []*os.File{f}
  return cmd.Start()
}
 
func signalHandler() {
  ch := make(chan os.Signal, 1)
  signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM, syscall.SIGUSR2)
  for {
    sig := <-ch
    log.Printf("signal: %v", sig)
 
    // timeout context for shutdown
    ctx, _ := context.WithTimeout(context.Background(), 20*time.Second)
    switch sig {
    case syscall.SIGINT, syscall.SIGTERM:
      // stop
      log.Printf("stop")
      signal.Stop(ch)
      server.Shutdown(ctx)
      log.Printf("graceful shutdown")
      return
    case syscall.SIGUSR2:
      // reload
      log.Printf("reload")
      err := reload()
      if err != nil {
        log.Fatalf("graceful restart error: %v", err)
      }
      server.Shutdown(ctx)
      log.Printf("graceful reload")
      return
    }
  }
}

references

Graceful Restart in Golang

facebookgo/grace