I messed with my youtube-dl
wrapper again, to be a bit more robust and handling concurrent downloads. It reads the path to binaries and download directories from a configuration file now too. Here are two screenshots from the running application:


The application is split into different files (and has some tests, but I will omit them for this article). It’s not pretty, it’s not the cleanest code – but it works and I have a good reason to learn more about Go. đ
main.go
This file is the starting point and the main application loop. I’m using gorilla/mux to start a HTTP server to serve my static HTML files and execute the youtube-dl
processes.
package main import ( "bufio" "encoding/json" "fmt" "github.com/gorilla/mux" "log" "net/http" "net/url" "os" "os/exec" "time" ) var ( Log *log.Logger Configuration Config ) func _execute(url string) error { // reset last output delete(Output, url) o := Download{CreatedAt: time.Now()} template := templ(url, Configuration.BaseDir) //Log.Printf("Using template %s", template) cmd := exec.Command(Configuration.Executable, "-f", "best", "--newline", "--no-call-home", "--no-check-certificate", "-o", template, url) out, err := cmd.StdoutPipe() if err != nil { return err } cmd.Start() scanner := bufio.NewScanner(out) for scanner.Scan() { o.Output += scanner.Text() + "\n" Output[url] = o } cmd.Wait() o.CompletedAt = time.Now() Output[url] = o //Log.Printf("Download of %s complete", url) return nil } func showOutput(w http.ResponseWriter, r *http.Request) { //Log.Printf("query: %s", r.URL.RawQuery) u := r.URL.Query().Get("url") //Log.Printf("Showing current output for %s", u) m := make(map[string]Download) m["output"] = Output[u] content, _ := json.Marshal(m) w.WriteHeader(http.StatusOK) w.Header().Set("Content-Type", "application/json") w.Write(content) } func showLastDownloads(w http.ResponseWriter, r *http.Request) { type UrlInfo struct { Url string Timestamp time.Time } urls := make([]UrlInfo, 0) for k, v := range Output { entry := UrlInfo{} entry.Url = k entry.Timestamp = v.CreatedAt urls = append(urls, entry) } m := make(map[string][]UrlInfo) m["recent"] = urls content, _ := json.Marshal(m) w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusOK) w.Write(content) } func reset(w http.ResponseWriter, r *http.Request) { clearBuffer() w.Header().Set("Location", "/static/") w.WriteHeader(http.StatusFound) } func download(w http.ResponseWriter, r *http.Request) { u := r.PostFormValue("url") //Log.Printf("Adding %s to download queue", u) go _execute(u) w.Header().Set("Location", fmt.Sprintf("/static/success.html?url=%s", url.QueryEscape(u))) w.WriteHeader(http.StatusFound) } func redirect(w http.ResponseWriter, r *http.Request) { w.Header().Set("Location", "/static/") w.WriteHeader(http.StatusFound) } func main() { Log = log.New(os.Stdout, "youtube-ex2", log.Lshortfile) configFilename := "/opt/app/config.yml" if len(os.Args) >= 2 { args := os.Args[1:] configFilename = args[0] } Configuration, err := loadConfiguration(configFilename) if err != nil { Log.Fatalf("Can not read %s: %s", configFilename, err) os.Exit(1) } r := mux.NewRouter() r.HandleFunc("/", redirect) r.HandleFunc("/download", download).Methods("POST") r.HandleFunc("/progress", showOutput) r.HandleFunc("/recent", showLastDownloads) r.HandleFunc("/reset", reset) r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("static")))) sport := fmt.Sprintf(":%d", Configuration.Port) Log.Printf("Listening on %s", sport) Log.Fatal(http.ListenAndServe(sport, r)) }
config.go
This file has the struct
and a function to load the configuration from a YAML file. This was fairly easy with the package gopkg.in/yaml.v2 and I will probably use this in future projects too.
package main import ( "gopkg.in/yaml.v2" "io/ioutil" ) type Config struct { Port int BaseDir string Executable string } func loadConfiguration(filename string) (*Config, error) { // read configuration from file Log.Printf("Loading configuration from %s", filename) buf, err := ioutil.ReadFile(filename) if err != nil { return nil, err } Configuration := new(Config) err = yaml.Unmarshal(buf, &Configuration) if err != nil { return nil, err } return Configuration, nil }
downloadbuffer.go
This file contains all functions, constants and a variable Output
to store information about the actual video downloads. I set a limit to the number of elements in the buffer to avoid storing too much stuff on my tiny appliance. This information is also not very important to me, I only care about the actual downloaded file so I didn’t bother to persist this to a file.
package main import ( "time" ) type Download struct { CreatedAt time.Time CompletedAt time.Time Output string } const MAXIMUM_SIZE = 10 type Buffer map[string]Download var Output = make(Buffer) func appendDownload(url string, download Download) { Output[url] = download if len(Output) > MAXIMUM_SIZE { oldest := download var oldestKey string for k, v := range Output { if v.CreatedAt.Before(oldest.CreatedAt) { oldest = v oldestKey = k } } delete(Output, oldestKey) } } func clearBuffer() { Output = Buffer{} }
index.html
The frontend is very basic HTML+JS.
<html> <head> <meta name="viewport" content="width=device-width, initial-scale=1"> <link href="/static/css/bootstrap.css" rel="stylesheet"/> </head> <body> <main class="container"> <div class="py-5 px-3"> <h1>Video runterladen</h1> <form method="post" action="/download"> <div class="input-group mb-3"> <input type="url" name="url" class="form-control" placeholder="URL einfĂŒgen, z.B. https://www.youtube.com/watch?v=..."/> <button type="submit" class="btn btn-primary">Abschicken</button> </div> </form> <h1>Letzte Downloads</h1> <ul name="recent"> </ul> <div> <a href="/reset" class="btn btn-outline-danger">Liste löschen</a> </div> </div> </main> <script src="/static/js/bootstrap.js"></script> <script> const element = document.getElementsByName('recent')[0]; fetch('/recent') .then(resp => resp.json()) .then(json => json.recent.sort((a, b) => a.Timestamp < b.Timestamp).forEach(download => { const entry = document.createElement('li'); const link = document.createElement('a'); const params = new URLSearchParams(); params.append('url', download.Url); link.href = 'success.html?' + params; link.textContent = download.Url; entry.append(link); entry.append(' ' + new Date(download.Timestamp).toLocaleString()); element.append(entry); })); </script> </body> </html>
success.html
<html> <head> <meta name="viewport" content="width=device-width, initial-scale=1"> <link href="/static/css/bootstrap.css" rel="stylesheet"/> </head> <body> <main class="container"> <div class="py-5 px-3"> <h1>Lade runter...</h1> <pre></pre> <div class="spinner-border" role="status" name="spinner"></div> <div> <a href="/">ZurĂŒck</a> </div> </div> </main> <script src="/static/js/bootstrap.js"></script> <script> let interval; function update() { const url = new URLSearchParams(window.location.search).get('url'); const spinner = document.getElementsByName('spinner')[0]; const element = document.getElementsByTagName('pre')[0]; fetch('/progress?url=' + url) .then(resp => resp.json()) .then(json => { element.textContent = json.output.Output; if (new Date(json.output.CompletedAt).getTime() > 0) { // clear interval clearInterval(interval); spinner.style = 'display: none'; } }) .then(() => document.scrollingElement.scrollTop = document.scrollingElement.scrollHeight); } update(); interval = setInterval(update, 1000); </script> </body> </html>