Updates to my youtube-dl wrapper

December 28, 2020

I messed with my youtube-dl wrapper again, to be a bit more robust and handling concurrent downloads. It reads the path to binaries and download directories from a configuration file now too. Here are two screenshots from the running application:

The application is split into different files (and has some tests, but I will omit them for this article). It’s not pretty, it’s not the cleanest code - but it works and I have a good reason to learn more about Go. 😎

main.go

This file is the starting point and the main application loop. I’m using gorilla/mux to start a HTTP server to serve my static HTML files and execute the youtube-dl processes.

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"github.com/gorilla/mux"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"time"
)

var (
	Log           *log.Logger
	Configuration Config
)

func _execute(url string) error {
	// reset last output
	delete(Output, url)
	o := Download{CreatedAt: time.Now()}
	template := templ(url, Configuration.BaseDir)
	//Log.Printf("Using template %s", template)
	cmd := exec.Command(Configuration.Executable,
		"-f", "best",
		"--newline",
		"--no-call-home", "--no-check-certificate",
		"-o", template,
		url)
	out, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}

	cmd.Start()
	scanner := bufio.NewScanner(out)
	for scanner.Scan() {
		o.Output += scanner.Text() + "\n"
		Output[url] = o
	}
	cmd.Wait()
	o.CompletedAt = time.Now()
	Output[url] = o

	//Log.Printf("Download of %s complete", url)
	return nil
}

func showOutput(w http.ResponseWriter, r *http.Request) {
	//Log.Printf("query: %s", r.URL.RawQuery)
	u := r.URL.Query().Get("url")
	//Log.Printf("Showing current output for %s", u)

	m := make(map[string]Download)
	m["output"] = Output[u]
	content, _ := json.Marshal(m)

	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "application/json")
	w.Write(content)
}

func showLastDownloads(w http.ResponseWriter, r *http.Request) {
	type UrlInfo struct {
		Url       string
		Timestamp time.Time
	}

	urls := make([]UrlInfo, 0)
	for k, v := range Output {
		entry := UrlInfo{}
		entry.Url = k
		entry.Timestamp = v.CreatedAt
		urls = append(urls, entry)
	}

	m := make(map[string][]UrlInfo)
	m["recent"] = urls
	content, _ := json.Marshal(m)

	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)
	w.Write(content)
}

func reset(w http.ResponseWriter, r *http.Request) {
	clearBuffer()

	w.Header().Set("Location", "/static/")
	w.WriteHeader(http.StatusFound)
}

func download(w http.ResponseWriter, r *http.Request) {
	u := r.PostFormValue("url")
	//Log.Printf("Adding %s to download queue", u)
	go _execute(u)
	w.Header().Set("Location", fmt.Sprintf("/static/success.html?url=%s", url.QueryEscape(u)))
	w.WriteHeader(http.StatusFound)
}

func redirect(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Location", "/static/")
	w.WriteHeader(http.StatusFound)
}

func main() {
	Log = log.New(os.Stdout, "youtube-ex2", log.Lshortfile)

	configFilename := "/opt/app/config.yml"
	if len(os.Args) >= 2 {
		args := os.Args[1:]
		configFilename = args[0]
	}
	Configuration, err := loadConfiguration(configFilename)
	if err != nil {
		Log.Fatalf("Can not read %s: %s", configFilename, err)
		os.Exit(1)
	}

	r := mux.NewRouter()
	r.HandleFunc("/", redirect)
	r.HandleFunc("/download", download).Methods("POST")
	r.HandleFunc("/progress", showOutput)
	r.HandleFunc("/recent", showLastDownloads)
	r.HandleFunc("/reset", reset)
	r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))

	sport := fmt.Sprintf(":%d", Configuration.Port)
	Log.Printf("Listening on %s", sport)
	Log.Fatal(http.ListenAndServe(sport, r))
}

config.go

This file has the struct and a function to load the configuration from a YAML file. This was fairly easy with the package gopkg.in/yaml.v2 and I will probably use this in future projects too.

package main

import (
	"gopkg.in/yaml.v2"
	"io/ioutil"
)

type Config struct {
	Port       int
	BaseDir    string
	Executable string
}

func loadConfiguration(filename string) (*Config, error) {
	// read configuration from file
	Log.Printf("Loading configuration from %s", filename)
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	Configuration := new(Config)
	err = yaml.Unmarshal(buf, &Configuration)
	if err != nil {
		return nil, err
	}

	return Configuration, nil
}

downloadbuffer.go

This file contains all functions, constants and a variable Output to store information about the actual video downloads. I set a limit to the number of elements in the buffer to avoid storing too much stuff on my tiny appliance. This information is also not very important to me, I only care about the actual downloaded file so I didn’t bother to persist this to a file.

package main

import (
	"time"
)

type Download struct {
	CreatedAt   time.Time
	CompletedAt time.Time
	Output      string
}

const MAXIMUM_SIZE = 10

type Buffer map[string]Download

var Output = make(Buffer)

func appendDownload(url string, download Download) {
	Output[url] = download
	if len(Output) > MAXIMUM_SIZE {
		oldest := download
		var oldestKey string
		for k, v := range Output {
			if v.CreatedAt.Before(oldest.CreatedAt) {
				oldest = v
				oldestKey = k
			}
		}
		delete(Output, oldestKey)
	}
}

func clearBuffer() {
	Output = Buffer{}
}

index.html

The frontend is very basic HTML+JS.

<html>
<head>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <link href="/static/css/bootstrap.css" rel="stylesheet"/>
</head>
<body>

<main class="container">
    <div class="py-5 px-3">
        <h1>Video runterladen</h1>
        <form method="post" action="/download">
            <div class="input-group mb-3">
                <input type="url" name="url" class="form-control" placeholder="URL einfügen, z.B. https://www.youtube.com/watch?v=..."/>
                <button type="submit" class="btn btn-primary">Abschicken</button>
            </div>
        </form>
        <h1>Letzte Downloads</h1>
        <ul name="recent">
        </ul>
        <div>
            <a href="/reset" class="btn btn-outline-danger">Liste löschen</a>
        </div>

    </div>
</main>

<script src="/static/js/bootstrap.js"></script>
<script>
    const element = document.getElementsByName('recent')[0];
    fetch('/recent')
        .then(resp => resp.json())
        .then(json => json.recent.sort((a, b) => a.Timestamp < b.Timestamp).forEach(download => {
            const entry = document.createElement('li');
            const link = document.createElement('a');
            const params = new URLSearchParams();
            params.append('url', download.Url);
            link.href = 'success.html?' + params;
            link.textContent = download.Url;
            entry.append(link);
            entry.append(' ' + new Date(download.Timestamp).toLocaleString());
            element.append(entry);
        }));
</script>
</body>
</html>

success.html

<html>
<head>
    <meta name="viewport" content="width=device-width, initial-scale=1">
    <link href="/static/css/bootstrap.css" rel="stylesheet"/>
</head>
<body>

<main class="container">
    <div class="py-5 px-3">
        <h1>Lade runter...</h1>
        <pre></pre>
        <div class="spinner-border" role="status" name="spinner"></div>
        <div>
            <a href="/">Zurück</a>
        </div>
    </div>
</main>

<script src="/static/js/bootstrap.js"></script>
<script>
    let interval;

    function update() {
        const url = new URLSearchParams(window.location.search).get('url');
        const spinner = document.getElementsByName('spinner')[0];
        const element = document.getElementsByTagName('pre')[0];
        fetch('/progress?url=' + url)
            .then(resp => resp.json())
            .then(json => {
                element.textContent = json.output.Output;
                if (new Date(json.output.CompletedAt).getTime() > 0) {
                    // clear interval
                    clearInterval(interval);
                    spinner.style = 'display: none';
                }
            })
            .then(() => document.scrollingElement.scrollTop = document.scrollingElement.scrollHeight);
    }

    update();
    interval = setInterval(update, 1000);
</script>
</body>
</html>

youtube