Updates to my youtube-dl wrapper
Von Carsten
I messed with my youtube-dl wrapper again, to be a bit more robust and handling concurrent downloads. It reads the path to binaries and download directories from a configuration file now too. Here are two screenshots from the running application:
(Original anzeigen)
The application is split into different files (and has some tests, but I will omit them for this article). It’s not pretty, it’s not the cleanest code - but it works and I have a good reason to learn more about Go. 😎
main.go
This file is the starting point and the main application loop. I’m using gorilla/mux to start a HTTP server to serve my static HTML files and execute the youtube-dl processes.
package main
import (
"bufio"
"encoding/json"
"fmt"
"github.com/gorilla/mux"
"log"
"net/http"
"net/url"
"os"
"os/exec"
"time"
)
var (
Log *log.Logger
Configuration Config
)
func _execute(url string) error {
// reset last output
delete(Output, url)
o := Download{CreatedAt: time.Now()}
template := templ(url, Configuration.BaseDir)
//Log.Printf("Using template %s", template)
cmd := exec.Command(Configuration.Executable,
"-f", "best",
"--newline",
"--no-call-home", "--no-check-certificate",
"-o", template,
url)
out, err := cmd.StdoutPipe()
if err != nil {
return err
}
cmd.Start()
scanner := bufio.NewScanner(out)
for scanner.Scan() {
o.Output += scanner.Text() + "\n"
Output[url] = o
}
cmd.Wait()
o.CompletedAt = time.Now()
Output[url] = o
//Log.Printf("Download of %s complete", url)
return nil
}
func showOutput(w http.ResponseWriter, r *http.Request) {
//Log.Printf("query: %s", r.URL.RawQuery)
u := r.URL.Query().Get("url")
//Log.Printf("Showing current output for %s", u)
m := make(map[string]Download)
m["output"] = Output[u]
content, _ := json.Marshal(m)
w.WriteHeader(http.StatusOK)
w.Header().Set("Content-Type", "application/json")
w.Write(content)
}
func showLastDownloads(w http.ResponseWriter, r *http.Request) {
type UrlInfo struct {
Url string
Timestamp time.Time
}
urls := make([]UrlInfo, 0)
for k, v := range Output {
entry := UrlInfo{}
entry.Url = k
entry.Timestamp = v.CreatedAt
urls = append(urls, entry)
}
m := make(map[string][]UrlInfo)
m["recent"] = urls
content, _ := json.Marshal(m)
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(content)
}
func reset(w http.ResponseWriter, r *http.Request) {
clearBuffer()
w.Header().Set("Location", "/static/")
w.WriteHeader(http.StatusFound)
}
func download(w http.ResponseWriter, r *http.Request) {
u := r.PostFormValue("url")
//Log.Printf("Adding %s to download queue", u)
go _execute(u)
w.Header().Set("Location", fmt.Sprintf("/static/success.html?url=%s", url.QueryEscape(u)))
w.WriteHeader(http.StatusFound)
}
func redirect(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Location", "/static/")
w.WriteHeader(http.StatusFound)
}
func main() {
Log = log.New(os.Stdout, "youtube-ex2", log.Lshortfile)
configFilename := "/opt/app/config.yml"
if len(os.Args) >= 2 {
args := os.Args[1:]
configFilename = args[0]
}
Configuration, err := loadConfiguration(configFilename)
if err != nil {
Log.Fatalf("Can not read %s: %s", configFilename, err)
os.Exit(1)
}
r := mux.NewRouter()
r.HandleFunc("/", redirect)
r.HandleFunc("/download", download).Methods("POST")
r.HandleFunc("/progress", showOutput)
r.HandleFunc("/recent", showLastDownloads)
r.HandleFunc("/reset", reset)
r.PathPrefix("/static/").Handler(http.StripPrefix("/static/", http.FileServer(http.Dir("static"))))
sport := fmt.Sprintf(":%d", Configuration.Port)
Log.Printf("Listening on %s", sport)
Log.Fatal(http.ListenAndServe(sport, r))
}
config.go
This file has the struct and a function to load the configuration from a YAML file. This was fairly easy with the package gopkg.in/yaml.v2 and I will probably use this in future projects too.
package main
import (
"gopkg.in/yaml.v2"
"io/ioutil"
)
type Config struct {
Port int
BaseDir string
Executable string
}
func loadConfiguration(filename string) (*Config, error) {
// read configuration from file
Log.Printf("Loading configuration from %s", filename)
buf, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
Configuration := new(Config)
err = yaml.Unmarshal(buf, &Configuration)
if err != nil {
return nil, err
}
return Configuration, nil
}
downloadbuffer.go
This file contains all functions, constants and a variable Output to store information about the actual video downloads. I set a limit to the number of elements in the buffer to avoid storing too much stuff on my tiny appliance. This information is also not very important to me, I only care about the actual downloaded file so I didn’t bother to persist this to a file.
package main
import (
"time"
)
type Download struct {
CreatedAt time.Time
CompletedAt time.Time
Output string
}
const MAXIMUM_SIZE = 10
type Buffer map[string]Download
var Output = make(Buffer)
func appendDownload(url string, download Download) {
Output[url] = download
if len(Output) > MAXIMUM_SIZE {
oldest := download
var oldestKey string
for k, v := range Output {
if v.CreatedAt.Before(oldest.CreatedAt) {
oldest = v
oldestKey = k
}
}
delete(Output, oldestKey)
}
}
func clearBuffer() {
Output = Buffer{}
}
index.html
The frontend is very basic HTML+JS.
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="/static/css/bootstrap.css" rel="stylesheet"/>
</head>
<body>
<main class="container">
<div class="py-5 px-3">
<h1>Video runterladen</h1>
<form method="post" action="/download">
<div class="input-group mb-3">
<input type="url" name="url" class="form-control" placeholder="URL einfügen, z.B. https://www.youtube.com/watch?v=..."/>
<button type="submit" class="btn btn-primary">Abschicken</button>
</div>
</form>
<h1>Letzte Downloads</h1>
<ul name="recent">
</ul>
<div>
<a href="/reset" class="btn btn-outline-danger">Liste löschen</a>
</div>
</div>
</main>
<script src="/static/js/bootstrap.js"></script>
<script>
const element = document.getElementsByName('recent')[0];
fetch('/recent')
.then(resp => resp.json())
.then(json => json.recent.sort((a, b) => a.Timestamp < b.Timestamp).forEach(download => {
const entry = document.createElement('li');
const link = document.createElement('a');
const params = new URLSearchParams();
params.append('url', download.Url);
link.href = 'success.html?' + params;
link.textContent = download.Url;
entry.append(link);
entry.append(' ' + new Date(download.Timestamp).toLocaleString());
element.append(entry);
}));
</script>
</body>
</html>
success.html
<html>
<head>
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="/static/css/bootstrap.css" rel="stylesheet"/>
</head>
<body>
<main class="container">
<div class="py-5 px-3">
<h1>Lade runter...</h1>
<pre></pre>
<div class="spinner-border" role="status" name="spinner"></div>
<div>
<a href="/">Zurück</a>
</div>
</div>
</main>
<script src="/static/js/bootstrap.js"></script>
<script>
let interval;
function update() {
const url = new URLSearchParams(window.location.search).get('url');
const spinner = document.getElementsByName('spinner')[0];
const element = document.getElementsByTagName('pre')[0];
fetch('/progress?url=' + url)
.then(resp => resp.json())
.then(json => {
element.textContent = json.output.Output;
if (new Date(json.output.CompletedAt).getTime() > 0) {
// clear interval
clearInterval(interval);
spinner.style = 'display: none';
}
})
.then(() => document.scrollingElement.scrollTop = document.scrollingElement.scrollHeight);
}
update();
interval = setInterval(update, 1000);
</script>
</body>
</html>