youtube-dl-parallel

#!/bin/bash
# Copyright (c) 2013 DLH. See LICENSE.txt for the MIT license.

function error() {
    echo "Error: $@"
    echo
    echo "Traceback"
    echo "========="
    local frame=0
    while caller $frame; do
        ((frame++));
    done
    exit 1
}

function prefix_output() {
    local padding=$1
    local prefix=$2
    while read line; do
        printf "%-${padding}s: %s\n" "$prefix" "$line"
    done
}

function download() {
    local padding=$1
    local prefix=$2
    local url=$3
    shift 3
    youtube-dl "$@" --no-playlist --newline -- "$url" 2>&1 | prefix_output "$padding" "$prefix"
}

function get_title() {
    local title_directory=$1
    local url=$2
    local title=$(youtube-dl --no-playlist --youtube-skip-dash-manifest --get-title -- "$url" 2>&1)
    test $? -eq 0 || title="$url"
    echo "$title" > "$title_directory/$PARALLEL_SEQ"
}

function parse_options()  {
    while getopts "hj:" option "$@"; do
        case $option in
            h)
                echo "Usage: ${0##*/} [-hj] <url> [<url> ...] [-- <youtube-dl options>]"
                echo "    -h       : Shows this help message."
                echo "    -j <jobs>: The number of jobs to run in parallel. The default is 3."
                echo
                echo "If “-” is provided as an argument, then urls will additionally be read from stdin."
                exit;;
            j)
                JOBS="$OPTARG";;
            \?)
                error "Unknown option";;
        esac
    done

    # Special case. See test/test_options_ytdl_args_no_urls.sh
    if test $# -gt 0 -a "$1" != "--"; then
        shift $((OPTIND - 1))
    fi

    for url in "$@"; do
        if test "$url" = "-"; then
            while read line; do
                URLS+=($line)
            done
        elif test "$url" = "--"; then
            shift
            YOUTUBE_DL_ARGS="$@"
            break
        else
            URLS+=($url)
        fi
        shift
    done
}

function find_longest_title() {
    local title_directory=$1
    pushd "$title_directory" > /dev/null
    for title_file in $(ls | sort -n); do
        local title=$(cat "$title_file")
        local length=$(echo -n "$title" | wc -m | tr -d '[[:space:]]')
        if test $length -gt $TITLE_PADDING; then
            TITLE_PADDING=$length
        fi
        URL_TITLES+=("$title")
    done
    popd > /dev/null
}

function create_title_directory() {
    mktemp -d -t youtube-dl-parallel.XXX || error "mktemp failed"
}

function parallel_will_cite_option() {
    parallel --gnu --will-cite true ::: "" > /dev/null 2>&1 && echo "--will-cite"
    return 0
}

function main() {
    parse_options "$@"
    if test -z "$URLS"; then
        error "No urls specified. Try ${0##*/} -h for more information."
    fi

    local title_directory=$(create_title_directory)
    trap "rm -r '$title_directory'" EXIT

    # Export functions for use with parallel
    export -f prefix_output download get_title

    # * Check if we can use the --will-cite option. Ubuntu 12.04 LTS ships with a very
    #   old version of GNU Parallel that does not support it
    # * Ubuntu 12.04's parallel also uses the --tollef option by default, so we need
    #   to override that here.
    # * We use --will-cite because the end-user should not be spammed to cite our usage
    #   of GNU Parallel.
    local args="$(parallel_will_cite_option) --gnu --jobs $JOBS"
    parallel $args "get_title '$title_directory'" ::: "${URLS[@]}"
    find_longest_title "$title_directory"
    parallel $args --xapply --ungroup "download $((TITLE_PADDING + 1)) {1} {2} $YOUTUBE_DL_ARGS" ::: "${URL_TITLES[@]}" ::: "${URLS[@]}"
}

# Global variables
declare -a URLS URL_TITLES
JOBS=3
YOUTUBE_DL_ARGS=
TITLE_PADDING=0

if test "${BASH_SOURCE[0]}" = "${0}"; then
    main "$@"
fi