diff --git "a/js/youtube-metadata-bulk.js" "b/js/youtube-metadata-bulk.js" new file mode 100644--- /dev/null +++ "b/js/youtube-metadata-bulk.js" @@ -0,0 +1,2916 @@ +/** + * YouTube Metadata + * + * Grab everything publicly available from the YouTube API. + * + * @requires jquery + * @author mattwright324 + */ +const bulk = (function () { + 'use strict'; + + const elements = {}; + const controls = {}; + + const delaySubmitKey = "delaySubmitBulk"; + const can = { + submit: true, + }; + + const apiNextPageMs = 600; + const delay15Sec = 15; + const delay15SecMs = delay15Sec * 1000; + + function countdown(key, control, delay, flag) { + control.addClass("loading").addClass("disabled"); + can[flag] = false; + + let value = localStorage.getItem(key); + if (!moment(value).isValid()) { + console.warn('value for %s was not a valid date, resetting to now', key); + localStorage.setItem(key, new Date()); + value = localStorage.getItem(key); + } + if (moment(value).isAfter(moment())) { + console.warn('value for %s was set in the future, resetting to now', key); + localStorage.setItem(key, new Date()); + value = localStorage.getItem(key); + } + let count = (delay - moment().diff(value)) / 1000; + control.find(".countdown").text(Math.trunc(count)); + + function c(control, count) { + if (count <= 1) { + control.removeClass("loading").removeClass("disabled"); + control.find(".countdown").text(""); + + can[flag] = true; + } else { + control.find(".countdown").text(Math.trunc(count)); + setTimeout(function () { + c(control, count - 1) + }, 1000); + } + } + + setTimeout(function () { + c(control, count) + }, 1000); + } + + function countdownCheck(key, control, delayMs, flag) { + const value = localStorage.getItem(key); + if (key in localStorage && moment(value).isValid() && moment().diff(value) < delayMs) { + countdown(key, control, delayMs, flag); + } else { + control.removeClass("loading").removeClass("disabled"); + control.find(".countdown").text(""); + } + } + + function doneProgressMessage() { + const about = []; + if (rawVideoData.length) { + about.push(rawVideoData.length + " video(s)"); + } + if (Object.keys(unavailableData).length) { + about.push(Object.keys(unavailableData).length + " unavailable"); + } + if (Object.keys(failedData).length) { + about.push(Object.keys(failedData).length + " failed"); + } + if (Object.keys(rawChannelMap).length) { + about.push(Object.keys(rawChannelMap).length + " channel(s)"); + } + if (Object.keys(rawPlaylistMap).length) { + about.push(Object.keys(rawPlaylistMap).length + " playlist(s)"); + } + return about.join(", ") + } + + function unavailableProgressMessage() { + const about = []; + + if (Object.keys(unavailableData).length) { + about.push(Object.keys(unavailableData).length + " unavailable"); + + let noData = 0; + let filmot = 0; + for (let key in unavailableData) { + if (unavailableData[key].hasOwnProperty("filmot")) { + filmot = filmot + 1; + } else { + noData = noData + 1; + } + } + + if (filmot > 0) { + about.push(filmot + " data"); + } + if (noData > 0) { + about.push(noData + " no-data"); + } + } + + return about.join(", "); + } + + function processFromParsed(parsed) { + console.log(parsed); + + const channelUsers = []; + const channelHandles = []; + const channelCustoms = []; + const channelIds = []; + const channelIdsCreatedPlaylists = []; + const playlistIds = []; + const videoIds = []; + + parsed.forEach(function (p) { + if (p.type === 'video_id' && videoIds.indexOf(p.value) === -1) { + videoIds.push(p.value); + + controls.progress.update({ + text: videoIds.length + }); + } else if (p.type === "playlist_id" && playlistIds.indexOf(p.value) === -1) { + playlistIds.push(p.value); + } else if (p.type === "channel_id" && channelIds.indexOf(p.value) === -1 && shared.isValidChannelId(p.value)) { + channelIds.push(p.value); + } else if (p.type === "channel_handle" && channelHandles.indexOf(p.value) === -1) { + channelHandles.push(p.value); + } else if (p.type === "channel_custom" && channelCustoms.indexOf(p.value) === -1) { + channelCustoms.push(p.value); + } else if (p.type === "channel_user" && channelUsers.indexOf(p.value) === -1) { + channelUsers.push(p.value); + } + }); + + controls.progress.update({ + subtext: 'Grabbing unique video ids' + }); + + Promise.all([ + handleChannelCustoms(channelCustoms, channelIds), + handleChannelHandles(channelHandles, channelIds), + ]).then(function () { + return Promise.all([ + // Channels condense to uploads playlist ids and channel ids + handleChannelUsers(channelUsers, playlistIds, channelIdsCreatedPlaylists), + handleChannelIds(channelIds, playlistIds, channelIdsCreatedPlaylists) + ]); + }).then(function () { + // Created playlists condense to playlist ids (when option checked) + return handleChannelIdsCreatedPlaylists(channelIdsCreatedPlaylists, playlistIds); + }).then(function () { + // Grab playlist names + return handlePlaylistNames(playlistIds); + }).then(function () { + // Playlists condense to video ids + return handlePlaylistIds(playlistIds, videoIds); + }).then(function () { + controls.progress.update({ + subtext: 'Processing video ids' + }); + + // Videos are results to be displayed + return handleVideoIds(videoIds); + }).then(function () { + return new Promise(function (resolve) { + sliceLoad(rows, controls.videosTable, resolve); + }); + }).then(function () { + controls.progress.update({ + subtext: 'Processing channel ids' + }); + + // Ids for channels not in the original request, likely from playlists + const newChannelIds = []; + rawVideoData.forEach(function (video) { + const channelId = shared.idx(["snippet", "channelId"], video); + if (!rawChannelMap.hasOwnProperty(channelId) && newChannelIds.indexOf(channelId) === -1) { + newChannelIds.push(channelId); + } + }); + + return handleChannelIds(newChannelIds, [], []); + }).then(function () { + console.log(videoIds); + + const resultIds = []; + rawVideoData.forEach(function (video) { + resultIds.push(video.id); + }); + videoIds.forEach(function (videoId) { + if (!unavailableData.hasOwnProperty(videoId) && !failedData.hasOwnProperty(videoId) && resultIds.indexOf(videoId) === -1) { + unavailableData[videoId] = { + title: "Did not come back in API", + source: "" + }; + } + }); + + controls.videosTable.columns.adjust().draw(false); + }).then(function () { + controls.progress.update({ + text: doneProgressMessage(), + subtext: 'Done' + (Object.keys(failedData).length ? ' (with errors). Check browser console.' : '') + }); + + console.log(failedData) + + controls.unavailableTable.columns.adjust().draw(false); + + if (Object.keys(unavailableData).length > 0) { + controls.checkUnavailable.removeClass("disabled"); + } + + setTimeout(loadAggregateTables, 200); + }).catch(function (err) { + console.error(err); + }); + } + + function handleChannelUsers(channelUsers, playlistIds, channelIdsCreatedPlaylists) { + return new Promise(function (resolve) { + if (channelUsers.length === 0) { + console.log("no channelUsers") + resolve(); + return; + } + + function get(index) { + if (index >= channelUsers.length) { + console.log("finished channelUsers"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handleChannelUsers.get(" + index + ")") + console.log(channelUsers[index]) + + youtube.ajax("channels", { + part: "snippet,statistics,brandingSettings,contentDetails,localizations,status,topicDetails", + forUsername: channelUsers[index] + }).done(function (res) { + console.log(res); + + const channel = shared.idx(["items", 0], res); + if (!channel) { + get(index + 1); + return; + } + + const channelId = shared.idx(["id"], channel); + rawChannelMap[channelId] = channel; + + if (channelIdsCreatedPlaylists.indexOf(channelId) === -1) { + channelIdsCreatedPlaylists.push(channelId); + } + + const uploadsPlaylistId = shared.idx(["items", 0, "contentDetails", "relatedPlaylists", "uploads"], res); + console.log(uploadsPlaylistId); + + if (playlistIds.indexOf(uploadsPlaylistId) === -1) { + playlistIds.push(uploadsPlaylistId); + } + + get(index + 1); + }).fail(function (err) { + console.error(err); + get(index + 1); + }); + } + + get(0); + }); + } + + function handleChannelCustoms(channelCustoms, channelIds) { + return new Promise(function (resolve) { + if (channelCustoms.length === 0) { + console.log("no channelCustoms") + resolve(); + return; + } + + function get(index) { + if (index >= channelCustoms.length) { + console.log("finished channelCustoms"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handleChannelCustoms.get(" + index + ")") + + $.ajax({ + url: "https://cors-proxy-mw324.herokuapp.com/https://www.youtube.com/" + channelCustoms[index], + dataType: 'html' + }).then(function (res) { + const pageHtml = $("
").html(res); + const ogUrl = pageHtml.find("meta[property='og:url']").attr('content'); + console.log('Retrieved og:url ' + ogUrl); + + const newParsed = shared.determineInput(ogUrl); + if (newParsed.type === "channel_id") { + channelIds.push(newParsed.value); + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + } else { + console.log('Could not resolve custom url'); + console.warn(newParsed); + + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + } + }).fail(function (err) { + console.warn(err); + + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + }); + } + + get(0); + }); + } + + function handleChannelHandles(channelHandles, channelIds) { + return new Promise(function (resolve) { + if (channelHandles.length === 0) { + console.log("no channelHandles") + resolve(); + return; + } + + function get(index) { + if (index >= channelHandles.length) { + console.log("finished channelHandles"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handleChannelHandles.get(" + index + ")") + + $.ajax({ + url: "https://cors-proxy-mw324.herokuapp.com/https://www.youtube.com/@" + channelHandles[index], + dataType: 'html' + }).then(function (res) { + const pageHtml = $("
").html(res); + const channelId = pageHtml.find("meta[itemprop='channelId']").attr('content'); + const ogUrl = pageHtml.find("meta[property='og:url']").attr('content'); + const canonical = pageHtml.find("link[rel='canonical']").attr('href'); + + console.log('Retrieved [channelId=%s, ogUrl=%s, canonical=%s]', channelId, ogUrl, canonical); + + const newParsed = shared.determineInput(channelId || ogUrl || canonical); + if (newParsed.type === "channel_id") { + channelIds.push(newParsed.value); + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + } else { + console.log('Could not resolve handle'); + console.warn(newParsed); + + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + } + }).fail(function (err) { + console.warn(err); + + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + }); + } + + get(0); + }); + } + + function handleChannelIds(channelIds, playlistIds, channelIdsCreatedPlaylists) { + let processed = 0; + channelIds.forEach(function (channelId) { + if (channelIdsCreatedPlaylists.indexOf(channelId) === -1) { + channelIdsCreatedPlaylists.push(channelId); + } + }); + return new Promise(function (resolve) { + if (channelIds.length === 0) { + console.log("no channelIds") + resolve(); + return; + } + + controls.progress.update({ + value: 0, + max: channelIds.length, + text: "0 / " + channelIds.length + }); + + function get(index, slice) { + if (index >= channelIds.length) { + console.log("finished channelIds"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handleChannelIds.get(" + index + ", " + slice + ")") + + const ids = channelIds.slice(index, index + slice); + + youtube.ajax("channels", { + part: "snippet,statistics,brandingSettings,contentDetails,localizations,status,topicDetails", + id: ids.join(","), + maxResults: 50 + }).done(function (res) { + console.log(res); + + (res.items || []).forEach(function (channel) { + const channelId = shared.idx(["id"], channel); + rawChannelMap[channelId] = channel; + + const uploadsPlaylistId = shared.idx(["contentDetails", "relatedPlaylists", "uploads"], channel); + console.log(uploadsPlaylistId); + + if (playlistIds.indexOf(uploadsPlaylistId) === -1) { + playlistIds.push(uploadsPlaylistId); + } + }); + + processed = processed + ids.length; + + controls.progress.update({ + value: processed, + text: processed + " / " + channelIds.length + }); + + get(index + slice, slice); + }).fail(function (err) { + console.error(err); + get(index + slice, slice); + }); + } + + get(0, 50); + }); + } + + function handleChannelIdsCreatedPlaylists(channelIds, playlistIds) { + return new Promise(function (resolve) { + if (channelIds.length === 0) { + console.log("no handleChannelIdsCreatedPlaylists") + resolve(); + return; + } + + if (controls.createdPlaylists.is(":checked") === false) { + console.log("createdPlaylists not checked, skipping") + resolve(); + return; + } + + + function get(index) { + if (index >= channelIds.length) { + console.log("finished handleChannelIdsCreatedPlaylists"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handleChannelIdsCreatedPlaylists.get(" + index + ")") + + const id = channelIds[index]; + console.log(id); + + function paginate(pageToken) { + youtube.ajax("playlists", { + part: "snippet,status,localizations,contentDetails", + channelId: id, + maxResults: 50, + pageToken: pageToken + }).done(function (res) { + console.log(res); + + (res.items || []).forEach(function (playlist) { + const createdPlaylistId = shared.idx(["id"], playlist); + console.log(createdPlaylistId); + + rawPlaylistMap[createdPlaylistId] = playlist; + + playlistMap[createdPlaylistId] = shared.idx(["snippet", "title"], playlist); + + if (playlistIds.indexOf(createdPlaylistId) === -1) { + playlistIds.push(createdPlaylistId); + } + }); + + if (res.hasOwnProperty("nextPageToken")) { + paginate(res.nextPageToken); + } else { + get(index + 1); + } + }).fail(function (err) { + console.error(err); + get(index + 1); + }); + } + + paginate(""); + } + + get(0); + }); + } + + function handlePlaylistNames(playlistIds) { + return new Promise(function (resolve) { + if (playlistIds.length === 0) { + console.log("no playlistIds") + resolve(); + return; + } + + const notYetRetrieved = []; + playlistIds.forEach(function (id) { + if (!playlistMap.hasOwnProperty(id)) { + notYetRetrieved.push(id); + } + }); + console.log(notYetRetrieved); + + function get(index) { + if (index >= notYetRetrieved.length) { + console.log("finished notYetRetrieved"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handlePlaylistNames.get(" + index + ")") + + function paginate(pageToken) { + console.log(pageToken); + youtube.ajax("playlists", { + part: "snippet,status,localizations,contentDetails", + maxResults: 50, + id: notYetRetrieved[index], + pageToken: pageToken + }).done(function (res) { + console.log(res); + + (res.items || []).forEach(function (playlist) { + const playlistId = shared.idx(["id"], playlist); + console.log(playlistId); + + rawPlaylistMap[playlistId] = playlist; + playlistMap[playlistId] = shared.idx(["snippet", "title"], playlist); + }); + + if (res.hasOwnProperty("nextPageToken")) { + paginate(res.nextPageToken); + } else { + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + } + }).fail(function (err) { + console.error(err); + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + }); + } + + paginate(""); + } + + get(0); + }); + } + + function handlePlaylistIds(playlistIds, videoIds) { + return new Promise(function (resolve) { + if (playlistIds.length === 0) { + console.log("no playlistIds") + resolve(); + return; + } + + function get(index) { + if (index >= playlistIds.length) { + console.log("finished playlistIds"); + setTimeout(resolve, apiNextPageMs); + return; + } + + function paginate(pageToken) { + console.log("handlePlaylistIds.get(" + index + ")") + + youtube.ajax("playlistItems", { + part: "snippet", + maxResults: 50, + playlistId: playlistIds[index], + pageToken: pageToken + }).done(function (res) { + console.log(res); + + (res.items || []).forEach(function (video) { + const videoId = shared.idx(["snippet", "resourceId", "videoId"], video); + const videoOwnerChannelId = shared.idx(["snippet", "videoOwnerChannelId"], video); + const dateFormat = "YYYY-MM-DD"; + const dateAdded = shared.idx(["snippet", "publishedAt"], video); + + if (videoIds.indexOf(videoId) === -1) { + videoIds.push(videoId); + + availableData[videoId] = { + source: "Playlist: " + + "" + + playlistMap[playlistIds[index]] + + " (added " + moment(dateAdded).format(dateFormat) + ")" + } + } + if (!videoOwnerChannelId) { + unavailableData[videoId] = { + title: shared.idx(["snippet", "title"], video), + source: "Playlist: " + + "" + + playlistMap[playlistIds[index]] + + " (added " + moment(dateAdded).format(dateFormat) + ")" + } + } + }); + + controls.progress.update({ + text: videoIds.length + }) + + if (res.hasOwnProperty("nextPageToken")) { + setTimeout(function () { + paginate(res.nextPageToken); + }, apiNextPageMs); + } else { + setTimeout(function () { + get(index + 1); + }, apiNextPageMs); + } + }).fail(function (err) { + console.error(err); + setTimeout(function () { + get(index + 1); + }, 150); + }); + } + + paginate(""); + } + + get(0); + }); + } + + function handleUnavailableVideos() { + const videoIds = []; + for (let videoId in unavailableData) { + if (shared.isValidVideoId(videoId)) { + videoIds.push(videoId); + } else { + unavailableData[videoId].title = "Invalid ID"; + } + } + + let processed = 0; + return new Promise(function (resolve) { + const hostname = window.location.hostname; + if (hostname !== "localhost" && hostname !== "mattw.io") { + console.log("do not call filmot from other instances") + resolve(); + return; + } + + if (videoIds.length === 0) { + console.log("no videoIds") + resolve(); + return; + } + + console.log("checking " + videoIds.length + " videoIds"); + + function get(index, slice) { + if (index >= videoIds.length) { + console.log("finished videoIds"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handleUnavailableVideos.get(" + index + ", " + (index + slice) + ")") + + const ids = videoIds.slice(index, index + slice); + + // Note: Filmot has limited resources. Do not misuse, please contact about usage first. + // https://filmot.com/contactus + $.ajax({ + cache: false, + data: { + key: "md5paNgdbaeudounjp39", + id: ids.join(","), + flags: 1 // Get channel and description too, + }, + dataType: "json", + type: "GET", + timeout: 5000, + url: "https://filmot.com/api/getvideos", + }).done(function (res) { + console.log(res); + + res.forEach(function (video) { + unavailableData[video.id]["filmot"] = video; + }); + + processed = processed + ids.length; + + controls.unavailableProgress.update({ + value: processed, + max: videoIds.length, + text: processed + " / " + videoIds.length + }) + + setTimeout(function () { + get(index + slice, slice); + }, 1500); + }).fail(function (err) { + console.error(err); + setTimeout(function () { + get(index + slice, slice); + }, 1500); + }); + } + + get(0, 100); + }); + } + + function handleVideoIds(videoIds) { + let processed = 0; + + return new Promise(function (resolve) { + if (videoIds.length === 0) { + console.log("no videoIds") + resolve(); + return; + } + + console.log("checking " + videoIds.length + " videoIds"); + + function get(index, slice) { + if (index >= videoIds.length) { + console.log("finished videoIds"); + setTimeout(resolve, apiNextPageMs); + return; + } + + console.log("handleVideoIds.get(" + index + ", " + (index + slice) + ")") + + const ids = videoIds.slice(index, index + slice); + + console.log(ids.length); + console.log(ids); + + try { + youtube.ajax("videos", { + part: "snippet,statistics,recordingDetails," + + "status,liveStreamingDetails,localizations," + + "contentDetails,topicDetails", + maxResults: 50, + id: ids.join(",") + }).done(function (res) { + try { + console.log(res); + + (res.items || []).forEach(function (video) { + loadVideo(video, true); + }); + + processed = processed + ids.length; + + controls.progress.update({ + value: processed, + max: videoIds.length, + text: processed + " / " + videoIds.length + }); + + setTimeout(function () { + get(index + slice, slice); + }, apiNextPageMs); + } catch (error) { + controls.progress.addClass('error'); + console.error(error); + const reason = JSON.stringify(error, null, 0); + for (let i = 0; i < ids.length; i++) { + failedData[ids[i]] = {reason: reason} + } + setTimeout(function () { + get(index + slice, slice); + }, apiNextPageMs); + } + }).fail(function (err) { + controls.progress.addClass('error'); + console.warn(err) + const reason = shared.idx(["responseJSON", "error", "errors", 0, "reason"], err) || + JSON.stringify(err, null, 0); + for (let i = 0; i < ids.length; i++) { + failedData[ids[i]] = {reason: reason} + } + setTimeout(function () { + get(index + slice, slice); + }, apiNextPageMs); + }); + } catch (error) { + controls.progress.addClass('error'); + console.error(error); + const reason = JSON.stringify(error, null, 0); + for (let i = 0; i < ids.length; i++) { + failedData[ids[i]] = {reason: reason} + } + get(index + slice, slice); + } + } + + get(0, 50); + }); + } + + function loadVideo(video, skipAdd) { + const dataRow = []; + const csvDataRow = []; + const publishedAt = moment(shared.idx(["snippet", "publishedAt"], video)); + publishedAt.utc(); + + const tags = shared.idx(["snippet", "tags"], video); + (tags || []).forEach(function (tag) { + if (!tagsData[tag]) { + tagsData[tag] = {} + } + + tagsData[tag].count = ++tagsData[tag].count || 1; + + if (tagsData[tag].firstUsed) { + if (publishedAt.isBefore(tagsData[tag].firstUsed)) { + tagsData[tag].firstUsed = publishedAt; + tagsData[tag].firstVideo = video.id; + } + } else { + tagsData[tag].firstUsed = publishedAt; + tagsData[tag].firstVideo = video.id; + } + + if (tagsData[tag].lastUsed) { + if (publishedAt.isAfter(tagsData[tag].lastUsed)) { + tagsData[tag].lastUsed = publishedAt; + tagsData[tag].lastVideo = video.id; + } + } else { + tagsData[tag].lastUsed = publishedAt; + tagsData[tag].lastVideo = video.id; + } + }); + + const geotag = shared.idx(["recordingDetails"], video); + if (geotag.location) { + const latLng = geotag.location.latitude + "," + geotag.location.longitude; + const name = geotag.locationDescription; + if (geotagsData.hasOwnProperty(latLng)) { + const data = geotagsData[latLng]; + data.count = data.count + 1; + + if (data.names.indexOf(name) === -1) { + data.names.push(name); + } + } else { + geotagsData[latLng] = { + names: [name], + count: 1 + } + } + } + + const description = shared.idx(["snippet", "description"], video); + if (description) { + // https://stackoverflow.com/a/3809435/2650847 + //const URL_REGEX = /https?:\/\/(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9@:%_!+.~#?&/=]*)/gi; + //const matches = description.match(URL_REGEX); + const URL_XREGEX = XRegExp("(\\p{L}[\\p{L}\\d\\-+.]*)?:\\/\\/[-\\p{L}0-9@:%._+~#=]{1,256}\\.[\\p{L}0-9()]{1,6}\\b([-\\p{L}0-9@:%_!+.~#?&\\/=]*)", "gi"); + const matches = XRegExp.match(description, URL_XREGEX); + const END_CHARS = /[.()]*$/gi; // Remove periods and parenthesis on end of the link. + (matches || []).forEach(function (match) { + const link = match.replace(END_CHARS, ""); + if (!linksData[link]) { + linksData[link] = {} + } + linksData[link].count = ++linksData[link].count || 1; + + if (linksData[link].firstUsed) { + if (publishedAt.isBefore(linksData[link].firstUsed)) { + linksData[link].firstUsed = publishedAt; + linksData[link].firstVideo = video.id; + } + } else { + linksData[link].firstUsed = publishedAt; + linksData[link].firstVideo = video.id; + } + + if (linksData[link].lastUsed) { + if (publishedAt.isAfter(linksData[link].lastUsed)) { + linksData[link].lastUsed = publishedAt; + linksData[link].lastVideo = video.id; + } + } else { + linksData[link].lastUsed = publishedAt; + linksData[link].lastVideo = video.id; + } + }); + } + + for (let key in otherData) { + otherData[key].check(video); + } + + columns.forEach(function (column) { + const value = column._idx ? shared.idx(column._idx, video) : undefined; + const displayValue = column.hasOwnProperty("valueMod") ? column.valueMod(value, video) : value; + + dataRow.push(displayValue); + + const button = document.querySelector("button[title='" + column.title + "']"); + const input = document.querySelector("button[title='" + column.title + "'] input"); + if (!$(button).hasClass("active") && input.indeterminate === true && column._visibleIf && column._visibleIf(value)) { + button.click(); + } + + if (column.csvSkip) { + return; + } + + let csvValue = displayValue && displayValue.hasOwnProperty("display") ? + displayValue.display : displayValue; + + if (csvValue) { + csvValue = String(csvValue).replace(/([\r]?\n)/g, "
"); + } + + csvDataRow.push(csvValue); + }); + + tableRows.push(csvDataRow.join("\t")); + rows.push(dataRow); + rawVideoData.push(video); + + if (skipAdd) { + return dataRow; + } else { + controls.videosTable.row.add(dataRow).draw(false); + } + } + + function loadAggregateTables(callback) { + const dateFormat = "YYYY-MM-DD"; + + const tagRows = []; + for (let tag in tagsData) { + const tagData = tagsData[tag]; + tagRows.push([ + tag, + "" + tagData.firstUsed.format(dateFormat) + "", + "" + tagData.lastUsed.format(dateFormat) + "", + tagData.count + ]); + } + sliceLoad(tagRows, controls.tagsTable); + + const geotagRows = []; + for (let geotag in geotagsData) { + geotagRows.push([ + "" + geotag + "", + "" + geotagsData[geotag].names.join(", ") + "", + geotagsData[geotag].count + ]); + } + sliceLoad(geotagRows, controls.geotagsTable); + + const linksRows = []; + for (let link in linksData) { + const linkData = linksData[link]; + linksRows.push([ + link, + link.startsWith('http') ? "" + link + "" : link, + "" + linkData.firstUsed.format(dateFormat) + "", + "" + linkData.lastUsed.format(dateFormat) + "", + linkData.count + ]); + } + sliceLoad(linksRows, controls.linksTable); + + const timezoneOffset = controls.offset.val(); + const years = []; + const yearCount = {}; + rawVideoData.forEach(function (video) { + const timestamp = moment(shared.idx(["snippet", "publishedAt"], video)).utcOffset(String(timezoneOffset)); + const year = timestamp.format('yyyy'); + if (years.indexOf(year) === -1) { + years.push(year); + yearCount[year] = 1; + } else { + yearCount[year] = yearCount[year] + 1; + } + }); + years.sort(); + years.reverse(); + years.forEach(function (year) { + controls.year.append(""); + }); + loadChartData(timezoneOffset); + + console.log(unavailableData); + const unavailableRows = []; + for (let videoId in unavailableData) { + const video = unavailableData[videoId]; + const filmotTitle = shared.idx(["filmot", "title"], video) || "No data"; + const filmotDesc = shared.idx(["filmot", "description"], video) || ""; + const filmotAuthorName = shared.idx(["filmot", "channelname"], video); + const filmotAuthor = filmotAuthorName ? + "" + shared.idx(["filmot", "channelname"], video) + "" : ""; + const filmotUploadDate = shared.idx(["filmot", "uploaddate"], video) || ""; + const duration = shared.idx(["filmot", "duration"], video) || -1; + const filmotDuration = duration === -1 ? "" : shared.formatDuration(moment.duration({seconds: duration})); + unavailableRows.push([ + "" + + "youtube metadata icon" + + "", + "" + videoId + "", + String(video.title), + "Filmot · " + + "Archive Web · " + + "Archive Details · " + + "Archive Video · " + + "GhostArchive · " + + "Google", + video.source, + filmotTitle, + filmotAuthor, + filmotUploadDate, + {"display": filmotDuration, "num": duration}, + filmotDesc + ]); + } + sliceLoad(unavailableRows, controls.unavailableTable); + + unavailableColumns.forEach(function (column) { + const button = document.querySelector("button[title='" + column.title + "']"); + const input = document.querySelector("button[title='" + column.title + "'] input"); + if (!$(button).hasClass("active") && input.indeterminate === true && column._visibleIf && column._visibleIf(value)) { + button.click(); + } + }); + + controls.unavailableTable.columns.adjust(); + + console.log(otherData) + for (let key in otherData) { + const row = otherData[key]; + const value = row.value; + const displayValue = Number(value) === value ? Number(value).toLocaleString() : value; + + controls.otherTable.row.add([row.text, displayValue]).draw(false); + } + + if (callback) { + callback(); + } + } + + function loadChartData(timezoneOffset, yearFilter) { + if (rawVideoData.length === 0) { + return; + } + + console.log('Loading chart data [offset=' + timezoneOffset + ", yearFilter=" + yearFilter + "]") + + const days = ['Saturday', 'Friday', 'Thursday', 'Wednesday', 'Tuesday', 'Monday', 'Sunday'] + const rawChartData = {}; + days.forEach(function (dayName) { + rawChartData[dayName] = { + name: dayName, + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + }); + + rawVideoData.forEach(function (video) { + const timestamp = moment(shared.idx(["snippet", "publishedAt"], video)).utcOffset(String(timezoneOffset)); + const dayName = timestamp.format('dddd'); + const hour24 = timestamp.format('H'); + const year = timestamp.format('yyyy'); + + if (!yearFilter || yearFilter === "" || year === yearFilter) { + rawChartData[dayName].data[hour24] = rawChartData[dayName].data[hour24] + 1; + } + }); + + console.log(rawChartData); + + const newChartData = []; + days.forEach(function (dayName) { + for (let weekday in rawChartData) { + if (weekday === dayName) { + newChartData.push(rawChartData[weekday]); + chartData[rawChartData[weekday].name] = rawChartData[weekday].data; + } + } + }); + + controls.uploadFrequency.updateSeries(newChartData); + } + + function sliceLoad(data, table, callback) { + function slice(index, size) { + const toAdd = data.slice(index, index + size); + if (toAdd.length === 0) { + if (callback) { + callback(); + } + return; + } + + table.rows.add(toAdd).draw(false); + table.columns.adjust().draw(false); + + setTimeout(function () { + slice(index + size, size) + }, 200); + } + + slice(0, 1000); + } + + const columns = [ + { + title: " ", + type: "html", + visible: true, + valueMod: function (value, video) { + return "" + + "youtube metadata icon" + + "" + }, + csvSkip: true + }, + { + title: "Video ID", + type: "html", + visible: false, + _idx: ["id"], + valueMod: function (value, video) { + return "" + value + "" + } + }, + { + title: "Title", + type: "html", + visible: true, + _idx: ["snippet", "title"], + valueMod: function (value, video) { + return "" + value + "" + } + }, + { + title: "Channel ID", + type: "html", + visible: false, + _idx: ["snippet", "channelId"], + valueMod: function (value) { + return "" + value + "" + } + }, + { + title: "Author", + type: "html", + visible: true, + _idx: ["snippet", "channelTitle"], + valueMod: function (value, video) { + const channelId = shared.idx(["snippet", "channelId"], video); + return "" + value + "" + } + }, + { + title: "Description", + visible: false, + _idx: ["snippet", "description"] + }, + { + title: "Length", + type: "num", + visible: true, + _idx: ["contentDetails", "duration"], + valueMod: function (value) { + const duration = moment.duration(value); + const length = duration.asMilliseconds(); + + return length === 0 ? { + display: 'livestream', + num: length + } : { + display: shared.formatDuration(duration, false), + num: length + } + }, + render: { + _: 'display', + sort: 'num' + }, + className: "dt-nowrap" + }, + { + title: "Length (Seconds)", + type: "num", + visible: false, + _idx: ["contentDetails", "duration"], + valueMod: function (value) { + const duration = moment.duration(value); + const length = duration.asMilliseconds(); + + return length === 0 ? { + display: 'livestream', + num: length + } : { + display: length / 1000, + num: length + } + }, + render: { + _: 'display', + sort: 'num' + }, + className: "dt-nowrap" + }, + { + title: "Published", + type: "date", + visible: true, + _idx: ["snippet", "publishedAt"], + className: "dt-nowrap" + }, + { + title: "Language", + visible: false, + _idx: ["snippet", "defaultLanguage"] + }, + { + title: "Audio Language", + visible: false, + _idx: ["snippet", "defaultAudioLanguage"] + }, + { + title: "Views", + type: "num", + visible: true, + _idx: ["statistics", "viewCount"], + valueMod: function (value) { + return value ? { + display: Number(value).toLocaleString(), + num: value + } : { + display: "disabled", + num: -1 + }; + }, + render: { + _: 'display', + sort: 'num' + }, + className: "text-right dt-nowrap" + }, + { + title: "Likes", + type: "num", + visible: true, + _idx: ["statistics", "likeCount"], + valueMod: function (value) { + return value ? { + display: Number(value).toLocaleString(), + num: value + } : { + display: "disabled", + num: -1 + }; + }, + render: { + _: 'display', + sort: 'num' + }, + className: "text-right dt-nowrap" + }, + { + title: "Dislikes", + type: "num", + visible: false, + _visibleIf: function (value) { + return value; + }, + _idx: ["statistics", "dislikeCount"], + valueMod: function (value) { + return value ? { + display: Number(value).toLocaleString(), + num: value + } : { + display: "", + num: -1 + }; + }, + render: { + _: 'display', + sort: 'num' + }, + className: "text-right dt-nowrap" + }, + { + title: "Like Ratio", + type: "num", + visible: false, + _idx: ["statistics"], + valueMod: function (value) { + const likes = value.likeCount; + const dislikes = value.dislikeCount; + + if (likes && dislikes) { + const ratio = dislikes === 0 ? likes : likes / dislikes; + return { + display: Number(ratio).toLocaleString(), + num: ratio + } + } else { + return { + display: "", + num: -1 + } + } + }, + render: { + _: 'display', + sort: 'num' + }, + className: "text-right dt-nowrap" + }, + { + title: "Comments", + type: "num", + visible: true, + _idx: ["statistics", "commentCount"], + valueMod: function (value) { + return value ? { + display: Number(value).toLocaleString(), + num: value + } : { + display: "disabled", + num: "-1" + }; + }, + render: { + _: 'display', + sort: 'num' + }, + className: "text-right dt-nowrap" + }, + { + title: "Source", + type: "html", + visible: false, + valueMod: function (value, video) { + if (availableData.hasOwnProperty(video.id)) { + return availableData[video.id].source; + } else { + return "" + } + } + }, + { + title: "Kids", + type: "boolean", + visible: false, + _visibleIf: function (value) { + return value; + }, + _idx: ["status", "madeForKids"], + className: "text-center" + }, + { + title: "Embeddable", + type: "boolean", + visible: false, + _idx: ["status", "embeddable"], + className: "text-center" + }, + { + title: "Licensed Content", + type: "boolean", + visible: false, + _idx: ["contentDetails", "licensedContent"], + className: "text-center" + }, + { + title: "Dimension", + visible: false, + _idx: ["contentDetails", "dimension"] + }, + { + title: "Definition", + visible: false, + _idx: ["contentDetails", "definition"] + }, + { + title: "Caption", + visible: false, + _idx: ["contentDetails", "caption"] + }, + { + title: "Projection", + visible: false, + _idx: ["contentDetails", "projection"] + }, + { + title: "License", + visible: false, + _idx: ["status", "license"] + }, + { + title: "Privacy Status", + visible: false, + _visibleIf: function (value) { + return value && value !== "public"; + }, + _idx: ["status", "privacyStatus"] + }, + { + title: "Content Rating", + visible: false, + _idx: ["contentDetails", "contentRating"], + valueMod: function (value) { + if (!$.isEmptyObject(value)) { + console.log(value); + + const pairs = []; + Object.keys(value).forEach(function (key) { + pairs.push(key + "/" + value[key]); + }); + + return pairs.join(", "); + } + + return ""; + } + }, + { + title: "Region Restriction Count", + type: "num", + visible: false, + indeterminate: true, + _visibleIf: function (value) { + return !$.isEmptyObject(value) && (value.allowed || value.blocked).length > 0; + }, + _idx: ["contentDetails", "regionRestriction"], + valueMod: function (value) { + if (!$.isEmptyObject(value)) { + if (value.hasOwnProperty('allowed')) { + return { + display: value.allowed.length + " (allowed)", + num: value.allowed.length + }; + } else if (value.hasOwnProperty('blocked')) { + return { + display: value.blocked.length + " (blocked)", + num: value.blocked.length + }; + } + } + + return { + display: "", + num: 0 + }; + }, + render: { + _: 'display', + sort: 'num' + } + }, + { + title: "Region Restriction", + visible: false, + _idx: ["contentDetails", "regionRestriction"], + valueMod: function (value) { + if (!$.isEmptyObject(value)) { + console.log(value); + + if (value.hasOwnProperty('allowed')) { + value.allowed.sort(); + + return "Allowed [" + value.allowed.join(", ") + "]"; + } else if (value.hasOwnProperty('blocked')) { + value.blocked.sort(); + + return "Blocked [" + value.blocked.join(", ") + "]"; + } + } + + return ""; + } + }, + { + title: "Livestream", + type: "num", + visible: false, + _idx: ["liveStreamingDetails"], + valueMod: function (value) { + const is = value !== null; + + if (is) { + if (value.hasOwnProperty("actualEndTime")) { + return { + display: "true (ended)", + num: 1 + } + } else if (value.hasOwnProperty("actualStartTime")) { + return { + display: "true (on-going)", + num: 2 + } + } + + return { + display: "true (scheduled)", + num: 3 + } + } else { + return { + display: "", + num: 0 + } + } + }, + render: { + _: 'display', + sort: 'num' + } + }, + { + title: "Location Name", + type: "html", + visible: false, + _visibleIf: function (value) { + return !$.isEmptyObject(value) && value.locationDescription; + }, + _idx: ["recordingDetails"], + valueMod: function (value) { + if ($.isEmptyObject(value) || !value.locationDescription) { + return ""; + } + + if (!value.location) { + return value.locationDescription + } + + const latlng = value.location.latitude + "," + value.location.longitude; + + return "" + value.locationDescription + ""; + } + }, + { + title: "Location", + type: "html", + visible: false, + _visibleIf: function (value) { + // Display when there are coordinates but no location name, this sometimes happens and not sure how. + // return !$.isEmptyObject(value) && value.location && !value.locationDescription; + }, + _idx: ["recordingDetails"], + valueMod: function (value) { + if ($.isEmptyObject(value) || !value.location) { + return ""; + } + + const latlng = value.location.latitude + "," + value.location.longitude; + + return "" + latlng + ""; + } + }, + { + title: "Recording Date", + type: "date", + visible: false, + _idx: ["recordingDetails", "recordingDate"], + className: "dt-nowrap" + }, + { + title: "Tag Count", + type: "num", + visible: true, + _idx: ["snippet", "tags"], + valueMod: function (value) { + return value ? value.length : 0; + }, + className: "text-right dt-nowrap" + }, + { + title: "Tags", + visible: false, + _idx: ["snippet", "tags"], + valueMod: function (value) { + return value ? value.join(", ") : ""; + } + }, + { + title: "Localization Count", + type: "num", + visible: false, + _idx: ["localizations"], + valueMod: function (value) { + return value ? Object.keys(value).length : 0; + }, + className: "text-right dt-nowrap" + }, + { + title: "Localizations", + visible: false, + _idx: ["localizations"], + valueMod: function (value) { + return value ? Object.keys(value).sort().join(", ") : ""; + } + } + ]; + + const unavailableColumns = [ + { + title: " ", + type: "html", + visible: true + }, + { + title: "Video ID", + type: "html", + visible: true + }, + { + title: "Status", + type: "html", + visible: true + }, + { + title: "Research", + type: "html", + visible: true + }, + { + title: "Source", + type: "html", + visible: true + }, + { + title: "Title (Filmot)", + type: "html", + visible: false, + indeterminate: true, + _idx: ["filmot", "title"], + _visibleIf: function (value) { + return !$.isEmptyObject(value); + } + }, + { + title: "Author (Filmot)", + type: "html", + visible: false, + indeterminate: true, + _idx: ["filmot", "channelname"], + _visibleIf: function (value) { + return !$.isEmptyObject(value); + } + }, + { + title: "Published (Filmot)", + type: "html", + visible: false, + indeterminate: true, + _idx: ["filmot", "uploaddate"], + _visibleIf: function (value) { + return !$.isEmptyObject(value); + } + }, + { + title: "Length (Filmot)", + type: "num", + visible: false, + indeterminate: true, + _idx: ["filmot", "duration"], + _visibleIf: function (value) { + return !$.isEmptyObject(value); + }, + render: { + _: 'display', + sort: 'num' + }, + className: "dt-nowrap" + }, + { + title: "Description (Filmot)", + type: "html", + visible: false + } + ]; + + const csvHeaderRow = []; + for (let j = 0; j < columns.length; j++) { + const column = columns[j]; + if (column.csvSkip) { + continue; + } + csvHeaderRow.push(column.title); + } + + let tableRows = [csvHeaderRow.join("\t")]; + let rows = []; + let rawVideoData = []; + let availableData = {}; + let rawChannelMap = {}; + let rawPlaylistMap = {}; + let playlistMap = {}; + let unavailableData = {}; + let failedData = {}; // google requests failed, don't send to filmot + let tagsData = {}; + let geotagsData = {}; + let linksData = {}; + let chartData = {}; + let otherData = { + totalVideos: { + text: "Total videos", value: 0, check: function (video) { + if (video) { + this.value = this.value + 1; + } + } + }, + totalViews: { + text: "Total views", + value: 0, + check: function (video) { + const views = shared.idx(["statistics", "viewCount"], video); + this.value = this.value + (views ? Number(views) : 0); + } + }, + totalLikes: { + text: "Total likes", + value: 0, + check: function (video) { + const likes = shared.idx(["statistics", "likeCount"], video); + this.value = this.value + (likes ? Number(likes) : 0); + } + }, + totalDislikes: { + text: "Total dislikes", + value: 0, + check: function (video) { + const dislikes = shared.idx(["statistics", "dislikeCount"], video); + this.value = this.value + (dislikes ? Number(dislikes) : 0); + } + }, + totalComments: { + text: "Total comments", + value: 0, + check: function (video) { + const comments = shared.idx(["statistics", "commentCount"], video); + this.value = this.value + (comments ? Number(comments) : 0); + } + }, + totalLength: { + text: "Total video length", + value: '', + reset: function () { + this.totalSeconds = 0; + }, + totalSeconds: 0, + check: function (video) { + const duration = moment.duration(shared.idx(["contentDetails", "duration"], video)); + this.totalSeconds = this.totalSeconds + duration.asSeconds(); + this.value = shared.formatDuration(moment.duration({seconds: this.totalSeconds}), false); + } + }, + averageLength: { + text: "Average video length", + value: '', + check: function (video) { + const seconds = otherData.totalLength.totalSeconds / otherData.totalVideos.value; + this.value = shared.formatDuration(moment.duration({seconds: seconds}), false); + } + }, + withGeolocation: { + text: "Videos with geolocation", + value: 0, + check: function (video) { + const stat = shared.idx(["recordingDetails", "location", "latitude"], video); + if (stat) { + this.value = this.value + 1; + } + } + }, + withRecordingDate: { + text: "Videos with recordingDate", + value: 0, + check: function (video) { + const stat = shared.idx(["recordingDetails", "recordingDate"], video); + if (stat) { + this.value = this.value + 1; + } + } + }, + withTags: { + text: "Videos with tags", + value: 0, + check: function (video) { + const stat = shared.idx(["snippet", "tags"], video); + if (stat) { + this.value = this.value + 1; + } + } + }, + withLanguage: { + text: "Videos with language", + value: 0, + check: function (video) { + const stat = shared.idx(["snippet", "defaultLanguage"], video); + if (stat) { + this.value = this.value + 1; + } + } + }, + withAudioLanguage: { + text: "Videos with audio language", + value: 0, + check: function (video) { + const stat = shared.idx(["snippet", "defaultAudioLanguage"], video); + if (stat) { + this.value = this.value + 1; + } + } + }, + withLocalizations: { + text: "Videos with localizations", + value: 0, + check: function (video) { + const stat = shared.idx(["localizations"], video); + if (stat) { + this.value = this.value + 1; + } + } + }, + withContentRatings: { + text: "Videos with content rating(s)", + value: 0, + check: function (video) { + const stat = shared.idx(["contentDetails", "contentRating"], video); + if (!$.isEmptyObject(stat)) { + this.value = this.value + 1; + } + } + }, + withRegionRestrictions: { + text: "Videos with region restriction(s)", + value: 0, + check: function (video) { + const stat = shared.idx(["contentDetails", "regionRestriction"], video); + if (!$.isEmptyObject(stat)) { + this.value = this.value + 1; + } + } + }, + withCaptions: { + text: "Videos with captions", + value: 0, + check: function (video) { + const stat = shared.idx(["contentDetails", "caption"], video); + if (stat === "true") { + this.value = this.value + 1; + } + } + }, + withCommentsDisabled: { + text: "Videos with comments disabled", + value: 0, + check: function (video) { + const stat = shared.idx(["statistics", "commentCount"], video); + if (!stat) { + this.value = this.value + 1; + } + } + }, + withLikesDisabled: { + text: "Videos with likes disabled", + value: 0, + check: function (video) { + const stat = shared.idx(["statistics", "likeCount"], video); + if (!stat) { + this.value = this.value + 1; + } + } + }, + withViewsDisabled: { + text: "Videos with views disabled", + value: 0, + check: function (video) { + const stat = shared.idx(["statistics", "viewCount"], video); + if (!stat) { + this.value = this.value + 1; + } + } + }, + consideredLivestreams: { + text: "Videos considered livestreams", + value: 0, + check: function (video) { + const stat = shared.idx(["liveStreamingDetails"], video); + if (stat !== null) { + this.value = this.value + 1; + } + } + }, + isCreativeCommons: { + text: "Videos with license=creativeCommon", + value: 0, + check: function (video) { + const stat = shared.idx(["status", "license"], video); + if (stat === "creativeCommon") { + this.value = this.value + 1; + } + } + }, + isUnlisted: { + text: "Videos with privacyStatus=unlisted", + value: 0, + check: function (video) { + const stat = shared.idx(["status", "privacyStatus"], video); + if (stat === "unlisted") { + this.value = this.value + 1; + } + } + }, + isMadeForKids: { + text: "Videos with madeForKids=true", + value: 0, + check: function (video) { + const stat = shared.idx(["status", "madeForKids"], video); + if (stat === true) { + this.value = this.value + 1; + } + } + }, + isNotEmbeddable: { + text: "Videos with embeddable=false", + value: 0, + check: function (video) { + const stat = shared.idx(["status", "embeddable"], video); + if (stat === false) { + this.value = this.value + 1; + } + } + }, + is3d: { + text: "Videos with dimension=3d", + value: 0, + check: function (video) { + const stat = shared.idx(["contentDetails", "dimension"], video); + if (stat === "3d") { + this.value = this.value + 1; + } + } + }, + is360: { + text: "Videos with projection=360", + value: 0, + check: function (video) { + const stat = shared.idx(["contentDetails", "projection"], video); + if (stat === "360") { + this.value = this.value + 1; + } + } + } + }; + + const columnOptionsHtml = []; + for (let i = 0; i < columns.length; i++) { + const column = columns[i]; + + columnOptionsHtml.push("") + } + + const internal = { + init: function () { + controls.darkMode = $("#darkMode"); + controls.inputValue = $("#value"); + controls.inputValue.val(shared.randomFromList(EXAMPLE_BULK)); + controls.btnSubmit = $("#submit"); + controls.shareLink = $("#shareLink"); + controls.videosTable = $('#videosTable').DataTable({ + columns: columns, + columnDefs: [{ + "defaultContent": "", + "targets": "_all" + }], + order: [[8, 'desc']], + lengthMenu: [[10, 25, 50, 100, 250, -1], [10, 25, 50, 100, 250, "All"]], + deferRender: true, + bDeferRender: true, + + }); + controls.columnOptions = $("#column-options"); + controls.columnOptions.html(columnOptionsHtml.join("")); + controls.columnOptions.multiselect({ + buttonClass: 'form-select', + templates: { + button: '', + }, + onChange: function (option, checked, select) { + external.toggleResultsColumn($(option).val()) + } + }); + columns.forEach(function (column) { + if (column.hasOwnProperty("_visibleIf")) { + document.querySelector("button[title='" + column.title + "'] input").indeterminate = true; + } + }) + controls.progress = $("#progressBar"); + elements.progressText = $("#progressText") + controls.progress.progressData = { + min: 0, + value: 0, + max: 100 + } + controls.progress.update = function (options) { + console.log(options) + if (String(options["reset"]).toLowerCase() === "true") { + console.log('reset') + this.update({ + min: 0, + value: 0, + max: 100, + text: "", + subtext: 'Idle' + }); + return; + } + if (options.hasOwnProperty("subtext")) { + elements.progressText.text(options.subtext); + } + if (options.hasOwnProperty("text")) { + this.find('.label').text(options.text); + } + if (options.hasOwnProperty("min")) { + this.progressData.min = options.min; + } + if (options.hasOwnProperty("value")) { + this.progressData.value = options.value; + } + if (options.hasOwnProperty("max")) { + this.progressData.max = options.max; + } + + const data = this.progressData; + const percent = 100 * ((data.value - data.min) / (data.max - data.min)); + this.css('width', percent + "%"); + } + controls.unavailableProgress = $("#unavailableProgressBar"); + elements.unavailableProgressText = $("#unavailableProgressText") + controls.unavailableProgress.progressData = { + min: 0, + value: 0, + max: 100 + } + controls.unavailableProgress.update = function (options) { + console.log(options) + if (String(options["reset"]).toLowerCase() === "true") { + console.log('reset') + this.update({ + min: 0, + value: 0, + max: 100, + text: "", + subtext: 'Idle' + }); + return; + } + if (options.hasOwnProperty("subtext")) { + elements.unavailableProgressText.text(options.subtext); + } + if (options.hasOwnProperty("text")) { + this.find('.label').text(options.text); + } + if (options.hasOwnProperty("min")) { + this.progressData.min = options.min; + } + if (options.hasOwnProperty("value")) { + this.progressData.value = options.value; + } + if (options.hasOwnProperty("max")) { + this.progressData.max = options.max; + } + + const data = this.progressData; + const percent = 100 * ((data.value - data.min) / (data.max - data.min)); + this.css('width', percent + "%"); + } + controls.createdPlaylists = $("#createdPlaylists"); + controls.includeThumbs = $("#includeThumbs"); + elements.thumbProgress = $("#thumbProgress"); + + controls.tagsTable = $("#tagsTable").DataTable({ + columns: [ + {title: "Tag"}, + { + title: "First used", + type: 'datetime', + className: "dt-nowrap" + }, + { + title: "Last used", + type: 'datetime', + className: "dt-nowrap" + }, + { + title: "Count", + type: "num", + className: "text-right dt-nowrap" + } + ], + columnDefs: [{ + "defaultContent": "", + "targets": "_all" + }, { + "width": "100%", + "targets": 0 + }], + order: [[3, 'desc'], [0, 'asc']], + lengthMenu: [[10, 25, 50, 100, 250, -1], [10, 25, 50, 100, 250, "All"]], + deferRender: true, + bDeferRender: true + }); + controls.geotagsTable = $("#geotagsTable").DataTable({ + columns: [ + {title: "Coords"}, + {title: "Name(s)"}, + { + title: "Count", + type: "num", + className: "text-right dt-nowrap" + } + ], + columnDefs: [{ + "defaultContent": "", + "targets": "_all" + }], + order: [[2, 'desc'], [0, 'asc']], + lengthMenu: [[10, 25, 50, 100, 250, -1], [10, 25, 50, 100, 250, "All"]], + deferRender: true, + bDeferRender: true + }); + controls.linksTable = $("#linksTable").DataTable({ + columns: [ + { + title: "Link", + visible: true + }, + { + title: "Link (hyper)", + visible: false, + }, + { + title: "First used", + type: 'datetime', + className: "dt-nowrap" + }, + { + title: "Last used", + type: 'datetime', + className: "dt-nowrap" + }, + { + title: "Count", + type: "num", + className: "text-right dt-nowrap" + } + ], + columnDefs: [{ + "defaultContent": "", + "targets": "_all" + }, { + "width": "100%", + "className": "wrap", + "targets": [0, 1] + }], + order: [[4, 'desc'], [0, 'asc']], + lengthMenu: [[10, 25, 50, 100, 250, -1], [10, 25, 50, 100, 250, "All"]], + deferRender: true, + bDeferRender: true + }); + controls.offset = $("#offset"); + controls.offset.on('change', function () { + loadChartData(controls.offset.val(), controls.year.val()); + }); + controls.year = $("#year"); + controls.year.on('change', function () { + loadChartData(controls.offset.val(), controls.year.val()); + }); + const options = { + series: [ + { + name: 'Saturday', + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + { + name: 'Friday', + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + { + name: 'Thursday', + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + { + name: 'Wednesday', + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + { + name: 'Tuesday', + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + { + name: 'Monday', + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + { + name: 'Sunday', + data: [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + } + ], + xaxis: { + categories: [ + "12AM", "1AM", "2AM", "3AM", "4AM", "5AM", "6AM", "7AM", "8AM", "9AM", "10AM", "11AM", + "12PM", "1PM", "2PM", "3PM", "4PM", "5PM", "6PM", "7PM", "8PM", "9PM", "10PM", "11PM" + ] + }, + chart: { + height: 350, + type: 'heatmap', + }, + dataLabels: { + enabled: false + }, + colors: ["#008FFB"], + title: { + text: 'Day and Time Frequency' + }, + stroke: { + show: false + }, + grid: { + show: false + } + }; + controls.uploadFrequency = new ApexCharts(document.querySelector("#uploadFrequency"), options); + controls.uploadFrequency.render(); + controls.checkUnavailable = $("#checkUnavailable"); + + const unavailableColumnsHtml = []; + for (let i = 0; i < unavailableColumns.length; i++) { + const column = unavailableColumns[i]; + + unavailableColumnsHtml.push("") + } + controls.unavailableTable = $("#unavailableTable").DataTable({ + columns: unavailableColumns, + columnDefs: [{ + "defaultContent": "", + "targets": "_all" + }], + order: [[7, 'desc'], [4, 'asc']], + lengthMenu: [[10, 25, 50, 100, 250, -1], [10, 25, 50, 100, 250, "All"]], + deferRender: true, + bDeferRender: true + }); + controls.unavailableColumns = $("#unavailable-columns"); + controls.unavailableColumns.html(unavailableColumnsHtml.join("")); + controls.unavailableColumns.multiselect({ + buttonClass: 'form-select', + templates: { + button: '', + }, + onChange: function (option, checked, select) { + external.toggleUnavailableColumn($(option).val()) + } + }); + unavailableColumns.forEach(function (column) { + if (column.hasOwnProperty("_visibleIf")) { + document.querySelector("button[title='" + column.title + "'] input").indeterminate = true; + } + }) + + controls.otherTable = $("#otherTable").DataTable({ + columns: [ + {title: "Statistic"}, + { + title: "Value", + className: "text-right dt-nowrap" + } + ], + columnDefs: [{ + "defaultContent": "", + "targets": "_all" + }], + order: [], + lengthMenu: [[10, 25, 50, 100, 250, -1], [10, 25, 50, 100, 250, "All"]], + deferRender: true, + bDeferRender: true, + pageLength: -1 + }); + + controls.btnExport = $("#export"); + controls.btnImport = $("#import"); + controls.importFileChooser = $("#importFileChooser"); + + new ClipboardJS(".clipboard"); + + internal.buildPage(true); + }, + buildPage: function (doSetup) { + if (doSetup) { + internal.setupControls(); + } + }, + setupControls: function () { + function checkTheme() { + if (DarkMode.getColorScheme() === "dark") { + controls.uploadFrequency.updateOptions({ + theme: { + mode: 'dark' + } + }); + } else { + controls.uploadFrequency.updateOptions({ + theme: { + mode: 'light' + } + }); + } + } + + controls.darkMode.change(function () { + checkTheme(); + }); + checkTheme(); + + controls.progress.update({reset: true}); + + controls.inputValue.on('keypress', function (e) { + if (e.originalEvent.code === "Enter") { + controls.btnSubmit.click(); + } + }); + + countdownCheck(delaySubmitKey, controls.btnSubmit, delay15SecMs, "submit"); + + controls.btnSubmit.on('click', function () { + if (!can.submit) { + return; + } + localStorage.setItem(delaySubmitKey, new Date()); + countdownCheck(delaySubmitKey, controls.btnSubmit, delay15SecMs, "submit"); + + internal.reset(); + + const value = controls.inputValue.val(); + + const parsed = []; + value.split(/[,\s]+/g).forEach(function (valuePart) { + parsed.push(shared.determineInput(valuePart.trim())); + }); + if (parsed.length === 0) { + return; + } + + const checkCreatedPlaylists = controls.createdPlaylists.is(":checked"); + const optionalCreatedPlaylists = checkCreatedPlaylists ? "&createdPlaylists=true" : ""; + const minifiedInput = []; + parsed.forEach(function (input) { + if (input.type === "video_id" || input.type === "playlist_id" || input.type === "channel_id") { + minifiedInput.push(input.original); + } else { + minifiedInput.push(input.original); + } + }); + controls.shareLink.val(location.origin + location.pathname + "?url=" + encodeURIComponent(minifiedInput.join(",")) + optionalCreatedPlaylists + "&submit=true"); + controls.shareLink.attr("disabled", false); + + controls.progress.update({ + text: 'Indeterminate', + value: 100, + max: 100 + }); + + processFromParsed(parsed); + }); + + controls.checkUnavailable.on('click', function () { + controls.checkUnavailable.addClass("disabled"); + + if (Object.keys(unavailableData).length <= 0) { + return; + } + + controls.unavailableProgress.update({ + text: '0 / ' + Object.keys(unavailableData).length, + subtext: 'Processing unavailable ids', + value: 0, + max: Object.keys(unavailableData).length + }); + + handleUnavailableVideos().then(function () { + controls.unavailableTable.rows().every(function (rowIdx, tableLoop, rowLoop) { + const data = this.data(); + + const videoId = $(data[1]).text(); + const video = unavailableData[videoId]; + + const filmotTitle = shared.idx(["filmot", "title"], video) || "No data"; + const filmotDesc = shared.idx(["filmot", "description"], video) || ""; + const filmotAuthorName = shared.idx(["filmot", "channelname"], video); + const filmotAuthor = filmotAuthorName ? + "" + shared.idx(["filmot", "channelname"], video) + "" : ""; + const filmotUploadDate = shared.idx(["filmot", "uploaddate"], video) || ""; + const duration = shared.idx(["filmot", "duration"], video) || -1; + const filmotDuration = duration === -1 ? "" : shared.formatDuration(moment.duration({seconds: duration})); + data[5] = filmotTitle; + data[6] = filmotAuthor; + data[7] = filmotUploadDate; + data[8] = {"display": filmotDuration, "num": duration}; + data[9] = filmotDesc; + + this.data(data).draw(); + }); + + controls.unavailableTable.columns.adjust(); + + controls.unavailableProgress.update({ + text: unavailableProgressMessage(), + subtext: 'Done' + }); + }); + }); + + function getImageBinaryCorsProxy(fileName, imageUrl, zip, delay, imageStatuses) { + return new Promise(function (resolve) { + setTimeout(function () { + // CORS proxy workaround for downloading YouTube thumbnails in client-side app + // https://github.com/Rob--W/cors-anywhere/issues/301#issuecomment-962623118 + console.log('Attempting to download image over CORS proxy (' + delay + ' ms start delay): ' + imageUrl); + const start = new Date(); + JSZipUtils.getBinaryContent("https://cors-proxy-mw324.herokuapp.com/" + imageUrl, function (err, data) { + const ms = new Date() - start; + + if (err) { + console.log('Failed ' + fileName + " (" + ms + "ms)"); + console.warn("Could not get image: " + imageUrl) + console.warn(err); + imageStatuses[false] = imageStatuses[false] + 1 || 1; + } else { + console.log('Retrieved ' + fileName + " (" + ms + "ms)"); + console.log("Creating " + fileName + "..."); + zip.folder('thumbs').file(fileName, data, {binary: true}); + imageStatuses[true] = imageStatuses[true] + 1 || 1; + } + + elements.thumbProgress.text("(" + imageStatuses[true] + " downloaded / " + imageStatuses[false] + " failed)"); + + resolve(); + }); + }, delay); + }); + } + + controls.btnExport.on('click', function () { + const includeThumbs = controls.includeThumbs.is(":checked"); + const dateFormat = "YYYY-MM-DD"; + controls.btnExport.addClass("loading").addClass("disabled"); + + const zip = new JSZip(); + console.log("Creating about.txt...") + zip.file("about.txt", + "Downloaded by YouTube Metadata " + new Date().toLocaleString() + "\n\n" + + "URL: " + window.location + "\n\n" + + "Input: " + controls.inputValue.val() + "\n\n" + + "Created Playlists: " + controls.createdPlaylists.is(":checked") + ); + + console.log("Creating videos.json...") + zip.file("videos.json", JSON.stringify(rawVideoData)); + + console.log("Creating available.json...") + zip.file("available.json", JSON.stringify(availableData)); + + console.log("Creating channels.json...") + const rawChannelData = []; + for (let id in rawChannelMap) { + rawChannelData.push(rawChannelMap[id]); + } + zip.file("channels.json", JSON.stringify(rawChannelData)); + + console.log("Creating playlists.json...") + const rawPlaylistData = []; + for (let id in rawPlaylistMap) { + rawPlaylistData.push(rawPlaylistMap[id]); + } + zip.file("playlists.json", JSON.stringify(rawPlaylistData)); + + console.log("Creating unavailable.json...") + zip.file("unavailable.json", JSON.stringify(unavailableData)); + + console.log("Creating videos.csv...") + zip.file("videos.csv", tableRows.join("\r\n")); + + console.log("Creating tags.csv...") + const tagCsvRows = ["Tag\tCount\tFirst used\tFirst video\tLast used\tLast video"]; + for (let tag in tagsData) { + const tagData = tagsData[tag]; + tagCsvRows.push(tag + "\t" + tagData.count + "\t" + tagData.firstUsed.format(dateFormat) + "\t" + tagData.firstVideo + "\t" + tagData.lastUsed.format(dateFormat) + "\t" + tagData.lastVideo); + } + zip.file("tags.csv", tagCsvRows.join("\r\n")); + + console.log("Creating geotags.csv...") + const geotagCsvRows = ["Coords\tName(s)\tCount"]; + for (let geotag in geotagsData) { + const tag = geotagsData[geotag]; + geotagCsvRows.push(geotag + "\t" + tag.names.join(", ") + "\t" + tag.count); + } + zip.file("geotags.csv", geotagCsvRows.join("\r\n")); + + console.log("Creating links.csv...") + const linkCsvRows = ["Link\tCount\tFirst used\tFirst video\tLast used\tLast video"]; + for (let link in linksData) { + const linkData = linksData[link]; + linkCsvRows.push(link + "\t" + linkData.count + "\t" + linkData.firstUsed.format(dateFormat) + "\t" + linkData.firstVideo + "\t" + linkData.lastUsed.format(dateFormat) + "\t" + linkData.lastVideo); + } + zip.file("links.csv", linkCsvRows.join("\r\n")); + + console.log("Creating frequency.csv...") + const frequencyCsvRows = ["Weekday\t12AM\t1AM\t2AM\t3AM\t4AM\t5AM\t6AM\t7AM\t8AM\t9AM\t10AM\t11AM\t12PM\t1PM\t2PM\t3PM\t4PM\t5PM\t6PM\t7PM\t8PM\t9PM\t10PM\t11PM"]; + for (let row in chartData) { + const data = chartData[row]; + frequencyCsvRows.push(row + "\t" + data.join("\t")); + } + zip.file("frequency.csv", frequencyCsvRows.join("\r\n")); + + console.log("Creating other.csv...") + const otherCsvRows = ["Statistic\tValue"]; + for (let key in otherData) { + const row = otherData[key]; + const statistic = row.text; + const rowValue = row.value; + const displayValue = Number(rowValue) === rowValue ? Number(rowValue).toLocaleString() : rowValue; + otherCsvRows.push(statistic + "\t" + displayValue); + } + zip.file("other.csv", otherCsvRows.join("\r\n")); + + const optionalImages = []; + const imageStatuses = {true: 0, false: 0}; + if (includeThumbs) { + let delay = 0; + for (let i = 0; i < rawVideoData.length; i++) { + const video = rawVideoData[i]; + const fileName = video.id + ".png"; + const thumbs = shared.idx(["snippet", "thumbnails"], video) || {}; + const thumbUrl = (thumbs.maxres || thumbs.high || thumbs.medium || thumbs.default || {url: null}).url; + if (thumbUrl) { + optionalImages.push(getImageBinaryCorsProxy(fileName, thumbUrl, zip, delay * 100, imageStatuses)); + } + delay++; + } + for (let id in rawPlaylistMap) { + console.log(id); + const playlist = rawPlaylistMap[id]; + const fileName = id + ".png"; + const thumbs = shared.idx(["snippet", "thumbnails"], playlist) || {}; + const thumbUrl = (thumbs.maxres || thumbs.high || thumbs.medium || thumbs.default || {url: null}).url; + if (thumbUrl) { + optionalImages.push(getImageBinaryCorsProxy(fileName, thumbUrl, zip, delay * 100, imageStatuses)); + } + delay++; + } + for (let id in rawChannelMap) { + console.log(id); + const channel = rawChannelMap[id]; + const fileName = id + ".png"; + const thumbs = shared.idx(["snippet", "thumbnails"], channel) || {}; + const thumbUrl = (thumbs.maxres || thumbs.high || thumbs.medium || thumbs.default || {url: null}).url; + if (thumbUrl) { + optionalImages.push(getImageBinaryCorsProxy(fileName, thumbUrl, zip, delay * 100, imageStatuses)); + } + delay++; + const bannerFileName = id + "-banner.png"; + const bannerUrl = shared.idx(["brandingSettings", "image", "bannerExternalUrl"], channel); + if (bannerUrl) { + optionalImages.push(getImageBinaryCorsProxy(bannerFileName, bannerUrl, zip, delay * 100, imageStatuses)); + } + delay++; + } + } + + Promise.all(optionalImages).then(function () { + const channelTitles = []; + rawVideoData.forEach(function (video) { + const channelTitle = shared.idx(["snippet", "channelTitle"], video) + if (channelTitles.indexOf(channelTitle) === -1) { + channelTitles.push(channelTitle); + } + }); + + console.log(channelTitles) + let hint = ''; + if (channelTitles.length === 0) { + hint = ' (none)' + } else if (channelTitles.length === 1) { + hint = ' (' + channelTitles[0].substr(0, 15) + ")" + } else { + hint = ' (' + channelTitles[0].substr(0, 15) + " and " + (channelTitles.length - 1) + " others)" + } + + const fileName = shared.safeFileName("bulk_metadata" + hint + ".zip"); + + console.log("Saving as " + fileName); + zip.generateAsync({ + type: "blob", + compression: "DEFLATE", + compressionOptions: { + level: 9 + } + }).then(function (content) { + saveAs(content, fileName); + + controls.btnExport.removeClass("loading").removeClass("disabled"); + }); + }); + }); + + // Drag & Drop listener + document.addEventListener("dragover", function (event) { + event.preventDefault(); + }); + document.documentElement.addEventListener('drop', async function (e) { + e.stopPropagation(); + e.preventDefault(); + + let file = e.dataTransfer.files[0]; + if (file) { + controls.inputValue.val(file.name); + } else { + return; + } + + importFile(file); + }); + + controls.importFileChooser.on('change', function (event) { + console.log(event); + + let file = event.target.files[0]; + if (file) { + controls.inputValue.val(file.name); + } else { + return; + } + + importFile(file); + }); + + function importFile(file) { + console.log("Importing from file " + file.name); + + controls.btnImport.addClass("loading").addClass("disabled"); + + internal.reset(); + controls.progress.update({ + text: '', + subtext: 'Importing file', + value: 2, + max: 5 + }); + + function loadZipFile(fileName, process, onfail) { + return new Promise(function (resolve, reject) { + console.log('loading ' + fileName); + + JSZip.loadAsync(file).then(function (content) { + // if you return a promise in a "then", you will chain the two promises + return content.file(fileName).async("string"); + }).then(function (text) { + process(text); + + resolve(); + }).catch(function (err) { + console.warn(err); + console.warn(fileName + ' not in imported file'); + if (onfail) { + onfail() + reject() + } else { + resolve() + } + }); + }) + } + + loadZipFile('available.json', function (text) { + availableData = JSON.parse(text); + }).then(function () { + return Promise.all([ + loadZipFile('unavailable.json', function (text) { + unavailableData = JSON.parse(text); + }), + loadZipFile('playlists.json', function (text) { + rawPlaylistMap = JSON.parse(text); + }), + loadZipFile('channels.json', function (text) { + const channels = JSON.parse(text); + if (Array.isArray(channels)) { + channels.forEach(function (channel) { + rawChannelMap[channel.id] = channel; + }) + } else { + rawChannelMap = channels; + } + }), + loadZipFile('videos.json', function (text) { + const rows = []; + (JSON.parse(text) || []).forEach(function (video) { + rows.push(loadVideo(video, true)); + }); + + sliceLoad(rows, controls.videosTable); + }, function () { + controls.progress.update({ + value: 0, + max: 5, + subtext: 'Import failed (no videos.json)' + }); + }) + ]); + }).then(function () { + loadAggregateTables(function () { + controls.btnImport.removeClass("loading").removeClass("disabled"); + controls.progress.update({ + value: 5, + max: 5, + text: rows.length + " / " + rows.length + }); + }); + + controls.progress.update({ + value: 5, + max: 5, + text: doneProgressMessage(), + subtext: 'Import done' + }); + + controls.unavailableProgress.update({ + value: 5, + max: 5, + text: unavailableProgressMessage(), + subtext: 'Import done' + }); + + controls.btnImport.removeClass("loading").removeClass("disabled"); + }).catch(function () { + controls.btnImport.removeClass("loading").removeClass("disabled"); + }) + } + + const query = shared.parseQuery(window.location.search); + console.log(query); + if (String(query["searchMode"]).toLowerCase() === "true") { + elements.regularInput.attr("hidden", true); + elements.searchInput.attr("hidden", false); + $("#formatShare").hide(); + } + const input = query.url || query.id; + if (input) { + controls.inputValue.val(decodeURIComponent(input)); + } + if (String(query["createdPlaylists"]).toLowerCase() === "true") { + controls.createdPlaylists.prop("checked", true); + } else { + controls.createdPlaylists.prop("checked", false); + } + if (String(query["submit"]).toLowerCase() === "true") { + setTimeout(function () { + controls.btnSubmit.click(); + }, 500); + } + }, + reset: function () { + controls.progress.update({reset: true}); + controls.progress.removeClass('error'); + controls.checkUnavailable.addClass("disabled"); + + rows = []; + tableRows = [csvHeaderRow.join("\t")]; + rawVideoData = []; + availableData = {}; + rawChannelMap = {}; + rawPlaylistMap = {}; + failedData = {}; + controls.videosTable.clear(); + controls.videosTable.draw(false); + + columns.forEach(function (column) { + const button = document.querySelector("button[title='" + column.title + "']"); + const input = document.querySelector("button[title='" + column.title + "'] input"); + + // Reset still-indeterminate columns + if (input.indeterminate === true && button.className.indexOf('active') !== -1) { + button.click(); + } + }); + + tagsData = {}; + controls.tagsTable.clear(); + controls.tagsTable.draw(false); + + geotagsData = {}; + controls.geotagsTable.clear(); + controls.geotagsTable.draw(false); + + linksData = {}; + controls.linksTable.clear(); + controls.linksTable.draw(false); + + unavailableData = {}; + playlistMap = {}; + controls.unavailableTable.clear(); + controls.unavailableTable.draw(false); + controls.unavailableProgress.update({reset: true}); + + unavailableColumns.forEach(function (column) { + const button = document.querySelector("button[title='" + column.title + "']"); + const input = document.querySelector("button[title='" + column.title + "'] input"); + + // Reset still-indeterminate columns + if (input.indeterminate === true && button.className.indexOf('active') !== -1) { + button.click(); + } + }); + + controls.year.html("") + + for (let key in otherData) { + const row = otherData[key]; + if (row.hasOwnProperty('reset')) { + row.reset(); + } else { + row.value = 0; + } + } + controls.otherTable.clear(); + controls.otherTable.draw(false); + } + } + + $(document).ready(internal.init); + + const external = { + toggleResultsColumn(index) { + const column = controls.videosTable.column(index); + + column.visible(!column.visible()); + }, + toggleLinksColumn(index) { + console.log('toggle links ' + index) + const column = controls.linksTable.column(index); + + column.visible(!column.visible()); + }, + toggleUnavailableColumn(index) { + const column = controls.unavailableTable.column(index); + + column.visible(!column.visible()); + } + } + return external; +}());