File size: 3,618 Bytes
f8b5d42 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | const fs = require("fs");
const path = require("path");
const { default: slugify } = require("slugify");
const { v4 } = require("uuid");
const {
writeToServerDocuments,
sanitizeFileName,
documentsFolder,
} = require("../../files");
const { tokenizeString } = require("../../tokenizer");
const { YoutubeLoader } = require("./YoutubeLoader");
function validYoutubeVideoUrl(link) {
const UrlPattern = require("url-pattern");
const opts = new URL(link);
const url = `${opts.protocol}//${opts.host}${opts.pathname}${
opts.searchParams.has("v") ? `?v=${opts.searchParams.get("v")}` : ""
}`;
const shortPatternMatch = new UrlPattern(
"https\\://(www.)youtu.be/(:videoId)"
).match(url);
const fullPatternMatch = new UrlPattern(
"https\\://(www.)youtube.com/watch?v=(:videoId)"
).match(url);
const videoId =
shortPatternMatch?.videoId || fullPatternMatch?.videoId || null;
if (!!videoId) return true;
return false;
}
async function fetchVideoTranscriptContent({ url }) {
if (!validYoutubeVideoUrl(url)) {
return {
success: false,
reason: "Invalid URL. Should be youtu.be or youtube.com/watch.",
content: null,
metadata: {},
};
}
console.log(`-- Working YouTube ${url} --`);
const loader = YoutubeLoader.createFromUrl(url, { addVideoInfo: true });
const { docs, error } = await loader
.load()
.then((docs) => {
return { docs, error: null };
})
.catch((e) => {
return {
docs: [],
error: e.message?.split("Error:")?.[1] || e.message,
};
});
if (!docs.length || !!error) {
return {
success: false,
reason: error ?? "No transcript found for that YouTube video.",
content: null,
metadata: {},
};
}
const metadata = docs[0].metadata;
const content = docs[0].pageContent;
if (!content.length) {
return {
success: false,
reason: "No transcript could be parsed for that YouTube video.",
content: null,
metadata: {},
};
}
return {
success: true,
reason: null,
content,
metadata,
};
}
async function loadYouTubeTranscript({ url }) {
const transcriptResults = await fetchVideoTranscriptContent({ url });
if (!transcriptResults.success) {
return {
success: false,
reason:
transcriptResults.reason ||
"An unknown error occurred during transcription retrieval",
};
}
const { content, metadata } = transcriptResults;
const outFolder = sanitizeFileName(
slugify(`${metadata.author} YouTube transcripts`).toLowerCase()
);
const outFolderPath = path.resolve(documentsFolder, outFolder);
if (!fs.existsSync(outFolderPath))
fs.mkdirSync(outFolderPath, { recursive: true });
const data = {
id: v4(),
url: url + ".youtube",
title: metadata.title || url,
docAuthor: metadata.author,
description: metadata.description,
docSource: url,
chunkSource: `youtube://${url}`,
published: new Date().toLocaleString(),
wordCount: content.split(" ").length,
pageContent: content,
token_count_estimate: tokenizeString(content),
};
console.log(`[YouTube Loader]: Saving ${metadata.title} to ${outFolder}`);
writeToServerDocuments({
data,
filename: sanitizeFileName(`${slugify(metadata.title)}-${data.id}`),
destinationOverride: outFolderPath,
});
return {
success: true,
reason: "test",
data: {
title: metadata.title,
author: metadata.author,
destination: outFolder,
},
};
}
module.exports = {
loadYouTubeTranscript,
fetchVideoTranscriptContent,
};
|