Upload 11 files

clewd.js

@@ -7,7 +7,7 @@
 const {createServer: Server, IncomingMessage, ServerResponse} = require('node:http'), {createHash: Hash, randomUUID, randomInt, randomBytes} = require('node:crypto'), {TransformStream, ReadableStream} = require('node:stream/web'), {Readable, Writable} = require('node:stream'), {Blob} = require('node:buffer'), {existsSync: exists, writeFileSync: write, createWriteStream} = require('node:fs'), {join: joinP} = require('node:path'), {ClewdSuperfetch: Superfetch, SuperfetchAvailable} = require('./lib/clewd-superfetch'), {AI, fileName, genericFixes, bytesToSize, setTitle, checkResErr, Replacements, Main} = require('./lib/clewd-utils'), ClewdStream = require('./lib/clewd-stream');
 
 /******************************************************* */
-let currentIndex, Firstlogin = true, changeflag = 0, changetime = 0, totaltime, uuidOrgArray = [], model;
+let currentIndex, Firstlogin = true, changeflag = 0, changetime = 0, totaltime, uuidOrgArray = [], model, tokens, apiKey;
 
 const events = require('events'), CookieChanger = new events.EventEmitter();
 require('events').EventEmitter.defaultMaxListeners = 0;
@@ -22,20 +22,21 @@ CookieChanger.on('ChangeCookie', () => {
     }));
 });
 
-const CookieCleaner = () => {
+const convertToType = value => {
+    if (value === "true") return true;
+    if (value === "false") return false;
+    if (/^\d+$/.test(value)) return parseInt(value);
+    return value;
+}, CookieCleaner = () => {
     Config.CookieArray = Config.CookieArray.filter(item => item !== Config.Cookie);
     !process.env.Cookie && !process.env.CookieArray && writeSettings(Config);
     currentIndex = (currentIndex - 1 + Config.CookieArray.length) % Config.CookieArray.length;
 }, padtxt = content => {
-    const {encode} = require('gpt-tokenizer');
+    const {countTokens} = require('@anthropic-ai/tokenizer');
     const placeholder = Config.padtxt_placeholder || randomBytes(randomInt(5, 15)).toString('hex');
-    let count = Math.floor(Math.max(1000, Config.Settings.padtxt - encode(content).length) / encode(placeholder).length); 
-    let padding = '';
-    for (let i = 0; i < count; i++) {
-        padding += placeholder;
-    }
-    content = padding + '\n\n\n' + content;
-    return content.trim();
+    tokens = countTokens(content);
+    !apiKey && (content = placeholder.repeat(Math.floor(Math.max(1000, Config.Settings.padtxt - tokens) / countTokens(placeholder.trim()))) + '\n\n\n' + content.trim());
+    return content;
 }, xmlPlot = content => {
     // 检查内容中是否包含"<card>"
     const card = content.includes('<card>');
@@ -53,13 +54,14 @@ const CookieCleaner = () => {
     const MergeHumanDisable = content.includes('<\!-- Merge Human Disable -->');
     const MergeAssistantDisable = content.includes('<\!-- Merge Assistant Disable -->');
     if (!MergeDisable) {
-        if (content.includes('<\!-- Merge System Disable -->') || !card) {
+        if (content.includes('<\!-- Merge System Disable -->')) {
             content = content.replace(/(\n\n|^\s*)xmlPlot:\s*/gm, '$1');
         }
         if (!MergeHumanDisable) {
+            const Human = /^\s*Human:/.test(content);
             content = content.replace(/(\n\n|^\s*)xmlPlot:/g, '$1Human:');
             content = content.replace(/(?:\n\n|^\s*)Human:(.*?(?:\n\nAssistant:|$))/gs, function(match, p1) {return '\n\nHuman:' + p1.replace(/\n\nHuman:\s*/g, '\n\n')});
-            content = content.replace(/^\s*Human:\s*/, '');
+            !apiKey && !Human && (content = content.replace(/^\s*Human:\s*/, ''));
         }
         if (!MergeAssistantDisable) {
             content = content.replace(/\n\nAssistant:(.*?(?:\n\nHuman:|$))/gs, function(match, p1) {return '\n\nAssistant:' + p1.replace(/\n\nAssistant:\s*/g, '\n\n')});
@@ -130,7 +132,7 @@ const CookieCleaner = () => {
     content = content.replace(/(?<=(: |\n)<(card|hidden|example|plot|META|EOT)>\n)\s*/g, '');
     content = content.replace(/\s*(?=\n<\/(card|hidden|example|plot|META|EOT)>(\n|$))/g, '');
     content = content.replace(/(?<=\n)\n(?=\n)/g, '');
-    return content.trim();
+    return content.trim().replace(/^\s*Human:/, '\n\nHuman:');
 };
 /******************************************************* */
 
@@ -154,6 +156,7 @@ let uuidOrg, curPrompt = {}, prevPrompt = {}, prevMessages = [], prevImpersonate
     BufferSize: 1,
     SystemInterval: 3,
     rProxy: AI.end(),
+    api_rProxy: '',
     padtxt_placeholder: '',
     PromptExperimentFirst: '',
     PromptExperimentNext: '',
@@ -258,6 +261,7 @@ const updateParams = res => {
         console.log(`\n※※※Cookie cleanup completed※※※\n\n`);
         return process.exit();
     }
+    try {
 /***************************** */
     if ('SET YOUR COOKIE HERE' === Config.Cookie || Config.Cookie?.length < 1) {
         throw Error('Set your cookie inside config.js');
@@ -278,8 +282,7 @@ const updateParams = res => {
         CookieCleaner();
         console.log(`Expired!`);
         Config.Cookiecounter < 0 && console.log(`[progress]: ${percentage.toFixed(2)}%\n[length]: ${Config.CookieArray.length}\n`);
-        CookieChanger.emit('ChangeCookie');
-        return;
+        return CookieChanger.emit('ChangeCookie');
     }
 /**************************** */
     await checkResErr(accRes);
@@ -301,7 +304,7 @@ const updateParams = res => {
         }
     });
     await checkResErr(accountRes);
-    const accountInfo = (await accountRes.json());
+    const accountInfo = await accountRes.json();
 /**************************** */
     console.log(Config.CookieArray?.length > 0 ? `(index: ${currentIndex || Config.CookieArray.length}) Logged in %o` : 'Logged in %o', { //console.log('Logged in %o', {
         name: accInfo.name?.split('@')?.[0],
@@ -371,11 +374,8 @@ const updateParams = res => {
             return CookieChanger.emit('ChangeCookie');
         }
     }
-    if (Config.Cookiecounter < 0) {
-        console.log(`[progress]: ${percentage.toFixed(2)}%\n[length]: ${Config.CookieArray.length}\n`);
-        return CookieChanger.emit('ChangeCookie');
-    } else if (Exceededlimit) {
-        console.log(`Exceeded limit!\n`);
+    if (Config.Cookiecounter < 0 || Exceededlimit) {
+        console.log(Config.Cookiecounter < 0 ? `[progress]: ${percentage.toFixed(2)}%\n[length]: ${Config.CookieArray.length}\n` : 'Exceeded limit!\n');
         return CookieChanger.emit('ChangeCookie');
 /***************************** */
     }
@@ -388,6 +388,12 @@ const updateParams = res => {
     }), conversations = await convRes.json();
     updateParams(convRes);
     conversations.length > 0 && await Promise.all(conversations.map((conv => deleteChat(conv.uuid))));
+/***************************** */
+    } catch (err) {
+        console.error('Clewd:\n%o', err);
+        Config.CookieArray?.length > 0 && CookieChanger.emit('ChangeCookie');
+    }
+/***************************** */
 }, writeSettings = async (config, firstRun = false) => {
     write(ConfigPath, `/*\n* https://rentry.org/teralomaniac_clewd\n* https://github.com/teralomaniac/clewd\n*/\n\n// SET YOUR COOKIE BELOW\n\nmodule.exports = ${JSON.stringify(config, null, 4)}\n\n/*\n BufferSize\n * How many characters will be buffered before the AI types once\n * lower = less chance of \`PreventImperson\` working properly\n\n ---\n\n SystemInterval\n * How many messages until \`SystemExperiments alternates\`\n\n ---\n\n Other settings\n * https://gitgud.io/ahsk/clewd/#defaults\n * and\n * https://gitgud.io/ahsk/clewd/-/blob/master/CHANGELOG.md\n */`.trim().replace(/((?<!\r)\n|\r(?!\n))/g, '\r\n'));
     if (firstRun) {
@@ -408,7 +414,18 @@ const updateParams = res => {
       case '/v1/models':
         res.json({
             data: [ {
-                id: AI.mdl()
+/***************************** */
+                id: 'claude-2.1'                },{
+                id: 'claude-2.0'                },{
+                id: 'claude-v1.3'               },{
+                id: 'claude-v1.3-100k'          },{
+                id: 'claude-v1.2'               },{
+                id: 'claude-v1.0'               },{
+                id: 'claude-instant-1.2'        },{
+                id: 'claude-instant-v1.1'       },{
+                id: 'claude-instant-v1.1-100k'  },{
+                id: 'claude-instant-v1.0'       //id: AI.mdl()
+/***************************** */
             } ]
         });
         break;
@@ -416,7 +433,7 @@ const updateParams = res => {
       case '/v1/chat/completions':
         ((req, res) => {
             setTitle('recv...');
-            let fetchAPI, changer; //let fetchAPI;
+            let fetchAPI;
             const abortControl = new AbortController, {signal} = abortControl;
             res.socket.on('close', (async () => {
                 abortControl.signal.aborted || abortControl.abort();
@@ -431,8 +448,13 @@ const updateParams = res => {
                     const body = JSON.parse(Buffer.concat(buffer).toString()), temperature = Math.max(.1, Math.min(1, body.temperature));
                     let {messages} = body;
 /************************* */
-                    const authorization = Object.fromEntries(Object.entries(req.headers).map(([key, value]) => [key, value])).authorization;
-                    if (Config.ProxyPassword != '' && authorization != 'Bearer ' + Config.ProxyPassword) {
+                    apiKey = /(?<=^Bearer \s*)sk-ant-api[\w-]*(?=\s*)$/.exec(req.headers.authorization);
+                    let api_max_tokens, api_model;
+                    if (apiKey) {
+                        apiKey = apiKey[0];
+                        api_max_tokens = body.max_tokens;
+                        api_model = body.model;
+                    } else if (Config.ProxyPassword != '' && req.headers.authorization != 'Bearer ' + Config.ProxyPassword) {
                         throw Error('ProxyPassword Wrong');
                     }
 /************************* */
@@ -491,7 +513,7 @@ const updateParams = res => {
                     retryRegen = Config.Settings.RetryRegenerate && samePrompt && null != Conversation.uuid;
                     samePrompt || (prevMessages = JSON.parse(JSON.stringify(messages)));
                     let type = '';
-                    if (retryRegen) {
+                    if (apiKey) { type = 'api'; } else if (retryRegen) { //if (retryRegen) {
                         type = 'R';
                         fetchAPI = await (async (signal, model) => {
                             let res;
@@ -565,16 +587,16 @@ const updateParams = res => {
                             if (next && !Config.Settings.xmlPlot) { //if (next) {
                                 if ('name' in message && 'name' in next) {
                                     if (message.name === next.name) {
-                                        message.content += '\n\n' + next.content; //message.content += '\n' + next.content;
+                                        message.content += '\n' + next.content;
                                         next.merged = true;
                                     }
                                 } else if ('system' !== next.role) {
                                     if (next.role === message.role) {
-                                        message.content += '\n\n' + next.content; //message.content += '\n' + next.content;
+                                        message.content += '\n' + next.content;
                                         next.merged = true;
                                     }
                                 } else {
-                                    message.content += '\n\n' + next.content; //message.content += '\n' + next.content;
+                                    message.content += '\n' + next.content;
                                     next.merged = true;
                                 }
                             }
@@ -622,7 +644,7 @@ const updateParams = res => {
                             message.customname || delete message.name;
                         }));
                         let systems = [];
-                        if (![ 'r', 'R' ].includes(type)) {
+                        if (![ 'r', 'R', 'api' ].includes(type)) {
                             lastUser.strip = true;
                             systemMessages.forEach((message => message.discard = message.discard || 'c-c' === type ? !message.jailbreak : !message.jailbreak && !message.main));
                             systems = systemMessages.filter((message => !message.discard)).map((message => `"${message.content.substring(0, 25).replace(/\n/g, '\\n').trim()}..."`));
@@ -649,25 +671,45 @@ const updateParams = res => {
                             } //
                         }));
                         return {
-                            prompt: prompt.join('').trim(),//genericFixes(prompt.join('')).trim(),
+                            prompt: prompt.join('').trim(), //genericFixes(prompt.join('')).trim(),
                             systems
                         };
                     })(messages, type);
-                    console.log(`${model} [${type}]${!retryRegen && systems.length > 0 ? ' ' + systems.join(' / ') : ''}`);
+                    console.log(`${apiKey ? api_model : model} [${type}]${!retryRegen && systems.length > 0 ? ' ' + systems.join(' / ') : ''}`); //console.log(`${model} [${type}]${!retryRegen && systems.length > 0 ? ' ' + systems.join(' / ') : ''}`);
                     'R' !== type || prompt || (prompt = '...regen...');
 /******************************** */
                     prompt = Config.Settings.xmlPlot ? xmlPlot(prompt) : genericFixes(prompt);
-                    Config.Settings.FullColon && (prompt = prompt.replace(/(?<=\n\n(H(?:uman)?|A(?:ssistant)?)):[ ]?/g, '： '));
+                    !apiKey && Config.Settings.FullColon && (prompt = prompt.replace(/(?<=\n\n(H(?:uman)?|A(?:ssistant)?)):[ ]?/g, '： '));
                     Config.Settings.padtxt && (prompt = padtxt(prompt));
 /******************************** */
-                    Logger?.write(`\n\n-------\n[${(new Date).toLocaleString()}]\n####### PROMPT (${type}):\n${prompt}\n--\n####### REPLY:\n`);
+                    Logger?.write(`\n\n-------\n[${(new Date).toLocaleString()}]\n####### PROMPT (${type}):\n${prompt}\n--\n####### [Tokens: ${tokens}] REPLY:\n`); //Logger?.write(`\n\n-------\n[${(new Date).toLocaleString()}]\n####### PROMPT (${type}):\n${prompt}\n--\n####### REPLY:\n`);
                     retryRegen || (fetchAPI = await (async (signal, model, prompt, temperature, type) => {
-                        const attachments = [];
-                        if (Config.Settings.PromptExperiments) {
 /******************************** */
-                            let splitedprompt = prompt.split('\n\nPlainPrompt:');
-                            prompt = splitedprompt[0];
+                        if (apiKey) {
+                            const res = await fetch(`${Config.api_rProxy ? Config.api_rProxy : 'https://api.anthropic.com'}/v1/complete`, {
+                                method: 'POST',
+                                signal,
+                                headers: {
+                                    'Content-Type': 'application/json',
+                                    'x-api-key': apiKey,
+                                    'anthropic-version': '2023-06-01'
+                                },
+                                body: JSON.stringify({
+                                    model: api_model,
+                                    max_tokens_to_sample: api_max_tokens,
+                                    stream: true,
+                                    prompt,
+                                    temperature
+                                }),
+                            });
+                            await checkResErr(res);
+                            return res;
+                        }
 /******************************** */
+                        const attachments = [];
+                        if (Config.Settings.PromptExperiments) {
+                            let splitedprompt = prompt.split('\n\nPlainPrompt:'); //
+                            prompt = splitedprompt[0]; //
                             attachments.push({
                                 extracted_content: (prompt),
                                 file_name: 'paste.txt',  //fileName(),
@@ -675,9 +717,7 @@ const updateParams = res => {
                                 file_type: 'txt'  //'text/plain'
                             });
                             prompt = 'r' === type ? Config.PromptExperimentFirst : Config.PromptExperimentNext;
-/******************************** */
-                            splitedprompt.length > 1 && (prompt = prompt + splitedprompt[1]);
-/******************************** */
+                            splitedprompt.length > 1 && (prompt += splitedprompt[1]); //
                         }
                         let res;
                         const body = {
@@ -712,7 +752,13 @@ const updateParams = res => {
                     })(signal, model, prompt, temperature, type));
                     const response = Writable.toWeb(res);
                     clewdStream = new ClewdStream({
-                        config: Config,
+                        config: {
+                            ...Config,
+                            Settings: {
+                                ...Config.Settings,
+                                Superfetch: apiKey ? false : Config.Settings.Superfetch
+                            }
+                        }, //config: Config,
                         version: Main,
                         minSize: Config.BufferSize,
                         model,
@@ -721,7 +767,7 @@ const updateParams = res => {
                         source: fetchAPI
                     }, Logger);
                     titleTimer = setInterval((() => setTitle('recv ' + bytesToSize(clewdStream.size))), 300);
-                    Config.Settings.Superfetch ? await Readable.toWeb(fetchAPI.body).pipeThrough(clewdStream).pipeTo(response) : await fetchAPI.body.pipeThrough(clewdStream).pipeTo(response);
+                    (!apiKey && Config.Settings.Superfetch) ? await Readable.toWeb(fetchAPI.body).pipeThrough(clewdStream).pipeTo(response) : await fetchAPI.body.pipeThrough(clewdStream).pipeTo(response); //Config.Settings.Superfetch ? await Readable.toWeb(fetchAPI.body).pipeThrough(clewdStream).pipeTo(response) : await fetchAPI.body.pipeThrough(clewdStream).pipeTo(response);
                 } catch (err) {
                     if ('AbortError' === err.name) {
                         res.end();
@@ -743,22 +789,18 @@ const updateParams = res => {
                     prevImpersonated = clewdStream.impersonated;
                     setTitle('ok ' + bytesToSize(clewdStream.size));
                     429 == fetchAPI.status ? console.log(`Exceeded limit!\n`) : console.log(`${200 == fetchAPI.status ? '' : ''}${fetchAPI.status}!\n`); //console.log(`${200 == fetchAPI.status ? '' : ''}${fetchAPI.status}!\n`);
+                    clewdStream.empty();
+                }
+                if (!apiKey) { //if (prevImpersonated) {
+                    await deleteChat(Conversation.uuid);
 /******************************** */
                     changeflag += 1;
-                    if (Config.CookieArray?.length > 0 && (429 == fetchAPI.status || (Config.Cookiecounter && changeflag >= Config.Cookiecounter))) {
+                    if (Config.CookieArray?.length > 0 && (429 == fetchAPI.status || Config.Cookiecounter && changeflag >= Config.Cookiecounter)) {
                         changeflag = 0;
-                        changer = true;
+                        CookieChanger.emit('ChangeCookie');
                     }
 /******************************** */
-                    clewdStream.empty();
                 }
-                /*if (prevImpersonated) {
-                    await deleteChat(Conversation.uuid);
-                }*/
-/******************************** */
-                await deleteChat(Conversation.uuid);
-                changer && CookieChanger.emit('ChangeCookie');
-/******************************** */
             }));
         })(req, res);
         break;
@@ -819,38 +861,21 @@ const updateParams = res => {
             Config.Cookie = 'SET YOUR COOKIE HERE';
             writeSettings(Config, true);
         }
-/***************************** */
-        function convertToType(value) {
-            if (value === "true") return true;
-            if (value === "false") return false;
-            if (/^\d+$/.test(value)) return parseInt(value);
-            return value;
-        }
-        for (let key in Config) {
-            if (key === 'Settings') {
-                for (let setting in Config.Settings) {
-                    Config.Settings[setting] = convertToType(process.env[setting]) ?? Config.Settings[setting];
-                }
-            } else {
-                Config[key] = key === 'CookieArray' ? (process.env[key]?.split(',')?.map(x => x.replace(/[\[\]"\s]/g, '')) ?? Config[key]) : (convertToType(process.env[key]) ?? Config[key]);
-            }
-        }
-/***************************** */
     })();
 /***************************** */
-    !Config.rProxy && (Config.rProxy = AI.end());
-    Config.rProxy.endsWith('/') && (Config.rProxy = Config.rProxy.slice(0, -1));
-    let uniqueArr = [], seen = new Set();
-    for (let Cookie of Config.CookieArray) {
-        !/^sessionKey=/.test(Cookie) && (Cookie += 'sessionKey=');
-        if (!seen.has(Cookie)) {
-            uniqueArr.push(Cookie);
-            seen.add(Cookie);
+    for (let key in Config) {
+        if (key === 'Settings') {
+            for (let setting in Config.Settings) {
+                Config.Settings[setting] = convertToType(process.env[setting]) ?? Config.Settings[setting];
+            }
+        } else {
+            Config[key] = key === 'CookieArray' ? (process.env[key]?.split(',')?.map(x => x.replace(/[\[\]"\s]/g, '')) ?? Config[key]) : (convertToType(process.env[key]) ?? Config[key]);
         }
     }
-    Config.CookieArray = uniqueArr;
+    Config.rProxy = Config.rProxy.endsWith('/') ? Config.rProxy.slice(0, -1) : Config.rProxy || AI.end();
+    Config.CookieArray = [...new Set(Config.CookieArray)];
     !process.env.Cookie && !process.env.CookieArray && writeSettings(Config);
-    currentIndex = Config.CookieIndex > 0 ? Config.CookieIndex - 1 : Config.Cookiecounter >= 0 ? Math.floor(Math.random()*Config.CookieArray.length) : 0;
+    currentIndex = Config.CookieIndex > 0 ? Config.CookieIndex - 1 : Config.Cookiecounter >= 0 ? Math.floor(Math.random() * Config.CookieArray.length) : 0;
 /***************************** */
     Proxy.listen(Config.Port, Config.Ip, onListen);
     Proxy.on('error', (err => {

lib/clewd-stream.js

@@ -131,7 +131,7 @@ class ClewdStream extends TransformStream {
         } else {
             this.#recvLength += Buffer.byteLength(chunk);
         }
-        this.#compRaw += chunk;
+        this.#compRaw += chunk.replace(/event: (completion|ping)\s*|\r/gi,''); //this.#compRaw += chunk;
         const substr = this.#compRaw.split('\n\n'), lastMsg = substr.length - 1;
         0 !== substr[lastMsg].length ? this.#compRaw = substr[lastMsg] : this.#compRaw = '';
         for (let i = 0; i < lastMsg; i++) {

lib/clewd-utils.js

@@ -5,7 +5,7 @@
 'use strict';
 
 const {randomInt, randomBytes} = require('node:crypto'), {version: Version} = require('../package.json'), Encoder = (new TextDecoder, 
-new TextEncoder), Main = 'clewd修改版 v' + Version + '(10) by tera', Replacements = {
+new TextEncoder), Main = 'clewd修改版 v' + Version + '(11) by tera', Replacements = {
     user: 'Human',
     assistant: 'Assistant',
     system: '',

package.json

@@ -37,6 +37,6 @@
   },
   "dependencies": {
     "localtunnel": "^2.0.2",
-    "gpt-tokenizer": "^2.1.2"
+    "@anthropic-ai/tokenizer": "^0.0.4"
   }
 }