Spaces:
Sleeping
Sleeping
Sasha commited on
Commit ·
0b30711
1
Parent(s): 7347031
fix: add pagination to unpaginated Supabase queries to bypass 1000 row limit
Browse files- server/db.js +79 -35
server/db.js
CHANGED
|
@@ -886,20 +886,38 @@ export async function getChatMessages(streamId = null) {
|
|
| 886 |
return sqliteDb.prepare('SELECT message, username, display_name, timestamp FROM messages WHERE username NOT IN (?, ?)').all(...bots);
|
| 887 |
}
|
| 888 |
} else {
|
| 889 |
-
let
|
| 890 |
-
|
| 891 |
-
|
| 892 |
-
|
| 893 |
-
|
| 894 |
-
|
| 895 |
-
|
| 896 |
-
|
| 897 |
-
|
| 898 |
-
|
| 899 |
-
|
| 900 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 901 |
}
|
| 902 |
-
|
|
|
|
| 903 |
}
|
| 904 |
}
|
| 905 |
|
|
@@ -978,17 +996,32 @@ export async function getStreamMessageTimestamps(streamId) {
|
|
| 978 |
if (dbMode === 'sqlite') {
|
| 979 |
return sqliteDb.prepare('SELECT timestamp FROM messages WHERE stream_id = ? ORDER BY timestamp ASC').all(streamId);
|
| 980 |
} else {
|
| 981 |
-
|
| 982 |
-
|
| 983 |
-
|
| 984 |
-
.eq('stream_id', streamId)
|
| 985 |
-
.order('timestamp', { ascending: true });
|
| 986 |
|
| 987 |
-
|
| 988 |
-
|
| 989 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 990 |
}
|
| 991 |
-
return
|
| 992 |
}
|
| 993 |
}
|
| 994 |
|
|
@@ -1003,20 +1036,31 @@ export async function getModActionsSummaryRaw(streamId = null) {
|
|
| 1003 |
return sqliteDb.prepare('SELECT moderator, action_type FROM mod_actions').all();
|
| 1004 |
}
|
| 1005 |
} else {
|
| 1006 |
-
let
|
| 1007 |
-
|
| 1008 |
-
|
| 1009 |
-
|
| 1010 |
-
|
| 1011 |
-
query =
|
| 1012 |
-
|
| 1013 |
-
|
| 1014 |
-
|
| 1015 |
-
|
| 1016 |
-
|
| 1017 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1018 |
}
|
| 1019 |
-
return
|
| 1020 |
}
|
| 1021 |
}
|
| 1022 |
|
|
|
|
| 886 |
return sqliteDb.prepare('SELECT message, username, display_name, timestamp FROM messages WHERE username NOT IN (?, ?)').all(...bots);
|
| 887 |
}
|
| 888 |
} else {
|
| 889 |
+
let allData = [];
|
| 890 |
+
let from = 0;
|
| 891 |
+
const limit = 1000;
|
| 892 |
+
|
| 893 |
+
while(true) {
|
| 894 |
+
let query = supabase
|
| 895 |
+
.from('messages')
|
| 896 |
+
.select('message, username, display_name, timestamp')
|
| 897 |
+
.not('username', 'in', `(${bots.map(b => `"${b}"`).join(',')})`)
|
| 898 |
+
.range(from, from + limit - 1);
|
| 899 |
+
|
| 900 |
+
if (streamId) {
|
| 901 |
+
query = query.eq('stream_id', streamId);
|
| 902 |
+
}
|
| 903 |
+
|
| 904 |
+
const { data, error } = await query;
|
| 905 |
+
|
| 906 |
+
if (error) {
|
| 907 |
+
console.error('[Supabase] Error getting chat messages:', error);
|
| 908 |
+
break;
|
| 909 |
+
}
|
| 910 |
+
|
| 911 |
+
if (data && data.length > 0) {
|
| 912 |
+
allData = allData.concat(data);
|
| 913 |
+
if (data.length < limit) break; // Fetched the last incomplete page
|
| 914 |
+
from += limit;
|
| 915 |
+
} else {
|
| 916 |
+
break; // Empty page
|
| 917 |
+
}
|
| 918 |
}
|
| 919 |
+
|
| 920 |
+
return allData;
|
| 921 |
}
|
| 922 |
}
|
| 923 |
|
|
|
|
| 996 |
if (dbMode === 'sqlite') {
|
| 997 |
return sqliteDb.prepare('SELECT timestamp FROM messages WHERE stream_id = ? ORDER BY timestamp ASC').all(streamId);
|
| 998 |
} else {
|
| 999 |
+
let allData = [];
|
| 1000 |
+
let from = 0;
|
| 1001 |
+
const limit = 1000;
|
|
|
|
|
|
|
| 1002 |
|
| 1003 |
+
while(true) {
|
| 1004 |
+
const { data, error } = await supabase
|
| 1005 |
+
.from('messages')
|
| 1006 |
+
.select('timestamp')
|
| 1007 |
+
.eq('stream_id', streamId)
|
| 1008 |
+
.order('timestamp', { ascending: true })
|
| 1009 |
+
.range(from, from + limit - 1);
|
| 1010 |
+
|
| 1011 |
+
if (error) {
|
| 1012 |
+
console.error('[Supabase] Error fetching message timestamps:', error);
|
| 1013 |
+
break;
|
| 1014 |
+
}
|
| 1015 |
+
|
| 1016 |
+
if (data && data.length > 0) {
|
| 1017 |
+
allData = allData.concat(data);
|
| 1018 |
+
if (data.length < limit) break;
|
| 1019 |
+
from += limit;
|
| 1020 |
+
} else {
|
| 1021 |
+
break;
|
| 1022 |
+
}
|
| 1023 |
}
|
| 1024 |
+
return allData;
|
| 1025 |
}
|
| 1026 |
}
|
| 1027 |
|
|
|
|
| 1036 |
return sqliteDb.prepare('SELECT moderator, action_type FROM mod_actions').all();
|
| 1037 |
}
|
| 1038 |
} else {
|
| 1039 |
+
let allData = [];
|
| 1040 |
+
let from = 0;
|
| 1041 |
+
const limit = 1000;
|
| 1042 |
+
|
| 1043 |
+
while(true) {
|
| 1044 |
+
let query = supabase.from('mod_actions').select('moderator, action_type').range(from, from + limit - 1);
|
| 1045 |
+
if (streamId) {
|
| 1046 |
+
query = query.eq('stream_id', streamId);
|
| 1047 |
+
}
|
| 1048 |
+
|
| 1049 |
+
const { data, error } = await query;
|
| 1050 |
+
if (error) {
|
| 1051 |
+
console.error('[Supabase] Error getting mod actions summary:', error);
|
| 1052 |
+
break;
|
| 1053 |
+
}
|
| 1054 |
+
|
| 1055 |
+
if (data && data.length > 0) {
|
| 1056 |
+
allData = allData.concat(data);
|
| 1057 |
+
if (data.length < limit) break;
|
| 1058 |
+
from += limit;
|
| 1059 |
+
} else {
|
| 1060 |
+
break;
|
| 1061 |
+
}
|
| 1062 |
}
|
| 1063 |
+
return allData;
|
| 1064 |
}
|
| 1065 |
}
|
| 1066 |
|