Sasha commited on
Commit
0b30711
·
1 Parent(s): 7347031

fix: add pagination to unpaginated Supabase queries to bypass 1000 row limit

Browse files
Files changed (1) hide show
  1. server/db.js +79 -35
server/db.js CHANGED
@@ -886,20 +886,38 @@ export async function getChatMessages(streamId = null) {
886
  return sqliteDb.prepare('SELECT message, username, display_name, timestamp FROM messages WHERE username NOT IN (?, ?)').all(...bots);
887
  }
888
  } else {
889
- let query = supabase
890
- .from('messages')
891
- .select('message, username, display_name, timestamp')
892
- .not('username', 'in', `(${bots.map(b => `"${b}"`).join(',')})`);
893
-
894
- if (streamId) {
895
- query = query.eq('stream_id', streamId);
896
- }
897
- const { data, error } = await query;
898
- if (error) {
899
- console.error('[Supabase] Error getting chat messages:', error);
900
- return [];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
901
  }
902
- return data || [];
 
903
  }
904
  }
905
 
@@ -978,17 +996,32 @@ export async function getStreamMessageTimestamps(streamId) {
978
  if (dbMode === 'sqlite') {
979
  return sqliteDb.prepare('SELECT timestamp FROM messages WHERE stream_id = ? ORDER BY timestamp ASC').all(streamId);
980
  } else {
981
- const { data, error } = await supabase
982
- .from('messages')
983
- .select('timestamp')
984
- .eq('stream_id', streamId)
985
- .order('timestamp', { ascending: true });
986
 
987
- if (error) {
988
- console.error('[Supabase] Error fetching message timestamps:', error);
989
- return [];
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
990
  }
991
- return data || [];
992
  }
993
  }
994
 
@@ -1003,20 +1036,31 @@ export async function getModActionsSummaryRaw(streamId = null) {
1003
  return sqliteDb.prepare('SELECT moderator, action_type FROM mod_actions').all();
1004
  }
1005
  } else {
1006
- let query = supabase
1007
- .from('mod_actions')
1008
- .select('moderator, action_type');
1009
-
1010
- if (streamId) {
1011
- query = query.eq('stream_id', streamId);
1012
- }
1013
-
1014
- const { data, error } = await query;
1015
- if (error) {
1016
- console.error('[Supabase] Error fetching mod actions summary:', error);
1017
- return [];
 
 
 
 
 
 
 
 
 
 
 
1018
  }
1019
- return data || [];
1020
  }
1021
  }
1022
 
 
886
  return sqliteDb.prepare('SELECT message, username, display_name, timestamp FROM messages WHERE username NOT IN (?, ?)').all(...bots);
887
  }
888
  } else {
889
+ let allData = [];
890
+ let from = 0;
891
+ const limit = 1000;
892
+
893
+ while(true) {
894
+ let query = supabase
895
+ .from('messages')
896
+ .select('message, username, display_name, timestamp')
897
+ .not('username', 'in', `(${bots.map(b => `"${b}"`).join(',')})`)
898
+ .range(from, from + limit - 1);
899
+
900
+ if (streamId) {
901
+ query = query.eq('stream_id', streamId);
902
+ }
903
+
904
+ const { data, error } = await query;
905
+
906
+ if (error) {
907
+ console.error('[Supabase] Error getting chat messages:', error);
908
+ break;
909
+ }
910
+
911
+ if (data && data.length > 0) {
912
+ allData = allData.concat(data);
913
+ if (data.length < limit) break; // Fetched the last incomplete page
914
+ from += limit;
915
+ } else {
916
+ break; // Empty page
917
+ }
918
  }
919
+
920
+ return allData;
921
  }
922
  }
923
 
 
996
  if (dbMode === 'sqlite') {
997
  return sqliteDb.prepare('SELECT timestamp FROM messages WHERE stream_id = ? ORDER BY timestamp ASC').all(streamId);
998
  } else {
999
+ let allData = [];
1000
+ let from = 0;
1001
+ const limit = 1000;
 
 
1002
 
1003
+ while(true) {
1004
+ const { data, error } = await supabase
1005
+ .from('messages')
1006
+ .select('timestamp')
1007
+ .eq('stream_id', streamId)
1008
+ .order('timestamp', { ascending: true })
1009
+ .range(from, from + limit - 1);
1010
+
1011
+ if (error) {
1012
+ console.error('[Supabase] Error fetching message timestamps:', error);
1013
+ break;
1014
+ }
1015
+
1016
+ if (data && data.length > 0) {
1017
+ allData = allData.concat(data);
1018
+ if (data.length < limit) break;
1019
+ from += limit;
1020
+ } else {
1021
+ break;
1022
+ }
1023
  }
1024
+ return allData;
1025
  }
1026
  }
1027
 
 
1036
  return sqliteDb.prepare('SELECT moderator, action_type FROM mod_actions').all();
1037
  }
1038
  } else {
1039
+ let allData = [];
1040
+ let from = 0;
1041
+ const limit = 1000;
1042
+
1043
+ while(true) {
1044
+ let query = supabase.from('mod_actions').select('moderator, action_type').range(from, from + limit - 1);
1045
+ if (streamId) {
1046
+ query = query.eq('stream_id', streamId);
1047
+ }
1048
+
1049
+ const { data, error } = await query;
1050
+ if (error) {
1051
+ console.error('[Supabase] Error getting mod actions summary:', error);
1052
+ break;
1053
+ }
1054
+
1055
+ if (data && data.length > 0) {
1056
+ allData = allData.concat(data);
1057
+ if (data.length < limit) break;
1058
+ from += limit;
1059
+ } else {
1060
+ break;
1061
+ }
1062
  }
1063
+ return allData;
1064
  }
1065
  }
1066