// Bulk indexing โ€“ /bulkindex [from_msg_id] [to_msg_id] // Uses ForwardMessage (singular) to get full media info per message. // Credit: Bot code made by Claude package plugins import ( "fmt" "regexp" "strconv" "strings" "time" "github.com/Jisin0/Go-Filter-Bot/database" "github.com/Jisin0/Go-Filter-Bot/utils" "github.com/PaulSonOfLars/gotgbot/v2" "github.com/PaulSonOfLars/gotgbot/v2/ext" ) // activeBulkJobs prevents duplicate bulk index runs per channel. var activeBulkJobs = map[int64]bool{} // BulkIndex handles /bulkindex [from_msg_id] [to_msg_id] // //nolint:errcheck func BulkIndex(bot *gotgbot.Bot, ctx *ext.Context) error { if !utils.IsAdmin(ctx.EffectiveUser.Id) { ctx.EffectiveMessage.Reply(bot, "โ›” Only bot admins can use this command!", &gotgbot.SendMessageOpts{}) return nil } args := strings.Fields(ctx.Message.Text) if len(args) < 2 { ctx.Message.Reply(bot, `๐Ÿ“ฆ Bulk Index Usage: /bulkindex <channel_id> <from_id> <to_id> Example: /bulkindex -1001234567890 1 500 โš ๏ธ Bot must be admin in the channel.`, &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) return nil } // Regex for parsing Telegram channel message links: // https://t.me/c/2565549147/106 or https://t.me/channel_name/106 linkRegex := regexp.MustCompile(`(?:https?://)?(?:t\.me/|telegram\.me/|telegram\.dog/)(?:c/)?([a-zA-Z0-9_]+)/(\d+)`) var ( chanID int64 fromID int64 = 1 toID int64 = 500 ) // Check if user passed message link(s) // Example 1: /bulkindex https://t.me/c/2565549147/106 https://t.me/c/2565549147/2237 // Example 2: /bulkindex https://t.me/c/2565549147/2237 (indexes 1 to 2237) // Example 3: /bulkindex -1002565549147 106 2237 match1 := linkRegex.FindStringSubmatch(args[1]) if len(match1) == 3 { targetChat := match1[1] msgID1, _ := strconv.ParseInt(match1[2], 10, 64) if strings.HasPrefix(targetChat, "1") || strings.HasPrefix(targetChat, "2") || regexp.MustCompile(`^\d+$`).MatchString(targetChat) { parsedID, _ := strconv.ParseInt(targetChat, 10, 64) if parsedID > 0 { chanID = -1000000000000 - parsedID } else { chanID = parsedID } } else { // Username based channel // Telegram usernames cannot be resolved directly via numeric GetChat API in gotgbot, // so prompt the user to use channel ID or invite link. ctx.Message.Reply(bot, fmt.Sprintf("โŒ For public channel %s, please use numeric ID (e.g. -100...) or full post link.", targetChat), &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) return nil } if len(args) >= 3 { match2 := linkRegex.FindStringSubmatch(args[2]) if len(match2) == 3 { // Two links provided: link1 is fromID, link2 is toID fromID = msgID1 toID, _ = strconv.ParseInt(match2[2], 10, 64) } else if v, e := strconv.ParseInt(args[2], 10, 64); e == nil { // One link + toID number fromID = msgID1 toID = v } } else { // Single link provided: index from 1 to that link's msgID toID = msgID1 } } else { // Raw Channel ID input (e.g. /bulkindex -1002565549147 106 2237) var err error chanID, err = strconv.ParseInt(args[1], 10, 64) if err != nil { ctx.Message.Reply(bot, "โŒ Invalid input! Send channel ID (-100xxx) or post link(s).\n\nExample:\n/bulkindex https://t.me/c/2565549147/106 https://t.me/c/2565549147/2237", &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) return nil } if len(args) >= 3 { if v, e := strconv.ParseInt(args[2], 10, 64); e == nil { fromID = v } } if len(args) >= 4 { if v, e := strconv.ParseInt(args[3], 10, 64); e == nil { toID = v } } } if activeBulkJobs[chanID] { ctx.Message.Reply(bot, "โš ๏ธ Bulk index already running for this channel.", &gotgbot.SendMessageOpts{}) return nil } chat, err := bot.GetChat(chanID, nil) if err != nil { ctx.Message.Reply(bot, fmt.Sprintf("โŒ Cannot access channel %d\nMake sure bot is admin there.", chanID), &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) return nil } statusMsg, _ := ctx.Message.Reply(bot, fmt.Sprintf("โณ Starting Bulk Index\n\n๐Ÿ“ข Channel: %s\n๐Ÿ“ Range: %d โ†’ %d", chat.Title, fromID, toID), &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) go runBulkIndex(bot, chanID, fromID, toID, statusMsg, ctx.Message.Chat.Id) return nil } // CleanGhostDocs handles /cleanghost โ€” removes bad "reference" docs. // //nolint:errcheck func CleanGhostDocs(bot *gotgbot.Bot, ctx *ext.Context) error { if !utils.IsAdmin(ctx.EffectiveUser.Id) { ctx.EffectiveMessage.Reply(bot, "โ›” Only bot admins can use this command!", &gotgbot.SendMessageOpts{}) return nil } statusMsg, _ := ctx.EffectiveMessage.Reply(bot, "๐Ÿงน Cleaning ghost documents...", &gotgbot.SendMessageOpts{}) removed := DB.CleanGhostFilters() text := fmt.Sprintf("โœ… Cleanup Done!\n๐Ÿ—‘ Removed %d ghost documents.", removed) if statusMsg != nil { statusMsg.EditText(bot, text, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) } return nil } func runBulkIndex(bot *gotgbot.Bot, chanID, fromID, toID int64, statusMsg *gotgbot.Message, reportChatID int64) { activeBulkJobs[chanID] = true defer func() { delete(activeBulkJobs, chanID) }() indexed := 0 skipped := 0 consecFails := 0 lastUpdate := time.Now() startMsgID := fromID lastProcessedID := fromID for msgID := fromID; msgID <= toID; msgID++ { lastProcessedID = msgID // ForwardMessage (singular) returns full *Message with all media fields fwdMsg, err := bot.ForwardMessage(reportChatID, chanID, msgID, &gotgbot.ForwardMessageOpts{ DisableNotification: true, }) if err != nil { // 1) Handle Telegram Rate Limits (429 FloodWait) if tErr, ok := err.(*gotgbot.TelegramError); ok { if tErr.Code == 429 || strings.Contains(strings.ToLower(tErr.Description), "too many requests") || strings.Contains(strings.ToLower(tErr.Description), "flood") { retrySec := int(tErr.ResponseParams.RetryAfter) if retrySec <= 0 { retrySec = 15 } if statusMsg != nil { statusMsg.EditText(bot, fmt.Sprintf( "โณ Bulk Indexing Paused (FloodWait)\n\n"+ "โš ๏ธ Telegram Rate Limit hit. Auto-pausing for %d seconds...\n\n"+ "๐Ÿ“ Progress: %d / %d\n"+ "๐Ÿ“ฅ Saved: %d | โญ Skipped: %d", retrySec, msgID, toID, indexed, skipped, ), &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck } time.Sleep(time.Duration(retrySec+2) * time.Second) msgID-- // retry this same post ID continue } } // 2) Handle temporary network / connection drop errors errStr := strings.ToLower(err.Error()) if strings.Contains(errStr, "timeout") || strings.Contains(errStr, "connection reset") || strings.Contains(errStr, "eof") || strings.Contains(errStr, "net/http") { time.Sleep(3 * time.Second) msgID-- // retry this same post ID continue } // 3) Actual missing / deleted post consecFails++ skipped++ // Increased limit to 500 consecutive missing posts so large deleted sections won't stop indexing if consecFails >= 500 { fmt.Printf("bulkindex: 500 consecutive fails at msg %d, stopping early\n", msgID) break } time.Sleep(50 * time.Millisecond) continue } consecFails = 0 // Extract media info from the full message var fileID, mediaType, rawName string var fileSize int64 var mimeType string switch { case fwdMsg.Document != nil: fileID = fwdMsg.Document.FileId mediaType = "document" rawName = fwdMsg.Document.FileName fileSize = fwdMsg.Document.FileSize mimeType = fwdMsg.Document.MimeType if rawName == "" { rawName = fwdMsg.Caption } case fwdMsg.Video != nil: fileID = fwdMsg.Video.FileId mediaType = "video" rawName = fwdMsg.Video.FileName fileSize = fwdMsg.Video.FileSize mimeType = fwdMsg.Video.MimeType if rawName == "" { rawName = fwdMsg.Caption } case fwdMsg.Audio != nil: fileID = fwdMsg.Audio.FileId mediaType = "audio" rawName = fwdMsg.Audio.Title fileSize = fwdMsg.Audio.FileSize mimeType = fwdMsg.Audio.MimeType if rawName == "" { rawName = fwdMsg.Audio.FileName } default: bot.DeleteMessage(reportChatID, fwdMsg.MessageId, nil) //nolint:errcheck skipped++ time.Sleep(100 * time.Millisecond) continue } // Delete forwarded copy immediately bot.DeleteMessage(reportChatID, fwdMsg.MessageId, nil) //nolint:errcheck if rawName == "" && fwdMsg.Caption != "" { rawName = fwdMsg.Caption } if fileID == "" || rawName == "" { skipped++ continue } keyword := cleanFileName(rawName) if len(keyword) < 2 { skipped++ continue } caption := "" if fwdMsg.Caption != "" { caption = fwdMsg.OriginalCaptionHTML() } f := &database.Filter{ ID: fileID, // file_id as _id for deduplication ChatID: globalNumber, Text: keyword, FileName: rawName, Content: caption, FileID: fileID, MediaType: mediaType, FileSize: fileSize, MimeType: mimeType, Length: len(keyword), } inserted, err := DB.SaveMfilterByFileID(f) if err != nil || !inserted { skipped++ } else { indexed++ } // Progress update every 5 seconds if statusMsg != nil && time.Since(lastUpdate) > 5*time.Second { pct := 0 total := toID - fromID + 1 if total > 0 { pct = int(float64(msgID-fromID+1) / float64(total) * 100) } statusMsg.EditText(bot, fmt.Sprintf( "โณ Bulk Indexing In Progress...\n\n"+ "๐Ÿ“ Start Post ID: %d\n"+ "๐ŸŽฏ Target End ID: %d\n"+ "๐Ÿ”„ Current Post ID: %d\n\n"+ "๐Ÿ“ฅ Saved Files: %d\n"+ "โญ Skipped/Non-Media: %d\n"+ "๐Ÿ“Š Progress: %d/%d (%d%%)", startMsgID, toID, msgID, indexed, skipped, msgID-fromID+1, total, pct, ), &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck lastUpdate = time.Now() } time.Sleep(200 * time.Millisecond) // flood protection } finalText := fmt.Sprintf( "โœ… Bulk Index Completed!\n\n"+ "๐Ÿ“ Start Post ID: %d\n"+ "๐Ÿ End Post ID: %d\n"+ "๐ŸŽฏ Target Range: %d โ†’ %d\n\n"+ "๐Ÿ“ฅ Successfully Saved: %d files\n"+ "โญ Skipped / Duplicates / Deleted: %d messages\n\n"+ "Run /cleanghost to cleanup any ghost filters.", startMsgID, lastProcessedID, fromID, toID, indexed, skipped, ) if statusMsg != nil { statusMsg.EditText(bot, finalText, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck } else { bot.SendMessage(reportChatID, finalText, &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck } } // MigrateIDs handles /migrateids โ€” fixes old documents with long Telegram file_id // as _id (which break inline button callback_data, causing BUTTON_DATA_INVALID). // //nolint:errcheck func MigrateIDs(bot *gotgbot.Bot, ctx *ext.Context) error { if !utils.IsAdmin(ctx.EffectiveUser.Id) { ctx.EffectiveMessage.Reply(bot, "โ›” Only bot admins can use this command!", &gotgbot.SendMessageOpts{}) return nil } statusMsg, _ := ctx.EffectiveMessage.Reply(bot, "๐Ÿ”„ Migrating old document IDs (this may take a moment)...", &gotgbot.SendMessageOpts{}) migrated, err := DB.MigrateLongIDs() if err != nil { ctx.EffectiveMessage.Reply(bot, fmt.Sprintf("โŒ Migration error: %v", err), &gotgbot.SendMessageOpts{}) return nil } text := fmt.Sprintf("โœ… Migration Complete!\n\n๐Ÿ”„ Migrated %d documents to short IDs.\n\nThis fixes BUTTON_DATA_INVALID errors on old indexed files.", migrated) if statusMsg != nil { statusMsg.EditText(bot, text, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) } else { ctx.EffectiveMessage.Reply(bot, text, &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) } return nil }