Spaces:

Thaimozhi
/

Go-Filter-Bot

Sleeping

Update plugins/bulkindex.go

438cdef verified 3 months ago

12.4 kB

// Bulk indexing – /bulkindex <channel_id> [from_msg_id] [to_msg_id]
// Uses ForwardMessage (singular) to get full media info per message.
// Credit: Bot code made by Claude

package plugins

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/Jisin0/Go-Filter-Bot/database"
	"github.com/Jisin0/Go-Filter-Bot/utils"
	"github.com/PaulSonOfLars/gotgbot/v2"
	"github.com/PaulSonOfLars/gotgbot/v2/ext"
)

// activeBulkJobs prevents duplicate bulk index runs per channel.
var activeBulkJobs = map[int64]bool{}

// BulkIndex handles /bulkindex <channel_id> [from_msg_id] [to_msg_id]
//
//nolint:errcheck
func BulkIndex(bot *gotgbot.Bot, ctx *ext.Context) error {
	if !utils.IsAdmin(ctx.EffectiveUser.Id) {
		ctx.EffectiveMessage.Reply(bot, "⛔ Only bot admins can use this command!", &gotgbot.SendMessageOpts{})
		return nil
	}

	args := strings.Fields(ctx.Message.Text)
	if len(args) < 2 {
		ctx.Message.Reply(bot, `<b>📦 Bulk Index Usage:</b>
<code>/bulkindex &lt;channel_id&gt; &lt;from_id&gt; &lt;to_id&gt;</code>
<b>Example:</b>
<code>/bulkindex -1001234567890 1 500</code>
⚠️ Bot must be admin in the channel.`,
			&gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
		return nil
	}

	// Regex for parsing Telegram channel message links:
	// https://t.me/c/2565549147/106  or  https://t.me/channel_name/106
	linkRegex := regexp.MustCompile(`(?:https?://)?(?:t\.me/|telegram\.me/|telegram\.dog/)(?:c/)?([a-zA-Z0-9_]+)/(\d+)`)

	var (
		chanID int64
		fromID int64 = 1
		toID   int64 = 500
	)

	// Check if user passed message link(s)
	// Example 1: /bulkindex https://t.me/c/2565549147/106 https://t.me/c/2565549147/2237
	// Example 2: /bulkindex https://t.me/c/2565549147/2237  (indexes 1 to 2237)
	// Example 3: /bulkindex -1002565549147 106 2237
	match1 := linkRegex.FindStringSubmatch(args[1])
	if len(match1) == 3 {
		targetChat := match1[1]
		msgID1, _ := strconv.ParseInt(match1[2], 10, 64)

		if strings.HasPrefix(targetChat, "1") || strings.HasPrefix(targetChat, "2") || regexp.MustCompile(`^\d+$`).MatchString(targetChat) {
			parsedID, _ := strconv.ParseInt(targetChat, 10, 64)
			if parsedID > 0 {
				chanID = -1000000000000 - parsedID
			} else {
				chanID = parsedID
			}
		} else {
			// Username based channel
			// Telegram usernames cannot be resolved directly via numeric GetChat API in gotgbot,
			// so prompt the user to use channel ID or invite link.
			ctx.Message.Reply(bot, fmt.Sprintf("❌ For public channel <code>%s</code>, please use numeric ID (e.g. <code>-100...</code>) or full post link.", targetChat), &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
			return nil
		}

		if len(args) >= 3 {
			match2 := linkRegex.FindStringSubmatch(args[2])
			if len(match2) == 3 {
				// Two links provided: link1 is fromID, link2 is toID
				fromID = msgID1
				toID, _ = strconv.ParseInt(match2[2], 10, 64)
			} else if v, e := strconv.ParseInt(args[2], 10, 64); e == nil {
				// One link + toID number
				fromID = msgID1
				toID = v
			}
		} else {
			// Single link provided: index from 1 to that link's msgID
			toID = msgID1
		}
	} else {
		// Raw Channel ID input (e.g. /bulkindex -1002565549147 106 2237)
		var err error
		chanID, err = strconv.ParseInt(args[1], 10, 64)
		if err != nil {
			ctx.Message.Reply(bot, "❌ Invalid input! Send channel ID (<code>-100xxx</code>) or post link(s).\n\n<b>Example:</b>\n<code>/bulkindex https://t.me/c/2565549147/106 https://t.me/c/2565549147/2237</code>", &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
			return nil
		}

		if len(args) >= 3 {
			if v, e := strconv.ParseInt(args[2], 10, 64); e == nil {
				fromID = v
			}
		}
		if len(args) >= 4 {
			if v, e := strconv.ParseInt(args[3], 10, 64); e == nil {
				toID = v
			}
		}
	}

	if activeBulkJobs[chanID] {
		ctx.Message.Reply(bot, "⚠️ Bulk index already running for this channel.", &gotgbot.SendMessageOpts{})
		return nil
	}

	chat, err := bot.GetChat(chanID, nil)
	if err != nil {
		ctx.Message.Reply(bot,
			fmt.Sprintf("❌ Cannot access channel <code>%d</code>\nMake sure bot is admin there.", chanID),
			&gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
		return nil
	}

	statusMsg, _ := ctx.Message.Reply(bot,
		fmt.Sprintf("⏳ <b>Starting Bulk Index</b>\n\n📢 <b>Channel:</b> %s\n📍 <b>Range:</b> %d → %d",
			chat.Title, fromID, toID),
		&gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})

	go runBulkIndex(bot, chanID, fromID, toID, statusMsg, ctx.Message.Chat.Id)
	return nil
}

// CleanGhostDocs handles /cleanghost — removes bad "reference" docs.
//
//nolint:errcheck
func CleanGhostDocs(bot *gotgbot.Bot, ctx *ext.Context) error {
	if !utils.IsAdmin(ctx.EffectiveUser.Id) {
		ctx.EffectiveMessage.Reply(bot, "⛔ Only bot admins can use this command!", &gotgbot.SendMessageOpts{})
		return nil
	}
	statusMsg, _ := ctx.EffectiveMessage.Reply(bot, "🧹 Cleaning ghost documents...", &gotgbot.SendMessageOpts{})
	removed := DB.CleanGhostFilters()
	text := fmt.Sprintf("✅ <b>Cleanup Done!</b>\n🗑 Removed <code>%d</code> ghost documents.", removed)
	if statusMsg != nil {
		statusMsg.EditText(bot, text, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML})
	}
	return nil
}

func runBulkIndex(bot *gotgbot.Bot, chanID, fromID, toID int64, statusMsg *gotgbot.Message, reportChatID int64) {
	activeBulkJobs[chanID] = true
	defer func() { delete(activeBulkJobs, chanID) }()

	indexed := 0
	skipped := 0
	consecFails := 0
	lastUpdate := time.Now()
	startMsgID := fromID
	lastProcessedID := fromID

	for msgID := fromID; msgID <= toID; msgID++ {
		lastProcessedID = msgID

		// ForwardMessage (singular) returns full *Message with all media fields
		fwdMsg, err := bot.ForwardMessage(reportChatID, chanID, msgID, &gotgbot.ForwardMessageOpts{
			DisableNotification: true,
		})

		if err != nil {
			// 1) Handle Telegram Rate Limits (429 FloodWait)
			if tErr, ok := err.(*gotgbot.TelegramError); ok {
				if tErr.Code == 429 || strings.Contains(strings.ToLower(tErr.Description), "too many requests") || strings.Contains(strings.ToLower(tErr.Description), "flood") {
					retrySec := int(tErr.ResponseParams.RetryAfter)
					if retrySec <= 0 {
						retrySec = 15
					}
					if statusMsg != nil {
						statusMsg.EditText(bot, fmt.Sprintf(
							"⏳ <b>Bulk Indexing Paused (FloodWait)</b>\n\n"+
								"⚠️ <i>Telegram Rate Limit hit. Auto-pausing for %d seconds...</i>\n\n"+
								"📍 <b>Progress:</b> <code>%d</code> / <code>%d</code>\n"+
								"📥 <b>Saved:</b> <code>%d</code> | ⏭ <b>Skipped:</b> <code>%d</code>",
							retrySec, msgID, toID, indexed, skipped,
						), &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
					}
					time.Sleep(time.Duration(retrySec+2) * time.Second)
					msgID-- // retry this same post ID
					continue
				}
			}

			// 2) Handle temporary network / connection drop errors
			errStr := strings.ToLower(err.Error())
			if strings.Contains(errStr, "timeout") || strings.Contains(errStr, "connection reset") || strings.Contains(errStr, "eof") || strings.Contains(errStr, "net/http") {
				time.Sleep(3 * time.Second)
				msgID-- // retry this same post ID
				continue
			}

			// 3) Actual missing / deleted post
			consecFails++
			skipped++
			// Increased limit to 500 consecutive missing posts so large deleted sections won't stop indexing
			if consecFails >= 500 {
				fmt.Printf("bulkindex: 500 consecutive fails at msg %d, stopping early\n", msgID)
				break
			}
			time.Sleep(50 * time.Millisecond)
			continue
		}

		consecFails = 0

		// Extract media info from the full message
		var fileID, mediaType, rawName string
		var fileSize int64
		var mimeType string

		switch {
		case fwdMsg.Document != nil:
			fileID = fwdMsg.Document.FileId
			mediaType = "document"
			rawName = fwdMsg.Document.FileName
			fileSize = fwdMsg.Document.FileSize
			mimeType = fwdMsg.Document.MimeType
			if rawName == "" {
				rawName = fwdMsg.Caption
			}

		case fwdMsg.Video != nil:
			fileID = fwdMsg.Video.FileId
			mediaType = "video"
			rawName = fwdMsg.Video.FileName
			fileSize = fwdMsg.Video.FileSize
			mimeType = fwdMsg.Video.MimeType
			if rawName == "" {
				rawName = fwdMsg.Caption
			}

		case fwdMsg.Audio != nil:
			fileID = fwdMsg.Audio.FileId
			mediaType = "audio"
			rawName = fwdMsg.Audio.Title
			fileSize = fwdMsg.Audio.FileSize
			mimeType = fwdMsg.Audio.MimeType
			if rawName == "" {
				rawName = fwdMsg.Audio.FileName
			}

		default:
			bot.DeleteMessage(reportChatID, fwdMsg.MessageId, nil) //nolint:errcheck
			skipped++
			time.Sleep(100 * time.Millisecond)
			continue
		}

		// Delete forwarded copy immediately
		bot.DeleteMessage(reportChatID, fwdMsg.MessageId, nil) //nolint:errcheck

		if rawName == "" && fwdMsg.Caption != "" {
			rawName = fwdMsg.Caption
		}

		if fileID == "" || rawName == "" {
			skipped++
			continue
		}

		keyword := cleanFileName(rawName)
		if len(keyword) < 2 {
			skipped++
			continue
		}

		caption := ""
		if fwdMsg.Caption != "" {
			caption = fwdMsg.OriginalCaptionHTML()
		}

		f := &database.Filter{
			ID:        fileID, // file_id as _id for deduplication
			ChatID:    globalNumber,
			Text:      keyword,
			FileName:  rawName,
			Content:   caption,
			FileID:    fileID,
			MediaType: mediaType,
			FileSize:  fileSize,
			MimeType:  mimeType,
			Length:    len(keyword),
		}

		inserted, err := DB.SaveMfilterByFileID(f)
		if err != nil || !inserted {
			skipped++
		} else {
			indexed++
		}

		// Progress update every 5 seconds
		if statusMsg != nil && time.Since(lastUpdate) > 5*time.Second {
			pct := 0
			total := toID - fromID + 1
			if total > 0 {
				pct = int(float64(msgID-fromID+1) / float64(total) * 100)
			}
			statusMsg.EditText(bot, fmt.Sprintf(
				"⏳ <b>Bulk Indexing In Progress...</b>\n\n"+
					"📍 <b>Start Post ID:</b> <code>%d</code>\n"+
					"🎯 <b>Target End ID:</b> <code>%d</code>\n"+
					"🔄 <b>Current Post ID:</b> <code>%d</code>\n\n"+
					"📥 <b>Saved Files:</b> <code>%d</code>\n"+
					"⏭ <b>Skipped/Non-Media:</b> <code>%d</code>\n"+
					"📊 <b>Progress:</b> <code>%d/%d</code> (%d%%)",
				startMsgID, toID, msgID, indexed, skipped, msgID-fromID+1, total, pct,
			), &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
			lastUpdate = time.Now()
		}

		time.Sleep(200 * time.Millisecond) // flood protection
	}

	finalText := fmt.Sprintf(
		"✅ <b>Bulk Index Completed!</b>\n\n"+
			"📍 <b>Start Post ID:</b> <code>%d</code>\n"+
			"🏁 <b>End Post ID:</b> <code>%d</code>\n"+
			"🎯 <b>Target Range:</b> <code>%d → %d</code>\n\n"+
			"📥 <b>Successfully Saved:</b> <code>%d</code> files\n"+
			"⏭ <b>Skipped / Duplicates / Deleted:</b> <code>%d</code> messages\n\n"+
			"<i>Run /cleanghost to cleanup any ghost filters.</i>",
		startMsgID, lastProcessedID, fromID, toID, indexed, skipped,
	)
	if statusMsg != nil {
		statusMsg.EditText(bot, finalText, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
	} else {
		bot.SendMessage(reportChatID, finalText, &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
	}
}

// MigrateIDs handles /migrateids — fixes old documents with long Telegram file_id
// as _id (which break inline button callback_data, causing BUTTON_DATA_INVALID).
//
//nolint:errcheck
func MigrateIDs(bot *gotgbot.Bot, ctx *ext.Context) error {
	if !utils.IsAdmin(ctx.EffectiveUser.Id) {
		ctx.EffectiveMessage.Reply(bot, "⛔ Only bot admins can use this command!", &gotgbot.SendMessageOpts{})
		return nil
	}

	statusMsg, _ := ctx.EffectiveMessage.Reply(bot, "🔄 Migrating old document IDs (this may take a moment)...", &gotgbot.SendMessageOpts{})

	migrated, err := DB.MigrateLongIDs()
	if err != nil {
		ctx.EffectiveMessage.Reply(bot, fmt.Sprintf("❌ Migration error: %v", err), &gotgbot.SendMessageOpts{})
		return nil
	}

	text := fmt.Sprintf("✅ <b>Migration Complete!</b>\n\n🔄 Migrated <code>%d</code> documents to short IDs.\n\n<i>This fixes BUTTON_DATA_INVALID errors on old indexed files.</i>", migrated)
	if statusMsg != nil {
		statusMsg.EditText(bot, text, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML})
	} else {
		ctx.EffectiveMessage.Reply(bot, text, &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
	}
	return nil
}

	// Bulk indexing – /bulkindex <channel_id> [from_msg_id] [to_msg_id]
	// Uses ForwardMessage (singular) to get full media info per message.
	// Credit: Bot code made by Claude

	package plugins

	import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/Jisin0/Go-Filter-Bot/database"
	"github.com/Jisin0/Go-Filter-Bot/utils"
	"github.com/PaulSonOfLars/gotgbot/v2"
	"github.com/PaulSonOfLars/gotgbot/v2/ext"
	)

	// activeBulkJobs prevents duplicate bulk index runs per channel.
	var activeBulkJobs = map[int64]bool{}

	// BulkIndex handles /bulkindex <channel_id> [from_msg_id] [to_msg_id]
	//
	//nolint:errcheck
	func BulkIndex(bot gotgbot.Bot, ctx ext.Context) error {
	if !utils.IsAdmin(ctx.EffectiveUser.Id) {
	ctx.EffectiveMessage.Reply(bot, "⛔ Only bot admins can use this command!", &gotgbot.SendMessageOpts{})
	return nil
	}

	args := strings.Fields(ctx.Message.Text)
	if len(args) < 2 {
	ctx.Message.Reply(bot, `<b>📦 Bulk Index Usage:</b>

	<code>/bulkindex <channel_id> <from_id> <to_id></code>

	<b>Example:</b>
	<code>/bulkindex -1001234567890 1 500</code>

	⚠️ Bot must be admin in the channel.`,
	&gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
	return nil
	}

	// Regex for parsing Telegram channel message links:
	// https://t.me/c/2565549147/106 or https://t.me/channel_name/106
	linkRegex := regexp.MustCompile(`(?:https?://)?(?:t\.me/\|telegram\.me/\|telegram\.dog/)(?:c/)?([a-zA-Z0-9_]+)/(\d+)`)

	var (
	chanID int64
	fromID int64 = 1
	toID int64 = 500
	)

	// Check if user passed message link(s)
	// Example 1: /bulkindex https://t.me/c/2565549147/106 https://t.me/c/2565549147/2237
	// Example 2: /bulkindex https://t.me/c/2565549147/2237 (indexes 1 to 2237)
	// Example 3: /bulkindex -1002565549147 106 2237
	match1 := linkRegex.FindStringSubmatch(args[1])
	if len(match1) == 3 {
	targetChat := match1[1]
	msgID1, _ := strconv.ParseInt(match1[2], 10, 64)

	if strings.HasPrefix(targetChat, "1") \|\| strings.HasPrefix(targetChat, "2") \|\| regexp.MustCompile(`^\d+$`).MatchString(targetChat) {
	parsedID, _ := strconv.ParseInt(targetChat, 10, 64)
	if parsedID > 0 {
	chanID = -1000000000000 - parsedID
	} else {
	chanID = parsedID
	}
	} else {
	// Username based channel
	// Telegram usernames cannot be resolved directly via numeric GetChat API in gotgbot,
	// so prompt the user to use channel ID or invite link.
	ctx.Message.Reply(bot, fmt.Sprintf("❌ For public channel <code>%s</code>, please use numeric ID (e.g. <code>-100...</code>) or full post link.", targetChat), &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
	return nil
	}

	if len(args) >= 3 {
	match2 := linkRegex.FindStringSubmatch(args[2])
	if len(match2) == 3 {
	// Two links provided: link1 is fromID, link2 is toID
	fromID = msgID1
	toID, _ = strconv.ParseInt(match2[2], 10, 64)
	} else if v, e := strconv.ParseInt(args[2], 10, 64); e == nil {
	// One link + toID number
	fromID = msgID1
	toID = v
	}
	} else {
	// Single link provided: index from 1 to that link's msgID
	toID = msgID1
	}
	} else {
	// Raw Channel ID input (e.g. /bulkindex -1002565549147 106 2237)
	var err error
	chanID, err = strconv.ParseInt(args[1], 10, 64)
	if err != nil {
	ctx.Message.Reply(bot, "❌ Invalid input! Send channel ID (<code>-100xxx</code>) or post link(s).\n\n<b>Example:</b>\n<code>/bulkindex https://t.me/c/2565549147/106 https://t.me/c/2565549147/2237</code>", &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
	return nil
	}

	if len(args) >= 3 {
	if v, e := strconv.ParseInt(args[2], 10, 64); e == nil {
	fromID = v
	}
	}
	if len(args) >= 4 {
	if v, e := strconv.ParseInt(args[3], 10, 64); e == nil {
	toID = v
	}
	}
	}

	if activeBulkJobs[chanID] {
	ctx.Message.Reply(bot, "⚠️ Bulk index already running for this channel.", &gotgbot.SendMessageOpts{})
	return nil
	}

	chat, err := bot.GetChat(chanID, nil)
	if err != nil {
	ctx.Message.Reply(bot,
	fmt.Sprintf("❌ Cannot access channel <code>%d</code>\nMake sure bot is admin there.", chanID),
	&gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
	return nil
	}

	statusMsg, _ := ctx.Message.Reply(bot,
	fmt.Sprintf("⏳ <b>Starting Bulk Index</b>\n\n📢 <b>Channel:</b> %s\n📍 <b>Range:</b> %d → %d",
	chat.Title, fromID, toID),
	&gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})

	go runBulkIndex(bot, chanID, fromID, toID, statusMsg, ctx.Message.Chat.Id)
	return nil
	}

	// CleanGhostDocs handles /cleanghost — removes bad "reference" docs.
	//
	//nolint:errcheck
	func CleanGhostDocs(bot gotgbot.Bot, ctx ext.Context) error {
	if !utils.IsAdmin(ctx.EffectiveUser.Id) {
	ctx.EffectiveMessage.Reply(bot, "⛔ Only bot admins can use this command!", &gotgbot.SendMessageOpts{})
	return nil
	}
	statusMsg, _ := ctx.EffectiveMessage.Reply(bot, "🧹 Cleaning ghost documents...", &gotgbot.SendMessageOpts{})
	removed := DB.CleanGhostFilters()
	text := fmt.Sprintf("✅ <b>Cleanup Done!</b>\n🗑 Removed <code>%d</code> ghost documents.", removed)
	if statusMsg != nil {
	statusMsg.EditText(bot, text, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML})
	}
	return nil
	}

	func runBulkIndex(bot gotgbot.Bot, chanID, fromID, toID int64, statusMsg gotgbot.Message, reportChatID int64) {
	activeBulkJobs[chanID] = true
	defer func() { delete(activeBulkJobs, chanID) }()

	indexed := 0
	skipped := 0
	consecFails := 0
	lastUpdate := time.Now()
	startMsgID := fromID
	lastProcessedID := fromID

	for msgID := fromID; msgID <= toID; msgID++ {
	lastProcessedID = msgID

	// ForwardMessage (singular) returns full *Message with all media fields
	fwdMsg, err := bot.ForwardMessage(reportChatID, chanID, msgID, &gotgbot.ForwardMessageOpts{
	DisableNotification: true,
	})

	if err != nil {
	// 1) Handle Telegram Rate Limits (429 FloodWait)
	if tErr, ok := err.(*gotgbot.TelegramError); ok {
	if tErr.Code == 429 \|\| strings.Contains(strings.ToLower(tErr.Description), "too many requests") \|\| strings.Contains(strings.ToLower(tErr.Description), "flood") {
	retrySec := int(tErr.ResponseParams.RetryAfter)
	if retrySec <= 0 {
	retrySec = 15
	}
	if statusMsg != nil {
	statusMsg.EditText(bot, fmt.Sprintf(
	"⏳ <b>Bulk Indexing Paused (FloodWait)</b>\n\n"+
	"⚠️ <i>Telegram Rate Limit hit. Auto-pausing for %d seconds...</i>\n\n"+
	"📍 <b>Progress:</b> <code>%d</code> / <code>%d</code>\n"+
	"📥 <b>Saved:</b> <code>%d</code> \| ⏭ <b>Skipped:</b> <code>%d</code>",
	retrySec, msgID, toID, indexed, skipped,
	), &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
	}
	time.Sleep(time.Duration(retrySec+2) * time.Second)
	msgID-- // retry this same post ID
	continue
	}
	}

	// 2) Handle temporary network / connection drop errors
	errStr := strings.ToLower(err.Error())
	if strings.Contains(errStr, "timeout") \|\| strings.Contains(errStr, "connection reset") \|\| strings.Contains(errStr, "eof") \|\| strings.Contains(errStr, "net/http") {
	time.Sleep(3 * time.Second)
	msgID-- // retry this same post ID
	continue
	}

	// 3) Actual missing / deleted post
	consecFails++
	skipped++
	// Increased limit to 500 consecutive missing posts so large deleted sections won't stop indexing
	if consecFails >= 500 {
	fmt.Printf("bulkindex: 500 consecutive fails at msg %d, stopping early\n", msgID)
	break
	}
	time.Sleep(50 * time.Millisecond)
	continue
	}

	consecFails = 0

	// Extract media info from the full message
	var fileID, mediaType, rawName string
	var fileSize int64
	var mimeType string

	switch {
	case fwdMsg.Document != nil:
	fileID = fwdMsg.Document.FileId
	mediaType = "document"
	rawName = fwdMsg.Document.FileName
	fileSize = fwdMsg.Document.FileSize
	mimeType = fwdMsg.Document.MimeType
	if rawName == "" {
	rawName = fwdMsg.Caption
	}

	case fwdMsg.Video != nil:
	fileID = fwdMsg.Video.FileId
	mediaType = "video"
	rawName = fwdMsg.Video.FileName
	fileSize = fwdMsg.Video.FileSize
	mimeType = fwdMsg.Video.MimeType
	if rawName == "" {
	rawName = fwdMsg.Caption
	}

	case fwdMsg.Audio != nil:
	fileID = fwdMsg.Audio.FileId
	mediaType = "audio"
	rawName = fwdMsg.Audio.Title
	fileSize = fwdMsg.Audio.FileSize
	mimeType = fwdMsg.Audio.MimeType
	if rawName == "" {
	rawName = fwdMsg.Audio.FileName
	}

	default:
	bot.DeleteMessage(reportChatID, fwdMsg.MessageId, nil) //nolint:errcheck
	skipped++
	time.Sleep(100 * time.Millisecond)
	continue
	}

	// Delete forwarded copy immediately
	bot.DeleteMessage(reportChatID, fwdMsg.MessageId, nil) //nolint:errcheck

	if rawName == "" && fwdMsg.Caption != "" {
	rawName = fwdMsg.Caption
	}

	if fileID == "" \|\| rawName == "" {
	skipped++
	continue
	}

	keyword := cleanFileName(rawName)
	if len(keyword) < 2 {
	skipped++
	continue
	}

	caption := ""
	if fwdMsg.Caption != "" {
	caption = fwdMsg.OriginalCaptionHTML()
	}

	f := &database.Filter{
	ID: fileID, // file_id as _id for deduplication
	ChatID: globalNumber,
	Text: keyword,
	FileName: rawName,
	Content: caption,
	FileID: fileID,
	MediaType: mediaType,
	FileSize: fileSize,
	MimeType: mimeType,
	Length: len(keyword),
	}

	inserted, err := DB.SaveMfilterByFileID(f)
	if err != nil \|\| !inserted {
	skipped++
	} else {
	indexed++
	}

	// Progress update every 5 seconds
	if statusMsg != nil && time.Since(lastUpdate) > 5*time.Second {
	pct := 0
	total := toID - fromID + 1
	if total > 0 {
	pct = int(float64(msgID-fromID+1) / float64(total) * 100)
	}
	statusMsg.EditText(bot, fmt.Sprintf(
	"⏳ <b>Bulk Indexing In Progress...</b>\n\n"+
	"📍 <b>Start Post ID:</b> <code>%d</code>\n"+
	"🎯 <b>Target End ID:</b> <code>%d</code>\n"+
	"🔄 <b>Current Post ID:</b> <code>%d</code>\n\n"+
	"📥 <b>Saved Files:</b> <code>%d</code>\n"+
	"⏭ <b>Skipped/Non-Media:</b> <code>%d</code>\n"+
	"📊 <b>Progress:</b> <code>%d/%d</code> (%d%%)",
	startMsgID, toID, msgID, indexed, skipped, msgID-fromID+1, total, pct,
	), &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
	lastUpdate = time.Now()
	}

	time.Sleep(200 * time.Millisecond) // flood protection
	}

	finalText := fmt.Sprintf(
	"✅ <b>Bulk Index Completed!</b>\n\n"+
	"📍 <b>Start Post ID:</b> <code>%d</code>\n"+
	"🏁 <b>End Post ID:</b> <code>%d</code>\n"+
	"🎯 <b>Target Range:</b> <code>%d → %d</code>\n\n"+
	"📥 <b>Successfully Saved:</b> <code>%d</code> files\n"+
	"⏭ <b>Skipped / Duplicates / Deleted:</b> <code>%d</code> messages\n\n"+
	"<i>Run /cleanghost to cleanup any ghost filters.</i>",
	startMsgID, lastProcessedID, fromID, toID, indexed, skipped,
	)
	if statusMsg != nil {
	statusMsg.EditText(bot, finalText, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
	} else {
	bot.SendMessage(reportChatID, finalText, &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML}) //nolint:errcheck
	}
	}

	// MigrateIDs handles /migrateids — fixes old documents with long Telegram file_id
	// as _id (which break inline button callback_data, causing BUTTON_DATA_INVALID).
	//
	//nolint:errcheck
	func MigrateIDs(bot gotgbot.Bot, ctx ext.Context) error {
	if !utils.IsAdmin(ctx.EffectiveUser.Id) {
	ctx.EffectiveMessage.Reply(bot, "⛔ Only bot admins can use this command!", &gotgbot.SendMessageOpts{})
	return nil
	}

	statusMsg, _ := ctx.EffectiveMessage.Reply(bot, "🔄 Migrating old document IDs (this may take a moment)...", &gotgbot.SendMessageOpts{})

	migrated, err := DB.MigrateLongIDs()
	if err != nil {
	ctx.EffectiveMessage.Reply(bot, fmt.Sprintf("❌ Migration error: %v", err), &gotgbot.SendMessageOpts{})
	return nil
	}

	text := fmt.Sprintf("✅ <b>Migration Complete!</b>\n\n🔄 Migrated <code>%d</code> documents to short IDs.\n\n<i>This fixes BUTTON_DATA_INVALID errors on old indexed files.</i>", migrated)
	if statusMsg != nil {
	statusMsg.EditText(bot, text, &gotgbot.EditMessageTextOpts{ParseMode: gotgbot.ParseModeHTML})
	} else {
	ctx.EffectiveMessage.Reply(bot, text, &gotgbot.SendMessageOpts{ParseMode: gotgbot.ParseModeHTML})
	}
	return nil
	}