main.go

package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/csv"
	"encoding/json"
	"io"
	"log"
	"net/http"
	"net/url"
	"strings"
	"time"
)

/* ---------- logging middleware ---------- */

type loggingResponseWriter struct {
	http.ResponseWriter
	statusCode int
}

func (lrw *loggingResponseWriter) WriteHeader(code int) {
	lrw.statusCode = code
	lrw.ResponseWriter.WriteHeader(code)
}

func loggingMiddleware(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		start := time.Now()

		lrw := &loggingResponseWriter{
			ResponseWriter: w,
			statusCode:     http.StatusOK,
		}

		next.ServeHTTP(lrw, r)

		log.Printf(
			"%s %s | %d | %s | %s",
			r.Method,
			r.URL.Path,
			lrw.statusCode,
			r.RemoteAddr,
			time.Since(start),
		)
	})
}

/* ---------- webhook payload ---------- */

type WebhookPayload struct {
	Source             string `json:"source"`
	DetectedHash       string `json:"detected_hash"`
	DetectionTimestamp string `json:"detection_timestamp"`
	ContentPreview     string `json:"content_preview"`
}

/* ---------- helpers ---------- */

func extractZipURLs(preview string) []string {
	var out []string
	scanner := bufio.NewScanner(strings.NewReader(preview))
	for scanner.Scan() {
		for _, f := range strings.Fields(scanner.Text()) {
			if strings.HasPrefix(f, "http") {
				out = append(out, f)
			}
		}
	}
	return out
}

func validURL(s string) bool {
	u, err := url.ParseRequestURI(s)
	return err == nil && u.Scheme != "" && u.Host != ""
}

func processZip(zipURL string, unique map[string]struct{}) error {
	resp, err := http.Get(zipURL)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	data, err := io.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if err != nil {
		return err
	}

	for _, f := range zr.File {
		if !strings.HasSuffix(strings.ToLower(f.Name), ".csv") {
			continue
		}

		rc, err := f.Open()
		if err != nil {
			continue
		}

		reader := csv.NewReader(bufio.NewReader(rc))
		reader.FieldsPerRecord = -1

		for {
			row, err := reader.Read()
			if err == io.EOF {
				break
			}
			if err != nil {
				continue
			}

			for _, cell := range row {
				cell = strings.TrimSpace(cell)
				if strings.HasPrefix(cell, "http") && validURL(cell) {
					unique[cell] = struct{}{}
				}
			}
		}

		rc.Close()
	}

	return nil
}

/* ---------- /latest handler (THE webhook) ---------- */

func latestHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method != http.MethodPost {
		http.Error(w, "POST only", http.StatusMethodNotAllowed)
		return
	}

	var payload WebhookPayload
	if err := json.NewDecoder(r.Body).Decode(&payload); err != nil {
		http.Error(w, "invalid json", http.StatusBadRequest)
		return
	}

	log.Println("Webhook source:", payload.Source)
	log.Println("Detection time:", payload.DetectionTimestamp)

	zipURLs := extractZipURLs(payload.ContentPreview)
	unique := make(map[string]struct{})

	for _, z := range zipURLs {
		log.Println("Processing ZIP:", z)
		if err := processZip(z, unique); err != nil {
			log.Println("Error:", err)
		}
	}

	log.Printf("UNIQUE URL COUNT = %d", len(unique))
	for u := range unique {
		log.Println(u)
	}

	w.WriteHeader(http.StatusOK)
	w.Write([]byte("ok"))
}

/* ---------- main ---------- */

func main() {
	mux := http.NewServeMux()
	mux.HandleFunc("/latest", latestHandler)

	port := "7860"
	log.Println("Listening on port", port)

	log.Fatal(http.ListenAndServe(":"+port, loggingMiddleware(mux)))
}