package main import ( "archive/zip" "bufio" "bytes" "encoding/csv" "encoding/json" "io" "log" "net/http" "net/url" "strings" "time" ) /* ---------- logging middleware ---------- */ type loggingResponseWriter struct { http.ResponseWriter statusCode int } func (lrw *loggingResponseWriter) WriteHeader(code int) { lrw.statusCode = code lrw.ResponseWriter.WriteHeader(code) } func loggingMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { start := time.Now() lrw := &loggingResponseWriter{ ResponseWriter: w, statusCode: http.StatusOK, } next.ServeHTTP(lrw, r) log.Printf( "%s %s | %d | %s | %s", r.Method, r.URL.Path, lrw.statusCode, r.RemoteAddr, time.Since(start), ) }) } /* ---------- webhook payload ---------- */ type WebhookPayload struct { Source string `json:"source"` DetectedHash string `json:"detected_hash"` DetectionTimestamp string `json:"detection_timestamp"` ContentPreview string `json:"content_preview"` } /* ---------- helpers ---------- */ func extractZipURLs(preview string) []string { var out []string scanner := bufio.NewScanner(strings.NewReader(preview)) for scanner.Scan() { for _, f := range strings.Fields(scanner.Text()) { if strings.HasPrefix(f, "http") { out = append(out, f) } } } return out } func validURL(s string) bool { u, err := url.ParseRequestURI(s) return err == nil && u.Scheme != "" && u.Host != "" } func processZip(zipURL string, unique map[string]struct{}) error { resp, err := http.Get(zipURL) if err != nil { return err } defer resp.Body.Close() data, err := io.ReadAll(resp.Body) if err != nil { return err } zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data))) if err != nil { return err } for _, f := range zr.File { if !strings.HasSuffix(strings.ToLower(f.Name), ".csv") { continue } rc, err := f.Open() if err != nil { continue } reader := csv.NewReader(bufio.NewReader(rc)) reader.FieldsPerRecord = -1 for { row, err := reader.Read() if err == io.EOF { break } if err != nil { continue } for _, cell := range row { cell = strings.TrimSpace(cell) if strings.HasPrefix(cell, "http") && validURL(cell) { unique[cell] = struct{}{} } } } rc.Close() } return nil } /* ---------- /latest handler (THE webhook) ---------- */ func latestHandler(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { http.Error(w, "POST only", http.StatusMethodNotAllowed) return } var payload WebhookPayload if err := json.NewDecoder(r.Body).Decode(&payload); err != nil { http.Error(w, "invalid json", http.StatusBadRequest) return } log.Println("Webhook source:", payload.Source) log.Println("Detection time:", payload.DetectionTimestamp) zipURLs := extractZipURLs(payload.ContentPreview) unique := make(map[string]struct{}) for _, z := range zipURLs { log.Println("Processing ZIP:", z) if err := processZip(z, unique); err != nil { log.Println("Error:", err) } } log.Printf("UNIQUE URL COUNT = %d", len(unique)) for u := range unique { log.Println(u) } w.WriteHeader(http.StatusOK) w.Write([]byte("ok")) } /* ---------- main ---------- */ func main() { mux := http.NewServeMux() mux.HandleFunc("/latest", latestHandler) port := "7860" log.Println("Listening on port", port) log.Fatal(http.ListenAndServe(":"+port, loggingMiddleware(mux))) }