llmcost/sync/sync_test.go

package sync

import (
	"context"
	"errors"
	"log/slog"
	"testing"

	"github.com/alpacahq/alpacadecimal"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"github.com/openmeterio/openmeter/openmeter/llmcost"
)

// mockFetcher implements Fetcher for testing.
type mockFetcher struct {
	source llmcost.PriceSource
	prices []llmcost.SourcePrice
	err    error
}

func (m *mockFetcher) Source() llmcost.PriceSource {
	return m.source
}

func (m *mockFetcher) Fetch(_ context.Context) ([]llmcost.SourcePrice, error) {
	return m.prices, m.err
}

func TestSyncJobRun(t *testing.T) {
	adapter := &mockAdapter{}
	logger := slog.Default()

	t.Run("fetches and reconciles from multiple sources", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			},
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{
				makePrice("source_b", "openai", "gpt-4", 0.01, 0.03),
			},
		}

		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher1, fetcher2},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Len(t, adapter.upsertedPrices, 1)
	})

	t.Run("continues on fetcher error", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			err:    errors.New("network error"),
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{
				makePrice("source_b", "openai", "gpt-4", 0.01, 0.03),
			},
		}
		fetcher3 := &mockFetcher{
			source: "source_c",
			prices: []llmcost.SourcePrice{
				makePrice("source_c", "openai", "gpt-4", 0.01, 0.03),
			},
		}

		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher1, fetcher2, fetcher3},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Len(t, adapter.upsertedPrices, 1)
	})

	t.Run("normalizes model IDs during sync", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "OpenAI", "GPT-4o-20241022", 0.01, 0.03),
			},
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{
				makePrice("source_b", "openai", "gpt-4o-20241022", 0.01, 0.03),
			},
		}

		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher1, fetcher2},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		require.Len(t, adapter.upsertedPrices, 1)

		upserted := adapter.upsertedPrices[0]
		assert.Equal(t, "openai", string(upserted.Provider))
		assert.Equal(t, "gpt-4o", upserted.ModelID)
	})

	t.Run("no fetchers produces no prices", func(t *testing.T) {
		adapter.upsertedPrices = nil

		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Empty(t, adapter.upsertedPrices)
	})

	t.Run("keeps azure and openai as separate providers", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "azure", "gpt-4", 0.01, 0.03),
				makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			},
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{
				makePrice("source_b", "azure", "gpt-4", 0.01, 0.03),
				makePrice("source_b", "openai", "gpt-4", 0.01, 0.03),
			},
		}

		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher1, fetcher2},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		require.Len(t, adapter.upsertedPrices, 2)

		providers := map[string]bool{}
		for _, p := range adapter.upsertedPrices {
			providers[string(p.Provider)] = true
		}
		assert.True(t, providers["azure"])
		assert.True(t, providers["openai"])
	})

	t.Run("filter excludes prices", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
				makePrice("source_a", "anthropic", "claude-3-5-sonnet", 0.003, 0.015),
			},
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{
				makePrice("source_b", "openai", "gpt-4", 0.01, 0.03),
				makePrice("source_b", "anthropic", "claude-3-5-sonnet", 0.003, 0.015),
			},
		}

		// Only include openai models
		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher1, fetcher2},
			Filter: func(p llmcost.SourcePrice) bool {
				return p.Provider == "openai"
			},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		require.Len(t, adapter.upsertedPrices, 1)
		assert.Equal(t, "openai", string(adapter.upsertedPrices[0].Provider))
	})

	t.Run("nil filter includes all prices", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
				makePrice("source_a", "anthropic", "claude-3-5-sonnet", 0.003, 0.015),
			},
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{
				makePrice("source_b", "openai", "gpt-4", 0.01, 0.03),
				makePrice("source_b", "anthropic", "claude-3-5-sonnet", 0.003, 0.015),
			},
		}

		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher1, fetcher2},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Len(t, adapter.upsertedPrices, 2)
	})

	t.Run("configurable min source agreement", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			},
		}

		job := NewSyncJob(SyncJobConfig{
			Repo:               adapter,
			Logger:             logger,
			Fetchers:           []Fetcher{fetcher1},
			MinSourceAgreement: 1,
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Len(t, adapter.upsertedPrices, 1)
	})
}

func TestDeduplicateSourcePrices(t *testing.T) {
	t.Run("no duplicates unchanged", func(t *testing.T) {
		prices := []llmcost.SourcePrice{
			makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			makePrice("source_a", "anthropic", "claude-3-5-sonnet", 0.003, 0.015),
			makePrice("source_b", "openai", "gpt-4", 0.01, 0.03),
		}

		result := deduplicateSourcePrices(prices)
		assert.Len(t, result, 3)
	})

	t.Run("removes duplicates within same source", func(t *testing.T) {
		prices := []llmcost.SourcePrice{
			makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			makePrice("source_a", "openai", "gpt-4", 0.0101, 0.0301), // duplicate after normalization
		}

		result := deduplicateSourcePrices(prices)
		assert.Len(t, result, 1)
		assert.Equal(t, "openai", string(result[0].Provider))
	})

	t.Run("keeps duplicates across different sources", func(t *testing.T) {
		prices := []llmcost.SourcePrice{
			makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			makePrice("source_b", "openai", "gpt-4", 0.01, 0.03),
		}

		result := deduplicateSourcePrices(prices)
		assert.Len(t, result, 2)
	})

	t.Run("prefers model name without provider prefix", func(t *testing.T) {
		p1 := makePrice("source_a", "openai", "gpt-4", 0.01, 0.03)
		p1.ModelName = "azure/gpt-4"

		p2 := makePrice("source_a", "openai", "gpt-4", 0.01, 0.03)
		p2.ModelName = "GPT-4"

		// First entry has prefix, second doesn't — should pick second
		result := deduplicateSourcePrices([]llmcost.SourcePrice{p1, p2})
		require.Len(t, result, 1)
		assert.Equal(t, "GPT-4", result[0].ModelName)
	})

	t.Run("deterministic tie-breaker when both have clean names", func(t *testing.T) {
		p1 := makePrice("source_a", "openai", "gpt-4", 0.01, 0.03)
		p1.ModelName = "GPT-4 Turbo"

		p2 := makePrice("source_a", "openai", "gpt-4", 0.0101, 0.0301)
		p2.ModelName = "GPT-4"

		// Second entry has lexicographically smaller name — should win regardless of input order
		result := deduplicateSourcePrices([]llmcost.SourcePrice{p1, p2})
		require.Len(t, result, 1)
		assert.Equal(t, "GPT-4", result[0].ModelName)

		// Reverse input order — same winner
		result = deduplicateSourcePrices([]llmcost.SourcePrice{p2, p1})
		require.Len(t, result, 1)
		assert.Equal(t, "GPT-4", result[0].ModelName)
	})

	t.Run("deterministic tie-breaker when both have prefixed names", func(t *testing.T) {
		p1 := makePrice("source_a", "openai", "gpt-4", 0.01, 0.03)
		p1.ModelName = "azure_ai/gpt-4"

		p2 := makePrice("source_a", "openai", "gpt-4", 0.0101, 0.0301)
		p2.ModelName = "azure/gpt-4"

		// Second entry has lexicographically smaller name — should win regardless of input order
		result := deduplicateSourcePrices([]llmcost.SourcePrice{p1, p2})
		require.Len(t, result, 1)
		assert.Equal(t, "azure/gpt-4", result[0].ModelName)

		// Reverse input order — same winner
		result = deduplicateSourcePrices([]llmcost.SourcePrice{p2, p1})
		require.Len(t, result, 1)
		assert.Equal(t, "azure/gpt-4", result[0].ModelName)
	})

	t.Run("empty input returns empty", func(t *testing.T) {
		result := deduplicateSourcePrices(nil)
		assert.Empty(t, result)
	})

	t.Run("multiple models with mixed duplicates", func(t *testing.T) {
		prices := []llmcost.SourcePrice{
			makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			makePrice("source_a", "openai", "gpt-4o", 0.005, 0.015),
			makePrice("source_a", "openai", "gpt-4", 0.0101, 0.0301), // dup of gpt-4
			makePrice("source_a", "anthropic", "claude-3-5-sonnet", 0.003, 0.015),
			makePrice("source_a", "openai", "gpt-4o", 0.005, 0.015), // dup of gpt-4o
		}

		result := deduplicateSourcePrices(prices)
		assert.Len(t, result, 3) // gpt-4, gpt-4o, claude-3-5-sonnet
	})
}

func TestSyncJobDeduplication(t *testing.T) {
	adapter := &mockAdapter{}
	logger := slog.Default()

	t.Run("prevents false agreement from provider normalization", func(t *testing.T) {
		adapter.upsertedPrices = nil

		// source_a returns same model under azure_ai and azure (both normalize to "azure").
		// Without dedup, the reconciler would see 2 entries from source_a and consider them
		// as agreement. With dedup, source_a contributes only 1 entry per (provider, model_id),
		// so 2 sources are still required.
		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "azure_ai", "gpt-4", 0.01, 0.03),
				makePrice("source_a", "azure", "gpt-4", 0.01, 0.03),
			},
		}
		// source_b does NOT have this model — so no agreement should be reached
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{
				makePrice("source_b", "openai", "gpt-3.5-turbo", 0.001, 0.002),
			},
		}

		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher1, fetcher2},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)

		// azure/gpt-4 should NOT reconcile: only source_a has it (dedup collapsed azure_ai → azure)
		// openai/gpt-3.5-turbo should NOT reconcile: only source_b has it
		assert.Empty(t, adapter.upsertedPrices)
	})

	t.Run("min agreement capped at number of fetchers", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{
				makePrice("source_a", "openai", "gpt-4", 0.01, 0.03),
			},
		}

		// MinSourceAgreement defaults to 2, but only 1 fetcher — should cap to 1
		job := NewSyncJob(SyncJobConfig{
			Repo:     adapter,
			Logger:   logger,
			Fetchers: []Fetcher{fetcher},
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Len(t, adapter.upsertedPrices, 1)
	})
}

func TestSyncJobDefaultFetchers(t *testing.T) {
	fetchers := DefaultFetchers(nil)
	assert.GreaterOrEqual(t, len(fetchers), 1)

	sources := make([]llmcost.PriceSource, len(fetchers))
	for i, f := range fetchers {
		sources[i] = f.Source()
	}

	assert.Contains(t, sources, llmcost.PriceSource("models_dev"))
}

func TestSyncJobTolerancePassthrough(t *testing.T) {
	adapter := &mockAdapter{}
	logger := slog.Default()

	t.Run("strict tolerance rejects slight differences", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{{
				Source:   "source_a",
				Provider: "openai",
				ModelID:  "gpt-4",
				Pricing: llmcost.ModelPricing{
					InputPerToken:  alpacadecimal.NewFromFloat(0.0100),
					OutputPerToken: alpacadecimal.NewFromFloat(0.0300),
				},
			}},
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{{
				Source:   "source_b",
				Provider: "openai",
				ModelID:  "gpt-4",
				Pricing: llmcost.ModelPricing{
					InputPerToken:  alpacadecimal.NewFromFloat(0.0101),
					OutputPerToken: alpacadecimal.NewFromFloat(0.0300),
				},
			}},
		}

		// Zero tolerance: prices must match exactly
		job := NewSyncJob(SyncJobConfig{
			Repo:           adapter,
			Logger:         logger,
			Fetchers:       []Fetcher{fetcher1, fetcher2},
			PriceTolerance: 0,
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Empty(t, adapter.upsertedPrices)
	})

	t.Run("loose tolerance accepts slight differences", func(t *testing.T) {
		adapter.upsertedPrices = nil

		fetcher1 := &mockFetcher{
			source: "source_a",
			prices: []llmcost.SourcePrice{{
				Source:   "source_a",
				Provider: "openai",
				ModelID:  "gpt-4",
				Pricing: llmcost.ModelPricing{
					InputPerToken:  alpacadecimal.NewFromFloat(0.0100),
					OutputPerToken: alpacadecimal.NewFromFloat(0.0300),
				},
			}},
		}
		fetcher2 := &mockFetcher{
			source: "source_b",
			prices: []llmcost.SourcePrice{{
				Source:   "source_b",
				Provider: "openai",
				ModelID:  "gpt-4",
				Pricing: llmcost.ModelPricing{
					InputPerToken:  alpacadecimal.NewFromFloat(0.0101),
					OutputPerToken: alpacadecimal.NewFromFloat(0.0300),
				},
			}},
		}

		// 5% tolerance: should accept
		job := NewSyncJob(SyncJobConfig{
			Repo:           adapter,
			Logger:         logger,
			Fetchers:       []Fetcher{fetcher1, fetcher2},
			PriceTolerance: 0.05,
		})

		err := job.Run(context.Background())
		require.NoError(t, err)
		assert.Len(t, adapter.upsertedPrices, 1)
	})
}