diff --git a/cache/cache_test.go b/cache/cache_test.go index 2be551e7..60213366 100644 --- a/cache/cache_test.go +++ b/cache/cache_test.go @@ -87,7 +87,7 @@ func testCacheOperation( prepareFakeCreateAdapter(t, operationName, tc) config := prepareFakeConfig(tc) - adaptor := GetAdapter(config, 3600*time.Second, "shorttoken", "10", tc.key) + adaptor := GetAdapter(config, 3600*time.Second, "shorttoken", "10", tc.key, false) generatedURL := operation(ctx, adaptor) assert.Equal(t, tc.expectedURL, generatedURL.URL) @@ -179,9 +179,10 @@ func defaultCacheConfig() *cacheconfig.Config { } type generateObjectNameTestCase struct { - key string - path string - shared bool + key string + path string + shared bool + sharded bool expectedObjectName string expectedError string @@ -235,6 +236,23 @@ func TestGenerateObjectName(t *testing.T) { key: "../10-outside", expectedError: "computed cache path outside of project bucket", }, + "sharded key uses first two chars as prefix": { + key: "d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", + sharded: true, + expectedObjectName: "runner/longtoken/project/10/d0/d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", + }, + "sharded key with path prefix": { + key: "d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", + path: "builds", + sharded: true, + expectedObjectName: "builds/runner/longtoken/project/10/d0/d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", + }, + "sharded key with shared runner": { + key: "d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", + shared: true, + sharded: true, + expectedObjectName: "project/10/d0/d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", + }, } for name, tc := range tests { @@ -255,7 +273,7 @@ func TestGenerateObjectName(t *testing.T) { createAdapter = oldCreateAdapter }) - adapter := GetAdapter(cache, 3600*time.Second, "longtoken", "10", tc.key) + adapter := GetAdapter(cache, 3600*time.Second, "longtoken", "10", tc.key, tc.sharded) if tc.expectedError != "" { // The error/warning cases return a nopAdaptor and log instead of returning an error diff --git a/shells/abstract_test.go b/shells/abstract_test.go index de6342fb..153d2be4 100644 --- a/shells/abstract_test.go +++ b/shells/abstract_test.go @@ -1642,6 +1642,22 @@ func getCacheKeyHasher(hash bool) func(string) string { } } +// getShardedObjectKey returns a function that, given a (hashed) cache key, +// returns the object path component used by GetAdapter. When sharded is true +// (i.e. FF_HASH_CACHE_KEYS is on), the first two hex characters are inserted +// as a prefix: "/". Otherwise the key is returned unchanged. +func getShardedObjectKey(sharded bool) func(string) string { + if !sharded { + return func(key string) string { return key } + } + return func(key string) string { + if len(key) < 2 { + return key + } + return key[:2] + "/" + key + } +} + func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { const cacheEnvFile = "/some/path/to/runner-cache-env" @@ -1865,6 +1881,7 @@ func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { for _, hashCacheKeys := range []bool{false, true} { hashed := getCacheKeyHasher(hashCacheKeys) + shardedObjectPath := getShardedObjectKey(hashCacheKeys) t.Run(fmt.Sprintf("%s:%t", featureflags.HashCacheKeys, hashCacheKeys), func(t *testing.T) { for tn, tc := range tests { @@ -1943,7 +1960,7 @@ func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { "--timeout", "10", "--url", - fmt.Sprintf("test://download/project/1000/%s", expectedHashedCacheKey), + fmt.Sprintf("test://download/project/1000/%s", shardedObjectPath(expectedHashedCacheKey)), ).Once() } else { mockWriter.On("DotEnvVariables", "gitlab_runner_cache_env", mock.Anything).Return(cacheEnvFile).Once() @@ -1955,7 +1972,7 @@ func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { "--timeout", "10", "--gocloud-url", - fmt.Sprintf("gocloud://test/project/1000/%s", expectedHashedCacheKey), + fmt.Sprintf("gocloud://test/project/1000/%s", shardedObjectPath(expectedHashedCacheKey)), "--env-file", cacheEnvFile, ).Once() } @@ -2081,6 +2098,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeys(t *testing.T) { for _, hashedCacheKey := range []bool{false, true} { hashed := getCacheKeyHasher(hashedCacheKey) + shardedObjectPath := getShardedObjectKey(hashedCacheKey) t.Run(fmt.Sprintf("%s:%t", featureflags.HashCacheKeys, hashedCacheKey), func(t *testing.T) { for tn, tc := range tests { @@ -2149,7 +2167,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeys(t *testing.T) { "--timeout", "10", "--url", - fmt.Sprintf("test://download/project/1000/%s", hashedCacheKey), + fmt.Sprintf("test://download/project/1000/%s", shardedObjectPath(hashedCacheKey)), ).Once() mockWriter.On("Noticef", "Successfully extracted cache").Once() mockWriter.On("Else").Once() @@ -2265,6 +2283,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeysWithCleanup(t *testin for _, hashedCacheKey := range []bool{false, true} { hashed := getCacheKeyHasher(hashedCacheKey) + shardedObjectPath := getShardedObjectKey(hashedCacheKey) t.Run(fmt.Sprintf("%s:%t", featureflags.HashCacheKeys, hashedCacheKey), func(t *testing.T) { for tn, tc := range tests { @@ -2337,7 +2356,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeysWithCleanup(t *testin "--timeout", "10", "--url", - fmt.Sprintf("test://download/project/1000/%s", hashedCacheKey), + fmt.Sprintf("test://download/project/1000/%s", shardedObjectPath(hashedCacheKey)), ).Once() mockWriter.On("Noticef", "Successfully extracted cache").Once() mockWriter.On("Else").Once()