| |
| |
| |
| |
| @@ -87,7 +87,7 @@ func testCacheOperation( |
| prepareFakeCreateAdapter(t, operationName, tc) |
| |
| config := prepareFakeConfig(tc) |
| - adaptor := GetAdapter(config, 3600*time.Second, "shorttoken", "10", tc.key) |
| + adaptor := GetAdapter(config, 3600*time.Second, "shorttoken", "10", tc.key, false) |
| generatedURL := operation(ctx, adaptor) |
| assert.Equal(t, tc.expectedURL, generatedURL.URL) |
| |
| @@ -179,9 +179,10 @@ func defaultCacheConfig() *cacheconfig.Config { |
| } |
| |
| type generateObjectNameTestCase struct { |
| - key string |
| - path string |
| - shared bool |
| + key string |
| + path string |
| + shared bool |
| + sharded bool |
| |
| expectedObjectName string |
| expectedError string |
| @@ -235,6 +236,23 @@ func TestGenerateObjectName(t *testing.T) { |
| key: "../10-outside", |
| expectedError: "computed cache path outside of project bucket", |
| }, |
| + "sharded key uses first two chars as prefix": { |
| + key: "d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", |
| + sharded: true, |
| + expectedObjectName: "runner/longtoken/project/10/d0/d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", |
| + }, |
| + "sharded key with path prefix": { |
| + key: "d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", |
| + path: "builds", |
| + sharded: true, |
| + expectedObjectName: "builds/runner/longtoken/project/10/d0/d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", |
| + }, |
| + "sharded key with shared runner": { |
| + key: "d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", |
| + shared: true, |
| + sharded: true, |
| + expectedObjectName: "project/10/d0/d03a852ba491ba611e907b1ef60ad5c4516a05b8f3aae6abb77f42bc60325aed", |
| + }, |
| } |
| |
| for name, tc := range tests { |
| @@ -255,7 +273,7 @@ func TestGenerateObjectName(t *testing.T) { |
| createAdapter = oldCreateAdapter |
| }) |
| |
| - adapter := GetAdapter(cache, 3600*time.Second, "longtoken", "10", tc.key) |
| + adapter := GetAdapter(cache, 3600*time.Second, "longtoken", "10", tc.key, tc.sharded) |
| |
| if tc.expectedError != "" { |
| // The error/warning cases return a nopAdaptor and log instead of returning an error |
| |
| |
| |
| |
| @@ -1642,6 +1642,22 @@ func getCacheKeyHasher(hash bool) func(string) string { |
| } |
| } |
| |
| +// getShardedObjectKey returns a function that, given a (hashed) cache key, |
| +// returns the object path component used by GetAdapter. When sharded is true |
| +// (i.e. FF_HASH_CACHE_KEYS is on), the first two hex characters are inserted |
| +// as a prefix: "<shard>/<key>". Otherwise the key is returned unchanged. |
| +func getShardedObjectKey(sharded bool) func(string) string { |
| + if !sharded { |
| + return func(key string) string { return key } |
| + } |
| + return func(key string) string { |
| + if len(key) < 2 { |
| + return key |
| + } |
| + return key[:2] + "/" + key |
| + } |
| +} |
| + |
| func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { |
| const cacheEnvFile = "/some/path/to/runner-cache-env" |
| |
| @@ -1865,6 +1881,7 @@ func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { |
| |
| for _, hashCacheKeys := range []bool{false, true} { |
| hashed := getCacheKeyHasher(hashCacheKeys) |
| + shardedObjectPath := getShardedObjectKey(hashCacheKeys) |
| |
| t.Run(fmt.Sprintf("%s:%t", featureflags.HashCacheKeys, hashCacheKeys), func(t *testing.T) { |
| for tn, tc := range tests { |
| @@ -1943,7 +1960,7 @@ func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { |
| "--timeout", |
| "10", |
| "--url", |
| - fmt.Sprintf("test://download/project/1000/%s", expectedHashedCacheKey), |
| + fmt.Sprintf("test://download/project/1000/%s", shardedObjectPath(expectedHashedCacheKey)), |
| ).Once() |
| } else { |
| mockWriter.On("DotEnvVariables", "gitlab_runner_cache_env", mock.Anything).Return(cacheEnvFile).Once() |
| @@ -1955,7 +1972,7 @@ func TestAbstractShell_extractCacheWithDefaultFallbackKey(t *testing.T) { |
| "--timeout", |
| "10", |
| "--gocloud-url", |
| - fmt.Sprintf("gocloud://test/project/1000/%s", expectedHashedCacheKey), |
| + fmt.Sprintf("gocloud://test/project/1000/%s", shardedObjectPath(expectedHashedCacheKey)), |
| "--env-file", cacheEnvFile, |
| ).Once() |
| } |
| @@ -2081,6 +2098,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeys(t *testing.T) { |
| |
| for _, hashedCacheKey := range []bool{false, true} { |
| hashed := getCacheKeyHasher(hashedCacheKey) |
| + shardedObjectPath := getShardedObjectKey(hashedCacheKey) |
| |
| t.Run(fmt.Sprintf("%s:%t", featureflags.HashCacheKeys, hashedCacheKey), func(t *testing.T) { |
| for tn, tc := range tests { |
| @@ -2149,7 +2167,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeys(t *testing.T) { |
| "--timeout", |
| "10", |
| "--url", |
| - fmt.Sprintf("test://download/project/1000/%s", hashedCacheKey), |
| + fmt.Sprintf("test://download/project/1000/%s", shardedObjectPath(hashedCacheKey)), |
| ).Once() |
| mockWriter.On("Noticef", "Successfully extracted cache").Once() |
| mockWriter.On("Else").Once() |
| @@ -2265,6 +2283,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeysWithCleanup(t *testin |
| |
| for _, hashedCacheKey := range []bool{false, true} { |
| hashed := getCacheKeyHasher(hashedCacheKey) |
| + shardedObjectPath := getShardedObjectKey(hashedCacheKey) |
| |
| t.Run(fmt.Sprintf("%s:%t", featureflags.HashCacheKeys, hashedCacheKey), func(t *testing.T) { |
| for tn, tc := range tests { |
| @@ -2337,7 +2356,7 @@ func TestAbstractShell_extractCacheWithMultipleFallbackKeysWithCleanup(t *testin |
| "--timeout", |
| "10", |
| "--url", |
| - fmt.Sprintf("test://download/project/1000/%s", hashedCacheKey), |
| + fmt.Sprintf("test://download/project/1000/%s", shardedObjectPath(hashedCacheKey)), |
| ).Once() |
| mockWriter.On("Noticef", "Successfully extracted cache").Once() |
| mockWriter.On("Else").Once() |
|
|