Reduce Redis key size

Author: ChrisBr

ruby/lib/ci/queue/redis.rb

@@ -11,6 +11,7 @@
 require 'ci/queue/redis/supervisor'
 require 'ci/queue/redis/grind_supervisor'
 require 'ci/queue/redis/test_time_record'
+require 'ci/queue/redis/key_shortener'
 
 module CI
   module Queue

ruby/lib/ci/queue/redis/base.rb

@@ -214,7 +214,7 @@ def measure
         end
 
         def key(*args)
-          ['build', build_id, *args].join(':')
+          KeyShortener.key(config.build_id, *args)
         end
 
         def build_id

ruby/lib/ci/queue/redis/build_record.rb

@@ -138,8 +138,45 @@ def record_stats(stats, pipeline: redis)
           end
         end
 
+        SUFFIX_ALIASES = {
+          'running' => 'r',
+          'processed' => 'p',
+          'queue' => 'q',
+          'owners' => 'o',
+          'error-reports' => 'e',
+          'requeues-count' => 'rc',
+          'assertions' => 'a',
+          'errors' => 'er',
+          'failures' => 'f',
+          'skips' => 's',
+          'requeues' => 'rq',
+          'total_time' => 't',
+          'test_failed_count' => 'fc',
+          'completed' => 'c',
+          'master-status' => 'm',
+          'created-at' => 'ca',
+          'workers' => 'w',
+          'worker' => 'w',
+          'warnings' => 'wn',
+          'worker-errors' => 'we',
+          'flaky-reports' => 'fl',
+        }.freeze
+
+        # We're transforming the key to a shorter format to minimize inter zone network traffic.
+        #
+        # Strategy:
+        # - Shorten prefix: 'b' instead of 'build'
+        # - Hash UUID: 8-char MD5 instead of 36-char UUID
+        # - Alias suffixes: single letters instead of full words
+        #
+        # Example:
+        #   build:unit:019aef0e-c010-433e-b706-c658d3c16372:running (55 bytes)
+        #   -> b:f03d3bef:r (13 bytes, 76% reduction)
         def key(*args)
-          ['build', config.build_id, *args].join(':')
+          digest = Digest::MD5.hexdigest(config.build_id)[0..7]
+          shortened_args = args.map { |arg| KeyShortener::SUFFIX_ALIASES[arg] || arg }
+
+          ['b', digest, *shortened_args].join(':')
         end
       end
     end

ruby/lib/ci/queue/redis/grind_record.rb

@@ -52,7 +52,10 @@ def pop_warnings
         attr_reader :redis, :config
 
         def key(*args)
-          ['build', config.build_id, *args].join(':')
+          digest = Digest::MD5.hexdigest(config.build_id)[0..7]
+          shortened_args = args.map { |arg| KeyShortener::SUFFIX_ALIASES[arg] || arg }
+
+          ['b', digest, *shortened_args].join(':')
         end
 
         def record_stats(stats, pipeline: redis)

ruby/lib/ci/queue/redis/key_shortener.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+require 'digest/md5'
+
+module CI
+  module Queue
+    module Redis
+      module KeyShortener
+        # Suffix mapping for common key patterns
+        SUFFIX_ALIASES = {
+          'running' => 'r',
+          'processed' => 'p',
+          'queue' => 'q',
+          'owners' => 'o',
+          'error-reports' => 'e',
+          'requeues-count' => 'rc',
+          'assertions' => 'a',
+          'errors' => 'er',
+          'failures' => 'f',
+          'skips' => 's',
+          'requeues' => 'rq',
+          'total_time' => 't',
+          'test_failed_count' => 'fc',
+          'completed' => 'c',
+          'master-status' => 'm',
+          'created-at' => 'ca',
+          'workers' => 'w',
+          'worker' => 'w',
+          'warnings' => 'wn',
+          'worker-errors' => 'we',
+          'flaky-reports' => 'fl',
+        }.freeze
+
+        def self.key(build_id, *args)
+          digest = Digest::MD5.hexdigest(build_id)[0..7]
+          shortened_args = args.map { |arg| SUFFIX_ALIASES[arg] || arg }
+
+          ['b', digest, *shortened_args].join(':')
+        end
+      end
+    end
+  end
+end

ruby/test/ci/queue/redis_test.rb

@@ -207,9 +207,9 @@ def test_acknowledge_returns_false_if_the_test_was_picked_up_by_another_worker
   end
 
   def test_workers_register
-    assert_equal 1, @redis.scard(('build:42:workers'))
+    assert_equal 1, @redis.scard(CI::Queue::Redis::KeyShortener.key('42', 'workers'))
     worker(2)
-    assert_equal 2, @redis.scard(('build:42:workers'))
+    assert_equal 2, @redis.scard(CI::Queue::Redis::KeyShortener.key('42', 'workers'))
   end
 
   def test_timeout_warning

ruby/test/integration/minitest_redis_test.rb

@@ -139,7 +139,7 @@ def test_debug_log
           )
         end
 
-      assert_includes File.read(log_file.path), 'INFO -- : Finished \'["exists", "build:1:worker:1:queue"]\': 0'
+      assert_includes File.read(log_file.path), 'INFO -- : Finished \'["exists", "b:c4ca4238:w:1:q"]\': 0'
       assert_empty err
       result = normalize(out.lines.last.strip)
       assert_equal '--- Ran 11 tests, 8 assertions, 2 failures, 1 errors, 1 skips, 4 requeues in X.XXs', result
@@ -444,7 +444,7 @@ def test_retry_fails_when_test_run_is_expired
       assert_equal 'Ran 100 tests, 100 assertions, 0 failures, 0 errors, 0 skips, 0 requeues in X.XXs', output
 
       one_day = 60 * 60 * 24
-      key = ['build', "1", "created-at"].join(':')
+      key = CI::Queue::Redis::KeyShortener.key("1", "created-at")
       @redis.set(key, Time.now - one_day)
 
       out, err = capture_subprocess_io do