From 55c44d2b998a7e6dcc67b6a03d366e49ae72c7c2 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Wed, 18 Mar 2026 02:47:05 -0400 Subject: [PATCH 1/2] Add configurable snapshot compression with async standby support --- cmd/api/api/api_test.go | 2 +- cmd/api/api/instances.go | 82 ++- cmd/api/api/snapshots.go | 33 +- cmd/api/config/config.go | 29 + integration/systemd_test.go | 2 +- integration/vgpu_test.go | 2 +- lib/builds/manager_test.go | 2 +- lib/devices/gpu_e2e_test.go | 2 +- lib/devices/gpu_inference_test.go | 2 +- lib/devices/gpu_module_test.go | 4 +- lib/instances/create.go | 6 + lib/instances/firecracker_test.go | 8 +- lib/instances/fork.go | 6 +- lib/instances/manager.go | 13 +- lib/instances/manager_darwin_test.go | 4 +- lib/instances/manager_test.go | 6 +- lib/instances/network_test.go | 2 +- lib/instances/qemu_test.go | 4 +- lib/instances/resource_limits_test.go | 2 +- lib/instances/restore.go | 3 + lib/instances/snapshot.go | 33 +- lib/instances/snapshot_compression.go | 410 +++++++++++++ lib/instances/snapshot_compression_test.go | 118 ++++ .../snapshot_integration_scenario_test.go | 2 +- lib/instances/standby.go | 26 + lib/instances/types.go | 21 +- lib/oapi/oapi.go | 549 +++++++++++------- lib/providers/providers.go | 11 +- lib/snapshot/types.go | 45 +- openapi.yaml | 72 +++ 30 files changed, 1244 insertions(+), 257 deletions(-) create mode 100644 lib/instances/snapshot_compression.go create mode 100644 lib/instances/snapshot_compression_test.go diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 38b7eadd..b109f646 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -43,7 +43,7 @@ func newTestService(t *testing.T) *ApiService { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Initialize network manager (creates bridge for network-enabled tests) if err := networkMgr.Initialize(ctx(), nil); err != nil { diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index aa3b4524..80446e1f 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/samber/lo" ) @@ -264,6 +265,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, } + if request.Body.SnapshotPolicy != nil { + snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_snapshot_policy", + Message: err.Error(), + }, nil + } + domainReq.SnapshotPolicy = snapshotPolicy + } inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { @@ -401,7 +412,19 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn } log := logger.FromContext(ctx) - result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id) + standbyReq := instances.StandbyInstanceRequest{} + if request.Body != nil && request.Body.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_snapshot_compression", + Message: err.Error(), + }, nil + } + standbyReq.Compression = compression + } + + result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id, standbyReq) if err != nil { switch { case errors.Is(err, instances.ErrInvalidState): @@ -858,6 +881,10 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if len(inst.Metadata) > 0 { oapiInst.Metadata = toOAPIMetadata(inst.Metadata) } + if inst.SnapshotPolicy != nil { + oapiPolicy, _ := toOAPISnapshotPolicy(*inst.SnapshotPolicy) + oapiInst.SnapshotPolicy = &oapiPolicy + } // Convert volume attachments if len(inst.Volumes) > 0 { @@ -892,3 +919,56 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { return oapiInst } + +func toDomainSnapshotCompressionConfig(cfg oapi.SnapshotCompressionConfig) (*snapshot.SnapshotCompressionConfig, error) { + out := &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != nil { + out.Algorithm = snapshot.SnapshotCompressionAlgorithm(*cfg.Algorithm) + } + if cfg.Level != nil { + level := *cfg.Level + out.Level = &level + } + return out, nil +} + +func toInstanceSnapshotPolicy(policy oapi.SnapshotPolicy) (*instances.SnapshotPolicy, error) { + out := &instances.SnapshotPolicy{} + if policy.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*policy.Compression) + if err != nil { + return nil, err + } + out.Compression = compression + } + return out, nil +} + +func toOAPISnapshotCompressionConfig(cfg snapshot.SnapshotCompressionConfig) (oapi.SnapshotCompressionConfig, error) { + out := oapi.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != "" { + algo := oapi.SnapshotCompressionConfigAlgorithm(cfg.Algorithm) + out.Algorithm = &algo + } + if cfg.Level != nil { + level := *cfg.Level + out.Level = &level + } + return out, nil +} + +func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) (oapi.SnapshotPolicy, error) { + out := oapi.SnapshotPolicy{} + if policy.Compression != nil { + compression, err := toOAPISnapshotCompressionConfig(*policy.Compression) + if err != nil { + return oapi.SnapshotPolicy{}, err + } + out.Compression = &compression + } + return out, nil +} diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index b5a70d0c..04fd47fd 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -10,6 +10,7 @@ import ( mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" + "github.com/kernel/hypeman/lib/snapshot" "github.com/samber/lo" ) @@ -27,11 +28,20 @@ func (s *ApiService) CreateInstanceSnapshot(ctx context.Context, request oapi.Cr if request.Body.Name != nil { name = *request.Body.Name } + var compression *snapshot.SnapshotCompressionConfig + if request.Body.Compression != nil { + var err error + compression, err = toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.CreateInstanceSnapshot400JSONResponse{Code: "invalid_snapshot_compression", Message: err.Error()}, nil + } + } result, err := s.InstanceManager.CreateSnapshot(ctx, inst.Id, instances.CreateSnapshotRequest{ - Kind: instances.SnapshotKind(request.Body.Kind), - Name: name, - Metadata: toMapMetadata(request.Body.Metadata), + Kind: instances.SnapshotKind(request.Body.Kind), + Name: name, + Metadata: toMapMetadata(request.Body.Metadata), + Compression: compression, }) if err != nil { log := logger.FromContext(ctx) @@ -207,6 +217,23 @@ func snapshotToOAPI(snapshot instances.Snapshot) oapi.Snapshot { SizeBytes: snapshot.SizeBytes, Name: lo.ToPtr(snapshot.Name), } + if snapshot.CompressionState != "" { + state := oapi.SnapshotCompressionState(snapshot.CompressionState) + out.CompressionState = &state + } + if snapshot.CompressionError != "" { + out.CompressionError = lo.ToPtr(snapshot.CompressionError) + } + if snapshot.Compression != nil { + compression, _ := toOAPISnapshotCompressionConfig(*snapshot.Compression) + out.Compression = &compression + } + if snapshot.CompressedSizeBytes != nil { + out.CompressedSizeBytes = snapshot.CompressedSizeBytes + } + if snapshot.UncompressedSizeBytes != nil { + out.UncompressedSizeBytes = snapshot.UncompressedSizeBytes + } if snapshot.Name == "" { out.Name = nil } diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 4ee3d442..1a0b1275 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -158,6 +158,18 @@ type HypervisorConfig struct { FirecrackerBinaryPath string `koanf:"firecracker_binary_path"` } +// SnapshotCompressionDefaultConfig holds default snapshot compression settings. +type SnapshotCompressionDefaultConfig struct { + Enabled bool `koanf:"enabled"` + Algorithm string `koanf:"algorithm"` + Level int `koanf:"level"` +} + +// SnapshotConfig holds snapshot defaults. +type SnapshotConfig struct { + CompressionDefault SnapshotCompressionDefaultConfig `koanf:"compression_default"` +} + // GPUConfig holds GPU-related settings. type GPUConfig struct { ProfileCacheTTL string `koanf:"profile_cache_ttl"` @@ -183,6 +195,7 @@ type Config struct { Oversubscription OversubscriptionConfig `koanf:"oversubscription"` Capacity CapacityConfig `koanf:"capacity"` Hypervisor HypervisorConfig `koanf:"hypervisor"` + Snapshot SnapshotConfig `koanf:"snapshot"` GPU GPUConfig `koanf:"gpu"` } @@ -302,6 +315,14 @@ func defaultConfig() *Config { FirecrackerBinaryPath: "", }, + Snapshot: SnapshotConfig{ + CompressionDefault: SnapshotCompressionDefaultConfig{ + Enabled: false, + Algorithm: "zstd", + Level: 1, + }, + }, + GPU: GPUConfig{ ProfileCacheTTL: "30m", }, @@ -400,5 +421,13 @@ func (c *Config) Validate() error { if c.Build.Timeout <= 0 { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } + if c.Snapshot.CompressionDefault.Level < 1 { + return fmt.Errorf("snapshot.compression_default.level must be >= 1, got %d", c.Snapshot.CompressionDefault.Level) + } + switch strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) { + case "", "zstd", "lz4": + default: + return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", c.Snapshot.CompressionDefault.Algorithm) + } return nil } diff --git a/integration/systemd_test.go b/integration/systemd_test.go index 986ebb11..7ae458bf 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -68,7 +68,7 @@ func TestSystemdMode(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Cleanup any orphaned instances t.Cleanup(func() { diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 17a92ad1..9fd608c5 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -77,7 +77,7 @@ func TestVGPU(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Track instance ID for cleanup var instanceID string diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index a3765d5f..1a68ded3 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -108,7 +108,7 @@ func (m *mockInstanceManager) ForkSnapshot(ctx context.Context, snapshotID strin return nil, instances.ErrNotFound } -func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string) (*instances.Instance, error) { +func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string, req instances.StandbyInstanceRequest) (*instances.Instance, error) { return nil, nil } diff --git a/lib/devices/gpu_e2e_test.go b/lib/devices/gpu_e2e_test.go index 86708446..a7fe6390 100644 --- a/lib/devices/gpu_e2e_test.go +++ b/lib/devices/gpu_e2e_test.go @@ -79,7 +79,7 @@ func TestGPUPassthrough(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Discover available GPUs t.Log("Step 1: Discovering available GPUs...") diff --git a/lib/devices/gpu_inference_test.go b/lib/devices/gpu_inference_test.go index c99d1b39..f521158e 100644 --- a/lib/devices/gpu_inference_test.go +++ b/lib/devices/gpu_inference_test.go @@ -116,7 +116,7 @@ func TestGPUInference(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Build custom CUDA+Ollama image t.Log("Step 1: Building custom CUDA+Ollama Docker image...") diff --git a/lib/devices/gpu_module_test.go b/lib/devices/gpu_module_test.go index cd9f1b76..f39d4690 100644 --- a/lib/devices/gpu_module_test.go +++ b/lib/devices/gpu_module_test.go @@ -80,7 +80,7 @@ func TestNVIDIAModuleLoading(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Find an NVIDIA GPU t.Log("Step 1: Discovering available GPUs...") @@ -326,7 +326,7 @@ func TestNVMLDetection(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Check if ollama-cuda:test image exists in Docker t.Log("Step 1: Checking for ollama-cuda:test Docker image...") diff --git a/lib/instances/create.go b/lib/instances/create.go index 1f762ff3..dfc63cdc 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -313,6 +313,7 @@ func (m *manager) createInstance( Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, SkipGuestAgent: req.SkipGuestAgent, + SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), } // 12. Ensure directories @@ -470,6 +471,11 @@ func validateCreateRequest(req CreateInstanceRequest) error { if err := tags.Validate(req.Metadata); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } + if req.SnapshotPolicy != nil && req.SnapshotPolicy.Compression != nil { + if _, err := normalizeCompressionConfig(req.SnapshotPolicy.Compression); err != nil { + return err + } + } // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index 940c1786..4ecefd29 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -46,7 +46,7 @@ func setupTestManagerForFirecracker(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -114,7 +114,7 @@ func TestFirecrackerStandbyAndRestore(t *testing.T) { require.NoError(t, err) assert.Equal(t, StateRunning, inst.State) - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) @@ -164,7 +164,7 @@ func TestFirecrackerStopClearsStaleSnapshot(t *testing.T) { require.Equal(t, StateRunning, inst.State) // Establish a realistic standby/restore lifecycle first. - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) require.Equal(t, StateStandby, inst.State) require.True(t, inst.HasSnapshot) @@ -264,7 +264,7 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) { require.Equal(t, 0, exitCode) require.Contains(t, output, "Connection successful") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 60d3a5cf..84d319fd 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -56,7 +56,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR log.InfoContext(ctx, "fork from running requested; transitioning source to standby", "source_instance_id", id, "hypervisor", source.HypervisorType) - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, "", fmt.Errorf("standby source instance: %w", err) } @@ -421,7 +421,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { return nil, fmt.Errorf("start forked instance for standby transition: %w", err) } - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) } case StateStandby: switch target { @@ -436,7 +436,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateRunning: switch target { case StateStandby: - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) case StateStopped: return returnWithReadiness(m.stopInstance(ctx, forkID)) } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 3b581e83..d31ff1cb 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -31,7 +31,7 @@ type Manager interface { DeleteSnapshot(ctx context.Context, snapshotID string) error ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) ForkSnapshot(ctx context.Context, snapshotID string, req ForkSnapshotRequest) (*Instance, error) - StandbyInstance(ctx context.Context, id string) (*Instance, error) + StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) RestoreInstance(ctx context.Context, id string) (*Instance, error) RestoreSnapshot(ctx context.Context, id string, snapshotID string, req RestoreSnapshotRequest) (*Instance, error) StopInstance(ctx context.Context, id string) (*Instance, error) @@ -80,6 +80,9 @@ type manager struct { instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks hostTopology *HostTopology // Cached host CPU topology metrics *Metrics + snapshotDefaults SnapshotPolicy + compressionMu sync.Mutex + compressionJobs map[string]*compressionJob // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -92,7 +95,7 @@ var platformStarters = make(map[hypervisor.Type]hypervisor.VMStarter) // NewManager creates a new instances manager. // If meter is nil, metrics are disabled. // defaultHypervisor specifies which hypervisor to use when not specified in requests. -func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, meter metric.Meter, tracer trace.Tracer) Manager { +func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, snapshotDefaults SnapshotPolicy, meter metric.Meter, tracer trace.Tracer) Manager { // Validate and default the hypervisor type if defaultHypervisor == "" { defaultHypervisor = hypervisor.TypeCloudHypervisor @@ -116,6 +119,8 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste hostTopology: detectHostTopology(), // Detect and cache host topology vmStarters: vmStarters, defaultHypervisor: defaultHypervisor, + snapshotDefaults: snapshotDefaults, + compressionJobs: make(map[string]*compressionJob), } // Initialize metrics if meter is provided @@ -241,11 +246,11 @@ func (m *manager) ForkSnapshot(ctx context.Context, snapshotID string, req ForkS } // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) -func (m *manager) StandbyInstance(ctx context.Context, id string) (*Instance, error) { +func (m *manager) StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) { lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.standbyInstance(ctx, id) + return m.standbyInstance(ctx, id, req, false) } // RestoreInstance restores an instance from standby diff --git a/lib/instances/manager_darwin_test.go b/lib/instances/manager_darwin_test.go index d38c24d8..5163543b 100644 --- a/lib/instances/manager_darwin_test.go +++ b/lib/instances/manager_darwin_test.go @@ -57,7 +57,7 @@ func setupVZTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -470,7 +470,7 @@ func TestVZStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Putting instance in standby...") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 641cd6a9..21ab1a28 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -55,7 +55,7 @@ func setupTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -1186,7 +1186,7 @@ func TestStorageOperations(t *testing.T) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Test metadata doesn't exist initially _, err := manager.loadMetadata("nonexistent") @@ -1304,7 +1304,7 @@ func TestStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index af7e25c4..68a43ae7 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -130,7 +130,7 @@ func TestCreateInstanceWithNetwork(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index ab5aa853..2fbb5392 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -53,7 +53,7 @@ func setupTestManagerForQEMU(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -826,7 +826,7 @@ func TestQEMUStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index d34b9c45..32cad66c 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -176,7 +176,7 @@ func createTestManager(t *testing.T, limits ResourceLimits) *manager { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 0, nil) - return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil).(*manager) + return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", SnapshotPolicy{}, nil, nil).(*manager) } func TestResourceLimits_StructValues(t *testing.T) { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 1ff09c55..43496e6c 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,6 +69,9 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) + if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id)); err != nil { + return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) + } starter, err := m.getVMStarter(stored.HypervisorType) if err != nil { return nil, fmt.Errorf("get vm starter: %w", err) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index 4df46078..4839d888 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -91,7 +91,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before running snapshot"); err != nil { return nil, err } - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, fmt.Errorf("standby source instance: %w", err) } restoreSource = true @@ -138,9 +138,27 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone + effectiveCompression, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + if err != nil { + return nil, err + } + if effectiveCompression.Enabled { + rec.Snapshot.Compression = cloneCompressionConfig(&effectiveCompression) + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateCompressing + } if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } + if effectiveCompression.Enabled { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForSnapshot(snapshotID), + OwnerID: stored.Id, + SnapshotID: snapshotID, + SnapshotDir: snapshotGuestDir, + Policy: effectiveCompression, + }) + } cu.Release() log.InfoContext(ctx, "snapshot created", "instance_id", id, "snapshot_id", snapshotID, "kind", req.Kind) return &rec.Snapshot, nil @@ -170,6 +188,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } @@ -221,6 +240,9 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } + m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) + m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { return nil, err } @@ -335,6 +357,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS }) defer cu.Clean() + m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) + m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork from snapshot requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) @@ -461,6 +486,12 @@ func validateCreateSnapshotRequest(req CreateSnapshotRequest) error { if req.Kind != SnapshotKindStandby && req.Kind != SnapshotKindStopped { return fmt.Errorf("%w: kind must be one of %s, %s", ErrInvalidRequest, SnapshotKindStandby, SnapshotKindStopped) } + if req.Kind == SnapshotKindStopped && req.Compression != nil && req.Compression.Enabled { + return fmt.Errorf("%w: compression is only supported for standby snapshots", ErrInvalidRequest) + } + if _, err := normalizeCompressionConfig(req.Compression); err != nil { + return err + } if err := tags.Validate(req.Metadata); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go new file mode 100644 index 00000000..8974ede5 --- /dev/null +++ b/lib/instances/snapshot_compression.go @@ -0,0 +1,410 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/logger" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" +) + +const ( + defaultSnapshotCompressionZstdLevel = 1 + maxSnapshotCompressionZstdLevel = 19 +) + +type compressionJob struct { + cancel context.CancelFunc + done chan struct{} +} + +type compressionTarget struct { + Key string + OwnerID string + SnapshotID string + SnapshotDir string + Policy snapshotstore.SnapshotCompressionConfig +} + +func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snapshotstore.SnapshotCompressionConfig { + if cfg == nil { + return nil + } + cloned := *cfg + if cfg.Level != nil { + v := *cfg.Level + cloned.Level = &v + } + return &cloned +} + +func cloneSnapshotPolicy(policy *SnapshotPolicy) *SnapshotPolicy { + if policy == nil { + return nil + } + return &SnapshotPolicy{ + Compression: cloneCompressionConfig(policy.Compression), + } +} + +func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if cfg == nil || !cfg.Enabled { + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil + } + + normalized := snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: cfg.Algorithm, + } + switch normalized.Algorithm { + case "": + normalized.Algorithm = snapshotstore.SnapshotCompressionAlgorithmZstd + case snapshotstore.SnapshotCompressionAlgorithmZstd, snapshotstore.SnapshotCompressionAlgorithmLz4: + default: + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + + switch normalized.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + if level < 1 || level > maxSnapshotCompressionZstdLevel { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid zstd level %d (must be between 1 and %d)", ErrInvalidRequest, level, maxSnapshotCompressionZstdLevel) + } + normalized.Level = &level + case snapshotstore.SnapshotCompressionAlgorithmLz4: + if cfg.Level != nil { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: lz4 does not support level", ErrInvalidRequest) + } + } + return normalized, nil +} + +func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if override != nil { + return normalizeCompressionConfig(override) + } + if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { + return normalizeCompressionConfig(stored.SnapshotPolicy.Compression) + } + if m.snapshotDefaults.Compression != nil { + return normalizeCompressionConfig(m.snapshotDefaults.Compression) + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil +} + +func (m *manager) snapshotJobKeyForInstance(instanceID string) string { + return "instance:" + instanceID +} + +func (m *manager) snapshotJobKeyForSnapshot(snapshotID string) string { + return "snapshot:" + snapshotID +} + +func (m *manager) startCompressionJob(ctx context.Context, target compressionTarget) { + if target.Key == "" || !target.Policy.Enabled { + return + } + + m.compressionMu.Lock() + if _, exists := m.compressionJobs[target.Key]; exists { + m.compressionMu.Unlock() + return + } + jobCtx, cancel := context.WithCancel(context.Background()) + job := &compressionJob{ + cancel: cancel, + done: make(chan struct{}), + } + m.compressionJobs[target.Key] = job + m.compressionMu.Unlock() + + go func() { + defer func() { + m.compressionMu.Lock() + delete(m.compressionJobs, target.Key) + m.compressionMu.Unlock() + close(job.done) + }() + + log := logger.FromContext(ctx) + rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) + if !ok { + if _, _, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, nil, nil) + } + return + } + + uncompressedSize, compressedSize, err := compressSnapshotMemoryFile(jobCtx, rawPath, target.Policy) + if err != nil { + if errors.Is(err, context.Canceled) { + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil) + } + return + } + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil) + } + log.WarnContext(ctx, "snapshot compression failed", "snapshot_dir", target.SnapshotDir, "error", err) + return + } + + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize) + } + }() +} + +func (m *manager) cancelCompressionJob(key string) { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job != nil { + job.cancel() + } +} + +func (m *manager) waitCompressionJob(key string, timeout time.Duration) { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job == nil { + return + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-job.done: + case <-timer.C: + } +} + +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string) error { + if jobKey != "" { + m.cancelCompressionJob(jobKey) + m.waitCompressionJob(jobKey, 2*time.Second) + } + + if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + return nil + } + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + if !ok { + return nil + } + return decompressSnapshotMemoryFile(ctx, compressedPath, algorithm) +} + +func (m *manager) updateSnapshotCompressionMetadata(snapshotID, state, compressionError string, cfg *snapshotstore.SnapshotCompressionConfig, compressedSize, uncompressedSize *int64) error { + rec, err := m.loadSnapshotRecord(snapshotID) + if err != nil { + return err + } + rec.Snapshot.CompressionState = state + rec.Snapshot.CompressionError = compressionError + rec.Snapshot.Compression = cloneCompressionConfig(cfg) + rec.Snapshot.CompressedSizeBytes = compressedSize + rec.Snapshot.UncompressedSizeBytes = uncompressedSize + + if state == snapshotstore.SnapshotCompressionStateCompressed { + sizeBytes, sizeErr := snapshotstore.DirectoryFileSize(m.paths.SnapshotGuestDir(snapshotID)) + if sizeErr == nil { + rec.Snapshot.SizeBytes = sizeBytes + } + } + return m.saveSnapshotRecord(rec) +} + +func findRawSnapshotMemoryFile(snapshotDir string) (string, bool) { + for _, candidate := range snapshotMemoryFileCandidates(snapshotDir) { + if st, err := os.Stat(candidate); err == nil && st.Mode().IsRegular() { + return candidate, true + } + } + return "", false +} + +func findCompressedSnapshotMemoryFile(snapshotDir string) (string, snapshotstore.SnapshotCompressionAlgorithm, bool) { + for _, raw := range snapshotMemoryFileCandidates(snapshotDir) { + zstdPath := raw + ".zst" + if st, err := os.Stat(zstdPath); err == nil && st.Mode().IsRegular() { + return zstdPath, snapshotstore.SnapshotCompressionAlgorithmZstd, true + } + lz4Path := raw + ".lz4" + if st, err := os.Stat(lz4Path); err == nil && st.Mode().IsRegular() { + return lz4Path, snapshotstore.SnapshotCompressionAlgorithmLz4, true + } + } + return "", "", false +} + +func snapshotMemoryFileCandidates(snapshotDir string) []string { + return []string{ + filepath.Join(snapshotDir, "memory-ranges"), + filepath.Join(snapshotDir, "memory"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory-ranges"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory"), + } +} + +func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapshotstore.SnapshotCompressionConfig) (int64, int64, error) { + rawInfo, err := os.Stat(rawPath) + if err != nil { + return 0, 0, fmt.Errorf("stat raw memory snapshot: %w", err) + } + uncompressedSize := rawInfo.Size() + + compressedPath := compressedPathFor(rawPath, cfg.Algorithm) + tmpPath := compressedPath + ".tmp" + _ = os.Remove(tmpPath) + _ = os.Remove(compressedPath) + + if err := runCompression(ctx, rawPath, tmpPath, cfg); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, err + } + if err := os.Rename(tmpPath, compressedPath); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, fmt.Errorf("finalize compressed snapshot: %w", err) + } + + compressedInfo, err := os.Stat(compressedPath) + if err != nil { + return 0, 0, fmt.Errorf("stat compressed snapshot: %w", err) + } + if err := os.Remove(rawPath); err != nil { + return 0, 0, fmt.Errorf("remove raw memory snapshot: %w", err) + } + return uncompressedSize, compressedInfo.Size(), nil +} + +func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + src, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("open source snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("create compressed snapshot: %w", err) + } + defer dst.Close() + + switch cfg.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + enc, err := zstd.NewWriter(dst, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) + if err != nil { + return fmt.Errorf("create zstd encoder: %w", err) + } + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close zstd encoder: %w", err) + } + case snapshotstore.SnapshotCompressionAlgorithmLz4: + enc := lz4.NewWriter(dst) + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close lz4 encoder: %w", err) + } + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + return nil +} + +func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + rawPath := strings.TrimSuffix(strings.TrimSuffix(compressedPath, ".zst"), ".lz4") + tmpRawPath := rawPath + ".tmp" + _ = os.Remove(tmpRawPath) + + src, err := os.Open(compressedPath) + if err != nil { + return fmt.Errorf("open compressed snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(tmpRawPath) + if err != nil { + return fmt.Errorf("create decompressed snapshot file: %w", err) + } + defer dst.Close() + + var reader io.Reader + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + dec, err := zstd.NewReader(src) + if err != nil { + return fmt.Errorf("create zstd decoder: %w", err) + } + defer dec.Close() + reader = dec + case snapshotstore.SnapshotCompressionAlgorithmLz4: + reader = lz4.NewReader(src) + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, algorithm) + } + + if err := copyWithContext(ctx, dst, reader); err != nil { + _ = os.Remove(tmpRawPath) + return err + } + if err := os.Rename(tmpRawPath, rawPath); err != nil { + _ = os.Remove(tmpRawPath) + return fmt.Errorf("finalize decompressed snapshot: %w", err) + } + return nil +} + +func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return rawPath + ".lz4" + default: + return rawPath + ".zst" + } +} + +func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { + buf := make([]byte, 1024*1024) + for { + if err := ctx.Err(); err != nil { + return err + } + n, readErr := src.Read(buf) + if n > 0 { + if _, writeErr := dst.Write(buf[:n]); writeErr != nil { + return fmt.Errorf("write compressed stream: %w", writeErr) + } + } + if readErr != nil { + if errors.Is(readErr, io.EOF) { + return nil + } + return fmt.Errorf("read snapshot stream: %w", readErr) + } + } +} diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go new file mode 100644 index 00000000..8c04c039 --- /dev/null +++ b/lib/instances/snapshot_compression_test.go @@ -0,0 +1,118 @@ +package instances + +import ( + "errors" + "testing" + + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeCompressionConfig(t *testing.T) { + t.Parallel() + + cfg, err := normalizeCompressionConfig(nil) + require.NoError(t, err) + assert.False(t, cfg.Enabled) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }) + require.NoError(t, err) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 1, *cfg.Level) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(25), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(1), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(2), + }, + }, + } + + stored := &StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + } + + cfg, err := m.resolveSnapshotCompressionPolicy(stored, &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(4), + }) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 4, *cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(&StoredMetadata{}, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 2, *cfg.Level) +} + +func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { + t.Parallel() + + err := validateCreateRequest(CreateInstanceRequest{ + Name: "compression-test", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(0), + }, + }, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestValidateCreateSnapshotRequestRejectsStoppedCompression(t *testing.T) { + t.Parallel() + + err := validateCreateSnapshotRequest(CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} diff --git a/lib/instances/snapshot_integration_scenario_test.go b/lib/instances/snapshot_integration_scenario_test.go index 37e1ae7a..796ac8db 100644 --- a/lib/instances/snapshot_integration_scenario_test.go +++ b/lib/instances/snapshot_integration_scenario_test.go @@ -64,7 +64,7 @@ func runStandbySnapshotScenario(t *testing.T, mgr *manager, tmpDir string, cfg s } }) - _, err = mgr.StandbyInstance(ctx, sourceID) + _, err = mgr.StandbyInstance(ctx, sourceID, StandbyInstanceRequest{}) requireNoErr(err) snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 1fca6dfb..79e39e54 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/trace" ) @@ -20,6 +21,8 @@ func (m *manager) standbyInstance( ctx context.Context, id string, + req StandbyInstanceRequest, + skipCompression bool, ) (*Instance, error) { start := time.Now() log := logger.FromContext(ctx) @@ -55,6 +58,19 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("%w: standby is not supported for instances with vGPU attached (driver limitation)", ErrInvalidState) } + // Resolve/validate compression policy early so invalid request/config + // fails before any state transition side effects. + var compressionPolicy *snapshotstore.SnapshotCompressionConfig + if !skipCompression { + policy, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + if err != nil { + return nil, err + } + if policy.Enabled { + compressionPolicy = &policy + } + } + // 3. Get network allocation BEFORE killing VMM (while we can still query it) // This is needed to delete the TAP device after VMM shuts down var networkAlloc *network.Allocation @@ -142,6 +158,16 @@ func (m *manager) standbyInstance( // Return instance with derived state (should be Standby now) finalInst := m.toInstance(ctx, meta) + + if compressionPolicy != nil { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForInstance(stored.Id), + OwnerID: stored.Id, + SnapshotDir: snapshotDir, + Policy: *compressionPolicy, + }) + } + log.InfoContext(ctx, "instance put in standby successfully", "instance_id", id, "state", finalInst.State) return &finalInst, nil } diff --git a/lib/instances/types.go b/lib/instances/types.go index bb548d6a..3a561936 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -92,6 +92,9 @@ type StoredMetadata struct { SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + // Snapshot policy defaults for this instance. + SnapshotPolicy *SnapshotPolicy + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -169,6 +172,7 @@ type CreateInstanceRequest struct { Cmd []string // Override image cmd (nil = use image default) SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance } // StartInstanceRequest is the domain request for starting a stopped instance @@ -202,9 +206,15 @@ type ListSnapshotsFilter = snapshot.ListSnapshotsFilter // CreateSnapshotRequest is the domain request for creating a snapshot. type CreateSnapshotRequest struct { - Kind SnapshotKind // Required: Standby or Stopped - Name string // Optional: unique per source instance - Metadata tags.Metadata // Optional user-defined key-value metadata + Kind SnapshotKind // Required: Standby or Stopped + Name string // Optional: unique per source instance + Metadata tags.Metadata // Optional user-defined key-value metadata + Compression *snapshot.SnapshotCompressionConfig // Optional compression override +} + +// StandbyInstanceRequest is the domain request for putting an instance into standby. +type StandbyInstanceRequest struct { + Compression *snapshot.SnapshotCompressionConfig // Optional compression override } // RestoreSnapshotRequest is the domain request for restoring a snapshot in-place. @@ -220,6 +230,11 @@ type ForkSnapshotRequest struct { TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots } +// SnapshotPolicy defines default snapshot behavior for an instance. +type SnapshotPolicy struct { + Compression *snapshot.SnapshotCompressionConfig +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 674b6e6d..5d8e8db1 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -122,6 +122,14 @@ const ( RestoreSnapshotRequestTargetHypervisorVz RestoreSnapshotRequestTargetHypervisor = "vz" ) +// Defines values for SnapshotCompressionState. +const ( + SnapshotCompressionStateCompressed SnapshotCompressionState = "compressed" + SnapshotCompressionStateCompressing SnapshotCompressionState = "compressing" + SnapshotCompressionStateError SnapshotCompressionState = "error" + SnapshotCompressionStateNone SnapshotCompressionState = "none" +) + // Defines values for SnapshotSourceHypervisor. const ( CloudHypervisor SnapshotSourceHypervisor = "cloud-hypervisor" @@ -130,6 +138,12 @@ const ( Vz SnapshotSourceHypervisor = "vz" ) +// Defines values for SnapshotCompressionConfigAlgorithm. +const ( + Lz4 SnapshotCompressionConfigAlgorithm = "lz4" + Zstd SnapshotCompressionConfigAlgorithm = "zstd" +) + // Defines values for SnapshotKind. const ( SnapshotKindStandby SnapshotKind = "Standby" @@ -357,7 +371,8 @@ type CreateInstanceRequest struct { // When true, DKMS (Dynamic Kernel Module Support) will not work, // preventing compilation of out-of-tree kernel modules (e.g., NVIDIA vGPU drivers). // Recommended for workloads that don't need kernel module compilation. - SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // Vcpus Number of virtual CPUs Vcpus *int `json:"vcpus,omitempty"` @@ -371,6 +386,8 @@ type CreateInstanceRequestHypervisor string // CreateSnapshotRequest defines model for CreateSnapshotRequest. type CreateSnapshotRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + // Kind Snapshot capture kind Kind SnapshotKind `json:"kind"` @@ -720,7 +737,8 @@ type Instance struct { OverlaySize *string `json:"overlay_size,omitempty"` // Size Base memory size (human-readable) - Size *string `json:"size,omitempty"` + Size *string `json:"size,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // StartedAt Start timestamp (RFC3339) StartedAt *time.Time `json:"started_at"` @@ -923,6 +941,16 @@ type RestoreSnapshotRequestTargetHypervisor string // Snapshot defines model for Snapshot. type Snapshot struct { + // CompressedSizeBytes Compressed memory payload size in bytes + CompressedSizeBytes *int64 `json:"compressed_size_bytes"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // CompressionError Compression error message when compression_state is error + CompressionError *string `json:"compression_error"` + + // CompressionState Compression status of the snapshot payload memory file + CompressionState *SnapshotCompressionState `json:"compression_state,omitempty"` + // CreatedAt Snapshot creation timestamp CreatedAt time.Time `json:"created_at"` @@ -949,17 +977,48 @@ type Snapshot struct { // SourceInstanceName Source instance name at snapshot creation time SourceInstanceName string `json:"source_instance_name"` + + // UncompressedSizeBytes Uncompressed memory payload size in bytes + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes"` } +// SnapshotCompressionState Compression status of the snapshot payload memory file +type SnapshotCompressionState string + // SnapshotSourceHypervisor Source instance hypervisor at snapshot creation time type SnapshotSourceHypervisor string +// SnapshotCompressionConfig defines model for SnapshotCompressionConfig. +type SnapshotCompressionConfig struct { + // Algorithm Compression algorithm (defaults to zstd when enabled) + Algorithm *SnapshotCompressionConfigAlgorithm `json:"algorithm,omitempty"` + + // Enabled Enable snapshot memory compression + Enabled bool `json:"enabled"` + + // Level Compression level for zstd only + Level *int `json:"level,omitempty"` +} + +// SnapshotCompressionConfigAlgorithm Compression algorithm (defaults to zstd when enabled) +type SnapshotCompressionConfigAlgorithm string + // SnapshotKind Snapshot capture kind type SnapshotKind string +// SnapshotPolicy defines model for SnapshotPolicy. +type SnapshotPolicy struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // SnapshotTargetState Target state when restoring or forking from a snapshot type SnapshotTargetState string +// StandbyInstanceRequest defines model for StandbyInstanceRequest. +type StandbyInstanceRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // Volume defines model for Volume. type Volume struct { // Attachments List of current attachments (empty if not attached) @@ -1194,6 +1253,9 @@ type CreateInstanceSnapshotJSONRequestBody = CreateSnapshotRequest // RestoreInstanceSnapshotJSONRequestBody defines body for RestoreInstanceSnapshot for application/json ContentType. type RestoreInstanceSnapshotJSONRequestBody = RestoreSnapshotRequest +// StandbyInstanceJSONRequestBody defines body for StandbyInstance for application/json ContentType. +type StandbyInstanceJSONRequestBody = StandbyInstanceRequest + // StartInstanceJSONRequestBody defines body for StartInstance for application/json ContentType. type StartInstanceJSONRequestBody StartInstanceJSONBody @@ -1377,8 +1439,10 @@ type ClientInterface interface { RestoreInstanceSnapshot(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // StandbyInstance request - StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // StandbyInstanceWithBody request with any body + StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // StartInstanceWithBody request with any body StartInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1857,8 +1921,20 @@ func (c *Client) RestoreInstanceSnapshot(ctx context.Context, id string, snapsho return c.Client.Do(req) } -func (c *Client) StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewStandbyInstanceRequest(c.Server, id) +func (c *Client) StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequest(c.Server, id, body) if err != nil { return nil, err } @@ -3279,8 +3355,19 @@ func NewRestoreInstanceSnapshotRequestWithBody(server string, id string, snapsho return req, nil } -// NewStandbyInstanceRequest generates requests for StandbyInstance -func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) { +// NewStandbyInstanceRequest calls the generic StandbyInstance builder with application/json body +func NewStandbyInstanceRequest(server string, id string, body StandbyInstanceJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewStandbyInstanceRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewStandbyInstanceRequestWithBody generates requests for StandbyInstance with any type of body +func NewStandbyInstanceRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { var err error var pathParam0 string @@ -3305,11 +3392,13 @@ func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), nil) + req, err := http.NewRequest("POST", queryURL.String(), body) if err != nil { return nil, err } + req.Header.Add("Content-Type", contentType) + return req, nil } @@ -4219,8 +4308,10 @@ type ClientWithResponsesInterface interface { RestoreInstanceSnapshotWithResponse(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*RestoreInstanceSnapshotResponse, error) - // StandbyInstanceWithResponse request - StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + // StandbyInstanceWithBodyWithResponse request with any body + StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + + StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) // StartInstanceWithBodyWithResponse request with any body StartInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) @@ -4970,6 +5061,7 @@ type StandbyInstanceResponse struct { Body []byte HTTPResponse *http.Response JSON200 *Instance + JSON400 *Error JSON404 *Error JSON409 *Error JSON500 *Error @@ -5691,9 +5783,17 @@ func (c *ClientWithResponses) RestoreInstanceSnapshotWithResponse(ctx context.Co return ParseRestoreInstanceSnapshotResponse(rsp) } -// StandbyInstanceWithResponse request returning *StandbyInstanceResponse -func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { - rsp, err := c.StandbyInstance(ctx, id, reqEditors...) +// StandbyInstanceWithBodyWithResponse request with arbitrary body returning *StandbyInstanceResponse +func (c *ClientWithResponses) StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstanceWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseStandbyInstanceResponse(rsp) +} + +func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstance(ctx, id, body, reqEditors...) if err != nil { return nil, err } @@ -7122,6 +7222,13 @@ func ParseStandbyInstanceResponse(rsp *http.Response) (*StandbyInstanceResponse, } response.JSON200 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -11183,7 +11290,8 @@ func (response RestoreInstanceSnapshot501JSONResponse) VisitRestoreInstanceSnaps } type StandbyInstanceRequestObject struct { - Id string `json:"id"` + Id string `json:"id"` + Body *StandbyInstanceJSONRequestBody } type StandbyInstanceResponseObject interface { @@ -11199,6 +11307,15 @@ func (response StandbyInstance200JSONResponse) VisitStandbyInstanceResponse(w ht return json.NewEncoder(w).Encode(response) } +type StandbyInstance400JSONResponse Error + +func (response StandbyInstance400JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + type StandbyInstance404JSONResponse Error func (response StandbyInstance404JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { @@ -12827,6 +12944,13 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id + var body StandbyInstanceJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) } @@ -13295,201 +13419,206 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XITO7boq6h8z6lxztiO80EIPrXr3ECAnbMJ5BKSuWe2uUbulm1NuqXektrBUPyd", - "B5hHnCe5pSWpv6y2O4E4ZGBqauN0q/WxtLS0vtfnVsDjhDPClGwNPrdkMCMxhp9HSuFgdsmjNCZvyR8p", - "kUo/TgRPiFCUQKOYp0yNEqxm+q+QyEDQRFHOWoPWGVYzdD0jgqA59ILkjKdRiMYEwXckbHVa5COOk4i0", - "Bq3tmKntECvc6rTUItGPpBKUTVtfOi1BcMhZtDDDTHAaqdZggiNJOpVhT3XXCEukP+nCN1l/Y84jglnr", - "C/T4R0oFCVuD34vLeJ815uO/kUDpwY/mmEZ4HJFjMqcBWQZDkApBmBqFgs6JWAbFM/M+WqAxT1mITDvU", - "ZmkUITpBjDOyVQIGm9OQakjoJnro1kCJlHggE8KcRjT07MCzE2Reo5Nj1J6Rj+VBdh+PD1v1XTIck+VO", - "f01jzLoauHparn9oW+z71b6vZ8rjOB1NBU+T5Z5P3pyeXiB4iVgaj4ko9ni4m/VHmSJTInSHSUBHOAwF", - "kdK/fveyOLd+v98f4N1Bv9/r+2Y5Jyzkohak5rUfpDv9kKzoshFIbf9LIH19eXJ8coSecZFwgeHbpZEq", - "iF0ET3FdRbQp74oP/5+mNAqXsX6sHxMxokwqzGpw8MS+1ODiE6RmBNnv0OUpak+4QCEZp9MpZdOtJviu", - "CVZEFAlHWC0PB1NFtg3lDCkaE6lwnLQ6rQkXsf6oFWJFuvpNowEFwWuG0y0aDbZ81FKzk6NY1vXumiDK", - "UEyjiEoScBbK4hiUqYP9+sUUDgwRgnso1HP9GMVESjwlqK3JpqbdDEmFVSoRlWiCaUTCRnvkQwSzmL/x", - "MaIhYYpOaPl8G3Tq4nGws7vnpR0xnpJRSKf2Jip3fwzPNYrpfhSC1v6F6IO2aLYOGFKQyfJ4L4B0wyCC", - "TIggGse/criYKAwX4OBz699g1Nb/2s4v6G17O2+f2nbv8FQCERR8Tpg+Zeu+hE04y5t/6bT+SElKRgmX", - "1KxsieLZNxr9YIsQfOFfK7xahSMFTJQKi9XnClp8gxNs5tcINuemaZWOApm03ZQoQi25fD4nzMMwBZwp", - "+6K84ld8iiLKCLItLHw1fdQD/BJxII/fYm2dVg7SZUKg530LQmYe1PSm33VahKWxBmbEp0VozggWakxK", - "wKy5zmxH+exqwX9WOhKVewtLMlpNTc4oYyREuqU95KYlSiVwrUvLh5NxRdVoToT0niOY1m9UIduitquI", - "B1cTGpHRDMuZmTEOQziDODorrcTDuZVYYZxogug6BI5CIsXR+a9Hu48OkB3AA0PJUxGYGSyvpPC17t60", - "RQqLMY4iL27Uo9vN7+tlDPFjwHl2MOruoQwDHWIa6tWyu6m777SSVM7ML6DjelZwD2oyoNEr0r/fexb9", - "DIiEkRjq5adbUnw/H/kmMUiCphHXe7FAKaN/pCUmvYdOtLyhkL40aEjCDsLwQpNvnCrenRJGhKZvaCJ4", - "DBxbgZFGbdKb9jpoqHnLruaku3i32+93+8NWmRWO9rvTJNUgxEoRoSf4/37H3U9H3b/2u0/e5z9Hve77", - "P/+bD3GacveOs7TrbDua0UFuskWWvzrR1eLACo7aR33Mtp9omrGpXX92ssyImHWHPLgiokf5dkTHAovF", - "NptS9nEQYUWkKkNhddu1cIG5rQAIm2qQbQgkFYEK0Lsd8WsiAk3RI6IRUnY0UadKdhDWMjkQQ6Rv3f9E", - "AWb6jBgGhAtEWIiuqZohDO3KkIsXXZzQLjVLbHVaMf74irCpmrUGB3tL+K+Rv21/dN//h3u09V/eIyDS", - "iHiQ/y1PFWVTBK8NlzCjEuVzoIrEa9kCtytpBKxgTNmJ+WwnmwkWAi/8u+0mt2rXjfBXu+1B7JEU3syJ", - "EDR0N++z02PUjugVseiMRMrQMO339wJoAD+JfRLwOMYsNM+2euhNTJW+8dL8Ijfao15xC39vkWDGgReJ", - "Iq4XlIGvhtFxcHGCtGeLjp3mRSIrzcPdi0GvBlv28uxiW1OxBEupZoKn01l5VpaE3mw+VF6NKB+NE9+c", - "qLxCJ9tvkCbwKKIaOhlB3+n3T59uy2FL//HI/bHVQ8cGZDB9vX9c2HtGzrAgwCWFiDP07OwC4SjigZVX", - "J5qZndBpKkjYq6hJoHcfwhOmxCLh1MckVzAjb7qMIN1u/vYGeLA9pmxb6m3oBjeDO2Hzr2DVnrM5FZzF", - "ml2eY0E13SoprT63Xr85fj56/vqyNdCHKEwDqwE6e/P2XWvQ2uv3+y0fN6QxaA0deHl28Qx2SrefcZVE", - "6XQk6ScPaT3K1odiEnNhRBT7DWrPypTXcHAINmfY2nv51CDXzkvAK7cpIZXQ2vViOi5jzO7Lpz5smS0S", - "IuZU+nQav2bv3M4X6KQhTGXclkTMiciQFrC4V+APg4inYbcwZKc1oYIEAmu0a3Vaf5BYMzzzTxp18rl7", - "vvOrGhpd7mtubRwllJHaa7vz0K/aay6uIo7D7s43vmkZUbrv5SW+Ni/KeGFxiWSo1OosiZksvKahmo1C", - "fs30lD302L5BWeOMKH/UK8HRP//+j8vTnI/deTlOLIXe2X30lRS6QpN1117ZNltImviXcZH4F3F5+s+/", - "/8Ot5H4XQZjGz7BkPzLqovJS/jIjakZE4aZ2G6wfGSEDPkcOXwrDl/RPRWPTElHmcyIivCgQWTun1k4f", - "KF1lVoIqOF/2O00yr5D+eA3J1b25C/1lVfDZ7fuJqmdSnjk91efb3gFNZpJNZGf31P7cXZ5SzYyuaDKa", - "ah5yhKeZ/myVGfD8iiYIvujCF2Ybo8gc3jDVPaMx56o3ZH+ZEYZg72CDyUcSAJ2SCit0dHYi0TWNIpCa", - "gRAsXyND9q5ACkxzqfR/Rco6aJwqJEjMFUGWQYVBUpgLNB4TlDLs7Iy9IStCxS6wilcWLFdEMBKNZgSH", - "RMiGkDEfIftRLXBgqRMsFRGGQqdJGV7Hv52eo/bxguGYBug30+spD9OIoPM00Wd4qwy9zpAlgswJA/lF", - "3zvUjssniKeqyyddJQhxU4yhs0zvYI1g85dnF9aMKrd6Q/aWaMASFpIQ5uxuCYnUDCsUcvYnfWJJWO62", - "OH4F6P6z3GnNgyQtQ3m3CuHXYLzU65lToVIcaZJV4ua8tkxjJfdw7cYIX5QeLCnKEA6rshGqqQBoegaT", - "+TJP65f5DKNSL/OdM5zIGVe1Mt8VZeG6eblOftNtvznPkunJpB3mrtmWRJBumkwFBuPwt2NaKjsEkK3f", - "mTW+HD6jXQapIJWKxwXTHWpXlIW0rFYsA2DOo67eF2Da7pgjNctcNp/HCzMFc8zq7r3RdOzReOvrjTI0", - "pVM8XqiyZLbTXz7M/qPj+vdtUZ1riTnwJBwpvtq4TifItW1iEwNHlJHio/mEenrO2KBcq0olCip+LJYM", - "6S66SUAtQe6g6xnVjJNEDghAky9Pi5qO3pB14RIZoONsgKzbrEt98EDzDl20uShMgoIRBY0XWwijy9Me", - "epfN9k8SMazonDhfmxmWaEwIQykw3CSE8eGCLE4glfpWoqr6ub19jFvOFih0uH3XQ1rQjLG9yfWxiLGi", - "ASjgx7SyHjCYmo3SI2mSzop8RKN7f5VLwlsypVKJikMCar998Wxvb+9JlQPcfdTt73R3Hr3b6Q/6+v9/", - "be678O09j3x9HZXpjDVpFCnRs4uT413LbpbHUZ/28ZPDjx+xenJAr+WTT/FYTP+2hzfim/RtydpxbsNB", - "7VQS0XWkVmOjz3JTMJDUWGZubXC5kTuVMw2vgoFZ3Tvd8i4csHzmfGtMvrmLVJV4rnUIKCxuaT36qeYU", - "8xNTUDhZ+1lAvRbGYyqvngqCr0J+zTz3uWbU5MjcV35NcKol6vECkY+aUSchEpyriTQapzLDurP/eP9w", - "72D/sN/3+B0tIz8P6CjQt1GjCbx5doIivCACwTeoDSJ/iMYRH5eR99HeweHj/pOd3abzMAJzMzhk/LT7", - "CrUtRP7sfFjdm9KkdncfH+zt7fUPDnb3G83KsvqNJuXEghLL8Xjv8f7O4e5+Iyj4FBDPnR9Y1T8l9Cl9", - "kySiRt3SlQkJ6IQGCDzJkP4AtWO4zkgm+5fP5BiHI2HZTu89ojCN5EpdsxnMtjRug3EaKZpExLyDDWkk", - "88DKj6Ennx6fMkbEKHOTu0FP1nturY7UrSVrgkpekCXQnVIJHEnOSFEShQNzQtfSOdjNfGLv6/DArqEh", - "NrzSYlI3InMSFZHAXEd6sjEXBGV4YjattCrK5jii4YiyJK3RUdeA8kUqgC81nSI85qkyShvYsOIgYIMH", - "mWSiyXUz15EXXFyttVrq23UkUsZ0N2v1LUdRxK/1Fl9p2MDNjJH92jnPFBjATLliVFD2vURvzRdGRZU/", - "TlKFKFNcS6IsHC86MBIJoR1DgkjFgZLi4Epzm7abppymnxd5rZkQpwA34+W0c0Pa/+7EKF+/pQlAYTEl", - "aiQVVms5Fo0p76D9OTRv7BChP1yrJGkAd0auNwF08ALparTtSoaTu4H4KjNepoPIG8EtLGhIeghOF9gF", - "nDdq5aSdK54kJMx0Pb0hOzdHJXskUZxK0HVeGTioGaECcUGntDywPTYbsAfeBBUdNt0aHYsfLnOo8BKU", - "4fWHHk8UEQaCzkG/6FlnN6HVaVnYtzotS4nKoHEPPRDJjdRLU3x5dnFT61wi+IRGnuWCZtm+tdKWs1u9", - "2u+fd3f+j7Fda3wDFo0yo42OeUh6lRgYaN/s5nl5dnFWN6csAAkVZ7e0psx+4KEcmUraQcRqxgPM0Jgg", - "K8E49NcXSzZIzns/8fGyE4FjMk4nEyJGsUd59kK/R6aBMRRRhk6flvlZzTcvd+2ngmelzQFReIIDGz/S", - "DPoe5VxlGZ0CNN/7t+stMddwnaep3iph21hn0x56nYV8oZdnFxLlNh+P1q68vbWeRmezhaQBjkyPxnGc", - "sqKyDZCzMYd8ln9o1ZIePjn28obuIKD2fJqkcAzP33ZP3lxuxyGZd0pzAjvNjEdEz3urQC3mzm80d4sq", - "EYl5nfbCIIZseoAKsMpOcGMgFc6rBzqKKxyNZMSVZzbv9EsEL1H78oXx39Mz6KCktJX6eQEKJfw+8J4Y", - "TZHqhj2HAavq09IBX6vJjo1EUVxeaVDfUfmV4MgEiJbxOQ9dcBvPr8obza/Wnl7biW/cE+dS08Dn8Nnp", - "sWEYAs4UpowIlKnvSg5iwA61Oq2uvqNCTGIwXE7+c7WzWI06PkOXVQrdZ0vRZXeizK2JhNBELpqTEMWY", - "0QmRykZClEaWM7z76GBgYrdCMtl/dNDr9W7q3fc8d+drtBXbxvmp4OjXk7Ov24c7cOJrspbPrbOjd7+2", - "Bq3tVIrtiAc42pZjygaFv7M/8xfww/w5pszr/Nco3I9OlsL8yuZLfWeZ5wO9EkaCDCE5CPB3FtpWIwdp", - "lI7oJxIir/e7wlMt1xhM/To3968IkMuju1UhMK7oA9AgSI5+Wq1BdQwVtLFjpkzRKI87XNad3ipyVK4M", - "qFkKpkkIy0Joosj8Cjib69Pki6cpEX73bmkzro1wNwqpB6v/YiW/UAthCnxT15+91jZOkvUo7GcaM1rY", - "NDbQetx7bqV7vwFuY3srj/5m+t9//F959vhvO3+8urz8n/nL/z5+Tf/nMjp7cy9+qKuDNO410mKlow0Y", - "nEoRFk3R6hSrwMNozbhUNVCzb5DiKNYf99AzEAgHQ9ZFr6giAkcDNGzhhPYsMHsBj4ct1CYfcaDMV4gz", - "pLuy/mRb+uMzoxbSH392MueXah+hdRwTFsiZj6dMxyGPMWVbQzZkti/kFiLB7q9/hSjAiUoF0Tuiedto", - "gcYCB7nDWD54B33GSfJla8hA8iUfldArSLBQWQSZGwE22s7K+BXY5iREcxylRFrJeciyewdUAboTo7vp", - "ZcoR0NlXNK41QPGKNVyUHR4P+x3PPiLdTm9kRKUiDGVaECoBeVHbea4e9ktk47B/uN6FJcOhFegH2L0c", - "2uWQssH5MAgMQxsiPpoplTTQsWs6Zc4I+vXduzMNBv3vOXId5bDIttgIfzhJIkqk0R2qCHgg6yy81fKp", - "xM3uNlyQUZ7BZ1ED38znMDB69+ocKSJiygzdbwcanBMa6PWBqZ9KmWpUpBgdPTt9vtVrkOQGYJvNf8U+", - "vstWWLEoO2VanY4ww3gN3w46Oe5oNsye0JxBA9ebF1ygyBCY/FwP0IUkZddG2Cpj7Tc7GS1yjZyh6sPW", - "lusxqVKKAXqb8YU4m0oW2Zojg+syP5fQrTXIGL+gpd475bmCx5OVlyxpAy8grJC1f8IVXk8KVh9/D8Th", - "zHNW1XXe7GwXlaR6MD9q5Ht/55zL3k1l15uGvJU91QuRCVnUW/NwtbsI+1qW4z5SNao1ziP92prindRx", - "eYpmWLI/KXhZkT129h43ShajR21q1i4atPnETCk7Vc7tPTPHmgCAKxpFxstB0inDEXqC2ucnL387efVq", - "C3XRmzen1a1Y9YVvfxpEvznUfnl2ASFlWI6cZajeMRLnzsPkI5VKLkcFNDKwro62+7UUEecNs9j6hmFy", - "ziq9tIxNBMDdp+vfv07w3cpwua+NebNM8h2FvNUSZV+4WJk+m8ffNnjtTqZTCkPz0ZUiL+H8uW8dedZp", - "UY8v65HUpJOE6OQsz+qRK6tc95U1Pdnt7Rwc9nb6/d5Ov5HKDwcrxj49etZ88P6uUWYM8HgQhAMyaTJ+", - "jerQIrZh+nB0jRcSDR1bPmwZOaAgABSOu2XdG5lzlwP8bhfPV2VE1kXs3SRCr1no3Yo0XeflBF2NebtH", - "f/2qXF6k6Y1uXSHsV6ObKMMJCngahZp/GuuTZ8QxElqpURKV5z6Dw3rBrhi/ZuWlG92mPr9/pEQs0OXp", - "aUmDLsjEpoFqsHBwoajZB57caBt217DYa2dTiILbRORblRIWbqBvHudWVL85N0uDdQ3UcDkn6TWNU2bA", - "rfd+xZoqCpSQzEdp6mOQ9CsXaHFxcXJc2nCMD3YO+4dPuofjnYPuftjf6eKdvYPu7iPcn+wFj/dqEi02", - "d425vbdL+YTWBzYB4EEZaWLYwoE+Q5m7yjhVKHNl04fzmeY0UYGlNWE8oB+wvkW6B7hdA/0mWmRc78qP", - "z7A+qO7bBP5a/cX5LFWaDYJv5CxVSP8FU9ZLsFLD6i7MmR+g1xy+Ec4HlPGq+GGag2/VcvOqqNK2Xj/O", - "OxQGswRsgF5kRCsje5bMtSWxPw0ttY7L4JS9VXKNs7tVcPPqtAwIW52Wgwy4gy07htmJeGMeinjjU9YT", - "HAENyx1vUkUj+skcOT11KhUNjLSGYTfrjp1NMUDCkblC68xwxpvDXrPZR+5UX56iNoQP/hlZYU7/tZWZ", - "7IpHaH/3yf6Tg8e7Tw4aBRHkE1xPjZ+Br9Hy5NaS5iBJRy7hbM3Sn51dwOWjLzaZxkY6t2sv+Gwmggea", - "26MM5Rls88Gf9J4UYydCno6jgrbHBl2Bg36TdMM1Nqo/aDSnkwn741Nwtfs3QeOdjwdyd+wVjrKB/Jzk", - "SVFDuSR2kXHXpJPxS4GAUELWRoC8JRJWgM6JQoA/XU2w9I2auQhZlHNxIhbiXsTa39vbO3z8aLcRXtnZ", - "FQ7OCOS/5Vme2hkUjhi0RO235+dou4Bwpk/nN5kIIvXiTDCk95whm9arX3Kp1LLHng9LahiWHGts3/O4", - "FuSXlmOxi7JAB0+njJtZOuVeaO/t9R/vPzp81OwYW4lnJD6upjC2nbX0CxIQOi/tfBu02u+OzpDuXUxw", - "UObwd3b39h8dPD680azUjWalBGYypkrdaGKHjw8e7e/t7jQLZfJprm2QXunAlmmX59B5kMKzGx5QLJPe", - "Tt1t4eMSS7qdFbrjgp/9rsaloqP9UfevxrEejXqD7V/+/L+77//j3/zBVSVdhySiG5IJSDJXZNEFW2bm", - "F4EUnspe2TMJFNyaAbaxSYrgGGK6gitikzPgj8WJP+pnN+niNY6X1rKzewipBrO/167Mnxx0Ca7Lbqsr", - "PWVz19uqn+VNHKvzQHkqoVda8OlFbc2cFhn9QrD3VhP9jf/q0ePUlQfQbHhTn+fVLs5nWM1O2IQvm3xu", - "IkhbxzFnCkg0QykhAXJIGCWhuxMyidryqOCKFkmCwpRYyBmeU2ALcGzMXglWMxAC4EPKpmUn/KUBm4i3", - "Zg6r0yLAuLZhE02c9DstvRMpwMro3CXCuftSIwMClSO/tLbcsSDTNMICVf36V0xZLuKIsqsmvctFPOYR", - "DZD+oKommfAo4tcj/Ur+AmvZarQ6/cEot7hX1B5mctbfwmxIZdx8Cb/oVW5VPL+Ao9o2329D/Zcmik2v", - "Ge6FFoqN6/sFox8LiF6OFd7f7dc5+tV0WnLxWw6buOmdaVHWd+JdRMNRll3NY+41BrWKZqAsX5TW61st", - "WGxXuTUuc1io7XSlLha7DNdCTHQjBqeZ0bhqFXCz2ZYkKI++f/jo8UHDoPSvEmFWVMj4CoFlHq8QVGp2", - "6rQJN3z46PDJk739R092b8R3OgNSzf7UGZGK+1NJoljhhR/14X83mpQxIfmnVGNGKk+olBDx1hP6suLo", - "5sFINdqMVdWp8p106pOyYNNMdFjBLR2VWK5CruA2mUwIKORGBm7dfDIVZ7VGcwhwggOqFh7JGl+D/w7K", - "mlSCahr0XpmsB6S2bxsXqSmXTMe5f0TbDY7+w0jMFVw4bJzbQqbjOun8TXVUI5sbh7ewovlpoHgxGOFz", - "UrjOgImusSxZS/TvQJGwU8gFXTWrmRbNq4o4XM8Ki+SOBr7AMH8RkeL2V7azIM2VmOQqxFddofVHUHME", - "4E3XxHDhuZE90WbBeieXCn2wF+DtvhqNi1lnVqb1KaWoyW/dm4/bLIv18nfmBrv5eAXPiJt8WE3AAfho", - "52BBnvfdKaFEDTYpLtbnVLyDMHpjG7hVIL01K2wklt4+vpP4+aXtOC+4hTV3gnRf+evElQy0B93+Xrd/", - "8G5nb/DoYLCzcxfRG5kxqE5F/vjTzvXjaBdP9qPDxeM/dmaPp7vxnted5TvK5VlJPlxJ7WnXnhBRTblS", - "TVUkSUQZ6crMHLXeMr8iRssoSRO8AOZwhSR3E/HBVWdacdrPy4ssHnqscuBUk8ZuwtHPzn6lDFSd/snx", - "6mnfyr5TnYgfwapTAXxqNhmILNxplo0OTpIXODUT9aFByUGhhJjvV1Cz3+whrqNa1mfezjBP6eEOiLPh", - "ljAhf70Edx+5XZ11pHIhGcNzMclL5q/7bVOOGLeRuqSssauaXEmPRU0dShtojwqNUZvEiVq4oFCnGN66", - "mRvLUdahlxf8xu74/SffIpDwYmXk4A+eHrjoceQGWetrtIQLteE6fi3TcdWb16hybZrDsvdpJXmbVCvK", - "tK4qCW5qc4Oe1obKTdNqLoEblAGv08znJ87VX3V1wNcpnFeaFwsrK8ykfm+Mu9lX1kyn0hVLvyXIrNZ0", - "feyZcdnRLFK3mgfTpGoRFNSwFkAGsBoEmWZ9WX2/2gv2FH/MRgDWCMslNg7WUahR9fIppF966/Ih0onr", - "AqZRrQby9OuKyTusWt6MVdXlnUOj9+BZ+rOCEtadrQpy5mN0Vhew16SLBKmganGuSaX11SdYEHGUGjQE", - "GgqLgMf54BB/+eULaJcnHiXTSy1e0AAdnZ0AlsSYgaUYXZ6iiE5IsAgiYsPnllzdQEB88+yka+J+s6IN", - "UONVAUBcbu2jsxNIz2urq7b6vd0e1LviCWE4oa1Ba6+3AwmINRhgiduQjgF+WvuRPodwA56E9qZ+apro", - "rwSOiYISGr977DCKCJPeQaLxIreY50b0BFNhjedJBCYiIy9Q3QG4/zoqP2gVEhGY2+umd5tUC6s8I8kb", - "u8/vNX7IhDNpdni336+UHMZ5Htftv0lj3cnHb8SCmFrwyz60S64Gjg2ye/Cl09rv79xoPmtTr/qGvWA4", - "VTMu6CcC03x0QyDcatATZjT6roIYsQ3zgwc4VTxyv7/X+yXTOMZi4cCVwyrhso5/IxJhSP44dpVse8gK", - "KRABKGc8jUKoKJOYVPearmKksOhNPyEsghmdkyGz14lJo4sFREjHSCOZ0cyUz4oZ2uy+oUNEqqc8XFSg", - "m3W3rbvrOr4tB/CNazFnEn5SU5TZR+JN6mkZcG/ObcIwU3kmY5Nz+oqAY9qEfvR22MjDUlNA2BYC5Q6y", - "iPvdLb8NEgLI/Ob74+ydKwlevvW0AEFZEKVhzhqUSzF7EzCZ0sA2NfcV8XBSL6GFBUox1s7dwYyHxMQ/", - "JQs148z8TscpU6n5PRb8WhKhb2obP21hbfPSWtSFegk0hhhmk6lFj7ltprj9+YosvvSG7CiMXWYdW4gJ", - "R5LbnOXGk5VKlJUDA9ytqeTvlwie2domJl9wMcWqmSZPVZKqHjILIcoGfUNzyMArZyQcMsXRZ2GKLyy+", - "bH/OR/wCLDbBocaTQhOzpO3PNPxSN2s5wnr1I2jqETwIAGDY0jfNsKV/TwXWLHYqZwgH4G+rHxa3tG0O", - "NhfAvmxVIRxghhKepJFmBgGpTCr2Uh+QQANHEVJwlNy3mimCnaxZjzUn+7JEWluyMf5VjhHkiywcpv7+", - "4VZrXcWFcvf/ff7mNTIMkd6FssPbkD03DNgAfR6Cg9uwNRg6F7dhqzNsETaHZ9YPbtj64l+hJIEgPq0A", - "TAAuS6iZDs3ysFLYJcpgeq78ll6/nhoOZm7mMyzRsEXDYSuvcb0F0EqlVbd3u8AL/qJn9osZpkPDX3q9", - "4ip//2x6GejTnMQjxa8IG7a+dFDhxZSqWTrO3r2vWXCNUfC8RIpQ29w+Wy4Zk15h4SI2NxdmIeKW2kcL", - "hFFOA4vKhzFlWCzqCs3zVNU7rJtcVbZZjlEH/f7WescZu1QPg11qqM/ilyVWbPebcSGWA1vmQmwtfRsa", - "o4FpC9oD77UBNugpDl2ei5/83hp+z0rbBU4Ovi9eCgZ9I2LUoxV2TIvnkWPHVsouBi0gNgxEEefmZiQR", - "6ti5HHmLMklVBF2WMfbrTlkAU4wc/u1vAP9g3DzjP4z7ZFPj4sjUqXL5rx8WOsJmOUTs+OXll0R9DxjX", - "3xQpdYVJ7hF/Hwr+vCSWCcyBVqFm21Dzs6iMqcYwC4JjaXsxjbXgeg5z6p4TptBzeNqz/zrxB8JDP0R8", - "+mGADAgjPkURZURaF4TM2qEvRQtL+Mikbcy+s1lQgxlmUyJR29yf//z7P2BSlE3/+fd/aNba/ILjvm3c", - "2yGC8sOMYKHGBKsPA/QbIUkXR3RO3GIg5InMiVigvb6tcgyvPDlV5ZAN2VuiUsFk5tiu1wUwMR3aEh96", - "PZSlRCIJIIQCdhPrcW2Uoh553p1lA8qNnujOkgBmV1BYgL4VHQ6ACx1lVFEcWWGs5VermTWXlGpV/e6S", - "xn89fVHkozLY2zUTvCGBARD7zh28sItG7fPz51s9BOy+wQrwqge5Ie/GSgK9nzRpPU0yFKVMUADKhjYV", - "0unXaoePbZtm6mHb4w+tH64rGFCvIDYKESJI6AD4U3hooiz2w80pjn3a22NXX7BefXv79RaHcH6KjSTj", - "b7fPDveWYW6LZ+Yguw+ZGLVt3bMsp2WpQud9If1GrpFCQdjsLkHcZNLcmJz2jLNJRAOFum4ukG4jJpns", - "VkaQh0IO3tpZI+zWVQ1oLV5426X4jNqrLwvVyO/Au789KoPe5BrJg25zXPt5k6xDnWMqA66/LWBLN8CJ", - "zehp+JnsnBaxaJ2G6hieZ1fOSv7pOCsVbQ/k5nRVduiUVe+GDRDF4wpBvEdCWMk2WAhTf0jYfJHtoqul", - "vEKV9X2hZn9zXNCm1Vo+NH9Ieq2wAjZNBWdZTas69LJVr+5wo+0InoWfE+FOtZmoyXKXL8t8ioIZCa7M", - "gmzJ71UcwYmrCt5EFjb9/dCisKk/dgMWxu7BT56lgfSbw2qVxHti8zfencALI9xI3v12lmCLYB4gg2/K", - "2Om0TWpELBcs2PqhjMEbud6qdcYf0Ek6S6PI2UTmRKi8mlrxUtj+DF5M65l9d9pW3g8Xb191CQs4uK1l", - "Lld+rsoVQfq2LL/ZMLOUn2jSREgEUDnEqOeov2L/jXchyjLq//vuC5tT/993X5is+v++d2Ty6m/dGbL0", - "N0WaN82CP2Dk0xw4LQMNSJMpVbSOZc1aNeRaXfsfm3G1te1uwrpmgP7JvTbhXovgWsnAZmUG75CFtdXb", - "7sdokyGbD9rwyrk0/mCs62b1gBYjXc4OKsuGEZuUkYu8YpotH/7wfC5phnHFe6ShQjs/kCvvE4e6J8cd", - "WwzPlLDLAkw2pN5289g4t2vH3bxu+yge02nKU1mMXYHah0TaYKeIlAnwQ+PD8+u5lhP/jrG0v8mrY+OM", - "9k+8vyMRoLqhhngbG9U6IcC1aioE2PZQZdAUvjCxb29dQQ2bXGSrxg/RlYtpisalakXL/pG+edUKJ+hC", - "iy+5zIBAjBgM2X+5T35XBMfvf3HhTWm/v3uQvSNs/v4XF+XETh3eEKYEJRJhQdDR62OwEk4hNh5yh+Xx", - "fdX5mIxgpra0LXv6Ly055UbT5qKTQ8+folMj0akArtWiU1bY5S5lJzPIvQlPDt98ALdJPH6KT5sQn2Q6", - "mdCAEqby5LlL/mU29/YDjFNj1pJU8Asp3cCNxae82tJqzjTP/LZxn6Bs8M1LTS7J3MP0t+cmwiZ0ckp+", - "GdYLKt8bPvQ3S5w3L6A8ZBQzkkAVdMuEaHtic/f6GYQXXFw1xTxPKspvjoDfnjsprvA75E309CBtyf2z", - "KHB5G7d8jTRlzmUDB3Ipv+h9eoM6SFip1wRYUjbNamReUzXjqUnXMrIPTf43fSpsIRZgeQLb632TFz36", - "BhjQ11whGicRiQnkh+sabILipGmScJGVRKOykI33ZuRPH5uib67JmmMrA3eQzVkMWrysqCko9Je3y0s1", - "Iz5dH6CbDe6iUT0RukN2IU32mA+GFf6AMiKLFEeSRCRQ6HpGgxlE6+pn0L8J5sVJ8iFLz7E1QC/hpBYT", - "hsDgbUkExREUnuSRqZn6YR7HHwbLueYuT0/hIxOoa7LKfRggl18uuyCkblWMvtWriLBU6LWNKW5rTBI8", - "isyOftC3UGF9WzYuN89kMmS+GF1Grm2HdII+FMJ1P9TE6zqC+krv0j3xS536DFhmLYojAYAzuElYWKMj", - "01DzR+ru9L0pUxtGDZtp3HHQ8NJkXvFpln2rhMo4SZqir50mYPE8jlfgMGoX8nlLFfJU/VmqkAgBH1vs", - "rkNu1MaB+UPhK42ozFbxchnRAf28ek2TAccLKk1UC+mXzV/zOG51WnY+noK+Xx99Xe1wWc2md6YQYv2T", - "075J8HSZ2Beipys3hy35UM9y20oWP7y850pu3zMa3oN+LJ8FZY5Vgb3Na5k/rKBLU+SkyouZXPO+M5JV", - "Sak/JWWl8nme1f5fUEQ1a62WttmwkJqB2CeZlSo83Lt0mhWc+CmhZhIqFyhMzXCVki8/rNiZERSUspLk", - "adnT28qeWcK6DMxQwo+tNAjkNG/7s/t5cgt24TuhhJ3aIil1qZHyRX8PJLemnFgjmntPfJK9VgsMwj2S", - "YFfYbNMUOIOKFvcyKvddkGFz4DJqXKQ5SmAmqatZ+JMYl9SARlN6W2LsmM8lXWCBPFPWTSJcR5ctn1pL", - "gG3RpB9eXstllR9cYgu4EMadDLzUHlKQY8FmWBA92wlOJelkB6bj7NaXp6dbdYdGqJVHRnwfBu3bcQ6V", - "ipZx6C8pLGjost8/Oz22ufKpRCJlPfQmppCS/oqQBNJbUp5KBP6AvWKVs5pKv3kZM8KUWCScMrV2FnnT", - "u5nMl1sl/N4wnbJh3j+8WsnWqH1oRApoh7697QJWC1XKFPfzmumc2YoykzBfMx94zFPd+1LlNTShEZEL", - "qUhsbHaTNIJDBJlBbCZZ+53xXesgqiQU3u6Ar09CREylpJzJIRuTieZKEiL02FCfkUakYH7wWbbOFc6o", - "5pkhfd+HaQuKsYE1B6s6qJXrsOEkcXXYfOaTrHTcraf0AmxVSC7iMY9ogCLKriRqR/TK8OBoLlGkf2yt", - "NHaN4LtvnSf39idLQ/qETbg3c6DB2QyZfwQKd1Iha86Y/+DI2ktSPCyO/sBG+8maXEvXBMERlB7N3GxR", - "qmhEPxlSpzuhUtHAFGPCGeygjowZrzdkp0QJ3QYLggIeRSRQTtewnQgebA/Tfn8vSCjER+wRmBwQvPrX", - "MYz47OwC2plaN50h039Ax++OzhDVMJ1gKzIXJmprwqOT7TdrzP/nAKZ/YXnMLHDVsfBv+E/L7s19KGvP", - "kKw5ojxZJQDx5IdXGFgO7qe24GFqC8CJPVtNeypwAEyxnKUq5NfMrxkwtVjl9mfz42RdKITCwezSFZr+", - "PrhdW5d23TBugQ/iUNo1hcRkNr0Xfb0tHfxAEz9pwLklABNTDOrw3wKmJPmPht3f3lhXhON3aKmzEHVZ", - "g7+bs7Xpm8/OwUX4FeHxUI65wTS3EqhEWdQ+ZeGMa2WzIBWCMAU5YnLWMsAJDqhadBCOXJlWW2op0yHl", - "JefHguArfdP2huxtFkhpSz1p6arjRCsUUnllerDSUw+9mRMh03E2OQSEych5AHxbqTXAUWBKnJLJhASK", - "zompPSprpK9sKneZ0TcfxLPR7qUF3UMTOfw4AbuXo4WVOkqecrV5Hc6zVs3yOmS9FrxhCp4iK32eR67h", - "CG6im6jsPINf0Vq3ePvqZt5rv+mPGo5d9pLyT8K++spV/rD5884L3ipNs0DkKP/QEjIUZl46uyWPr/WR", - "4Y1dvO7S5WpdZHg2+KYjw8+9Xj8PLHEVLvlx1YWEf3+I0N+su/GmQ8IfNm5p3kIuga6eEjUIDf8uMPBu", - "YsLv2d3+FjHh35UDKMT03p8j/nfl+mldGDPXz59R33fp8WlCvyHCtc7j01A9q4peKTld2jbN5Cbb4w/N", - "0lt15g0YercPP5O6NZAhCsBy13KF/sBlIO0JIHGiFk5fxSfgmZNnIJT0E/j3+ULrMrX03UW03UJj++3Q", - "w+Fprb72ZzK4jamE81TaJ8cPPwNc8cyVbpptfQ11sQhmdF6K6Fp1gi2IEkG6CU9AExsagFl4uMtNYdGb", - "fkK2+96QvZsR9xeiLp8GCVFIBQlUtECUKQ4UwYzxJ4kE16IBvOdi4VPwFk/uC8HjI7uaNRekPVNWXZY7", - "AsaLrr61unNHbVYo2b7CqHWKP9I4jYHgIcrQy6eoTT4qYdI7oIkWhRCdZCAlHwNCQgk4uVWc8E6/RvdJ", - "P5HRdNxklisSdbyxiVBQkErFY7f3J8eojVPFu1PC9F5o3n8CrG0i+JyGJr1uDtQ5jwxUd2oAelPNrGMu", - "kMJTaV3Hc7HDzPLeGZsmt9T0E03KtMJ4S7YGrTFlGCa6Nk9G+aAZx109HqbgPpcfKIdOrZ/3WrWuN2AT", - "FxkQFeco0nz/1s+77yHffUUHCHfRla7AZslPm/lENHRVuIvEp5m/zGaV25ffjxm/UAf5ASrY55mUWqdc", - "/75QsL+5+2HTSvXLB+z29ZI4ibygUIcOdI8+hHnFAxyhkMxJxJNY85qmbavTSkXUGrRmSiWD7e1It5tx", - "qQaH/cN+68v7L/8/AAD//9ibNZqCHgEA", + "H4sIAAAAAAAC/+x963LbOprgq6C0M9XytCTLlziOpk7NOnaS4zlx4o3j9E4fZRWIhCS0SYAHAOUoqfzt", + "B+hH7CfZwgeAN4ESndhy3MnUVB9HJHH58OG7Xz63Ah4nnBGmZGvwuSWDGYkx/HmkFA5m73iUxuQN+SMl", + "UumfE8ETIhQl8FLMU6ZGCVYz/a+QyEDQRFHOWoPWOVYzdD0jgqA5jILkjKdRiMYEwXckbHVa5COOk4i0", + "Bq3tmKntECvc6rTUItE/SSUom7a+dFqC4JCzaGGmmeA0Uq3BBEeSdCrTnumhEZZIf9KFb7LxxpxHBLPW", + "Fxjxj5QKErYGvxe38T57mY//RgKlJz+aYxrhcUROyJwGZBkMQSoEYWoUCjonYhkUx+Z5tEBjnrIQmfdQ", + "m6VRhOgEMc7IVgkYbE5DqiGhX9FTtwZKpMQDmRDWNKKh5wSOT5F5jE5PUHtGPpYn2X08PmzVD8lwTJYH", + "/TWNMetq4OplufHh3eLYL/d9I1Mex+loKniaLI98+vrs7BLBQ8TSeExEccTD3Ww8yhSZEqEHTAI6wmEo", + "iJT+/buHxbX1+/3+AO8O+v1e37fKOWEhF7UgNY/9IN3ph2TFkI1AasdfAumrd6cnp0fomIuECwzfLs1U", + "QewieIr7KqJN+VR8+P80pVG4jPVj/TMRI8qkwqwGB0/tQw0uPkFqRpD9Dr07Q+0JFygk43Q6pWy61QTf", + "NcGKiCLhCKvl6WCpyL5DOUOKxkQqHCetTmvCRaw/aoVYka5+0mhCQfCa6fQbjSZbvmqpOclRLOtGd68g", + "ylBMo4hKEnAWyuIclKmD/frNFC4MEYJ7KNQz/TOKiZR4SlBbk01NuxmSCqtUIirRBNOIhI3OyIcIZjN/", + "42NEQ8IUndDy/Tbo1MXjYGd3z0s7Yjwlo5BOLScqD38Cv2sU0+MoBG/7N6Iv2qLZPmBKQSbL8z0H0g2T", + "CDIhgmgc/8bpYqIwMMDB59a/wayt/7WdM+hty523z+x7b/FUAhEUfE6YvmXrvoRDOM9f/9Jp/ZGSlIwS", + "LqnZ2RLFs080+sERIfjCv1d4tApHCpgoFRar7xW8cQs32KyvEWwuzKtVOgpk0g5Togi15PLZnDCPwBRw", + "puyD8o5f8imKKCPIvmHhq+mjnuCXiAN5vI29dVo5SJcJgV73VxAy80PNaPpZp0VYGmtgRnxahOaMYKHG", + "pATMGnZmB8pXVwv+89KVqPAtLMloNTU5p4yREOk37SU3b6JUgtS6tH24GVdUjeZESO89gmX9RhWyb9QO", + "FfHgakIjMpphOTMrxmEIdxBH56WdeCS3kiiME00Q3YAgUUikOLr49Wj30QGyE3hgKHkqArOC5Z0UvtbD", + "m3eRwmKMo8iLG/XodnN+vYwhfgy4yC5GHR/KMNAhpqFeLXuaevhOK0nlzPwFdFyvCvigJgMavSL993vP", + "po+BSBiNoV5/+kqK75cjXycGSdA04vosFihl9I+0JKT30KnWNxTSTIOGJOwgDA80+cap4t0pYURo+oYm", + "gscgsRUEadQmvWmvg4ZatuxqSbqLd7v9frc/bJVF4Wi/O01SDUKsFBF6gf/vd9z9dNT9a7/75H3+56jX", + "ff/nf/MhTlPp3kmWdp9tRzM6yC22KPJXF7paHVghUfuojzn2U00zNnXqx6fLgojZd8iDKyJ6lG9HdCyw", + "WGyzKWUfBxFWRKoyFFa/uxYusLYVAGFTDbINgaSiUAF6tyN+TUSgKXpENELKjibqVMkOwlonB2KINNf9", + "TxRgpu+IEUC4QISF6JqqGcLwXhly8aKLE9qlZoutTivGH18SNlWz1uBgbwn/NfK37R/d9//hftr6L+8V", + "EGlEPMj/hqeKsimCx0ZKmFGJ8jVQReK1YoE7lTQCUTCm7NR8tpOtBAuBF/7TdotbdepG+as99iD2aAqv", + "50QIGjrOe3x2gtoRvSIWnZFIGRqm/f5eAC/An8T+EvA4xiw0v2310OuYKs3x0pyRG+tRr3iEv7dIMOMg", + "i0QR1xvKwFcj6Di4OEXac0QnzvIikdXmgfdisKvBkb04v9zWVCzBUqqZ4Ol0Vl6VJaE3Ww+VVyPKR+PE", + "tyYqr9Dp9mukCTyKqIZORtB3+v2zp9ty2NL/eOT+sdVDJwZksHx9flxYPiNnWBCQkkLEGTo+v0Q4inhg", + "9dWJFmYndJoKEvYqZhIY3YfwhCmxSDj1CckVzMhfXUaQbjd/egM82B5Tti31MXSDm8GdsPk3iGrP2JwK", + "zmItLs+xoJpulYxWn1uvXp88Gz179a410JcoTANrATp//eZta9Da6/f7LZ80pDFoDR14cX55DCel359x", + "lUTpdCTpJw9pPcr2h2ISc2FUFPsNas/KlNdIcAgOZ9jae/HUINfOC8ArdyghlfC2G8UMXMaY3RdPfdgy", + "WyREzKn02TR+zZ65ky/QSUOYyrgtiZgTkSEtYHGvIB8GEU/DbmHKTmtCBQkE1mjX6rT+ILEWeOafNOrk", + "a/d85zc1NGLua7g2jhLKSC3b7jx0VnvNxVXEcdjduWVOy4jSYy9v8ZV5UMYLi0skQ6VWZ0nNZOE1DdVs", + "FPJrppfsocf2CcpezojyR70THP3z7/94d5bLsTsvxoml0Du7j76RQldosh7aq9tmG0kT/zYuE/8m3p39", + "8+//cDu5300QpvEzLPmPjLmovJW/zIiaEVHg1O6A9U9GyYDPkcOXwvQl+1PR2bRElPmciAgvCkTWrqm1", + "0wdKV1mVoArul/1Ok8wrpD9eQ3L1aI6hv6gqPrt9P1H1LMqzpqf6flse0GQl2UJ2ds/sn7vLS6pZ0RVN", + "RlMtQ47wNLOfrXIDXlzRBMEXXfjCHGMUmcsbpnpkNOZc9YbsLzPCEJwdHDD5SAKgU1JhhY7OTyW6plEE", + "WjMQgmU2MmRvC6TAvC6V/l+Rsg4apwoJEnNFkBVQYZIU1gIvjwlKGXZ+xt6QFaFiN1jFKwuWKyIYiUYz", + "gkMiZEPImI+Q/agWOLDVCZaKCEOh06QMr5Pfzi5Q+2TBcEwD9JsZ9YyHaUTQRZroO7xVhl5nyBJB5oSB", + "/qL5DrXz8gniqerySVcJQtwSYxgssztYJ9j8xfmldaPKrd6QvSEasISFJIQ1Oy4hkZphhULO/qRvLAnL", + "wxbnrwDdf5c7LclwImdcjRIe0WCxjote2NfPzdtfOq15kKTlY9qtHtEr8H5qgMypUCmONM0riYNeZ6hx", + "s3vEfuPFL6oflpZlGItV2YvVVIM0I4PPfVko9iuNRtKpVxodxOqVRh4nWm21ZtYm0D/OP8kF3SvKwqYD", + "/KbfvXXhKTPYOaS6a/kpEaSbJlOBwUt9e9JT5aQBsvUnvCaoxOc9zCAVpFLxuOBDRO2K1ZKW7ZtlAMx5", + "1NXnAtLjHYvGZpvLfvx4YZZgrmsdAx5Nxx7Tu+azlKEpneLxQpVVxJ3+MlHwX0E3vu+I6mJcDOEg4Ujx", + "1V5+OkHu3SbOOYiIGSk+mk+oZ+RMHsvNu1SioBJQY8mZHqKbBNRyhg66nlEtwUnkgADM4d1Z0eTSG7Iu", + "cLMBOskmyIbNhtQXD1wAMESbi8IiKHhz0HixhTB6d9ZDb7PV/kkihhWdExf0M8MSjQlhKAXJn4QwP3Dq", + "4gJSqdkjVdXPLRs08UFbYFni9lkPaY03xlak0NcixooG4AkY08p+wHNrDkrPpFkDKwo0jQSQVbERb8iU", + "SiUqkRGo/eb58d7e3pOqKLr7qNvf6e48ervTH/T1//+1eRDF7YdA+cY6KtMZ61spUqLjy9OTXSv3ludR", + "n/bxk8OPH7F6ckCv5ZNP8VhM/7aHNxIkdbtk7SR3JqF2KonoOlKrsdHnQip4ampcRF/t+blRXJfzUa+C", + "gdndW/3mXUSC+eIKrFf75rFaVeK5NjKhsLml/ehftcSZ35iC5cs68gLqdXWeUHn1VBB8FfJr5uHnWuCT", + "I8Ov/CbpVKv24wUiH7XGQEIkOFcTaUxfZcF3Z//x/uHewf5hv+8JgFpGfh7QUaC5UaMFvD4+RRFeEIHg", + "G9QG20OIxhEfl5H30d7B4eP+k53dpuswmnszOGRyufsKtS1E/uyCad2T0qJ2dx8f7O3t9Q8Odvcbrcqq", + "DI0W5dSLksjxeO/x/s7h7n4jKPgsIc9cQFpVyA991uckiaix+3RlQgI6oQGCkDakP0DtGNgZyYwQ5Ts5", + "xuFIWLHTy0cUppFcafQ2k9k3TfxinEaKJhExz+BAGulOsPMTGMnnUKCMETHK4vVuMJIN41trrHV7yV5B", + "pXDMEujOqASJJBekKInCgbmha+kcnGa+sPd1eGD30BAbXmo1qRuROYmKSGDYkV5szAVBGZ6YQyvtirI5", + "jmg4oixJa4zlNaB8ngqQS82gCI95qoz1CA6sOAkEA4BOMtHkulkMy3Murta6TzV3HYmUMT3MWsPPURTx", + "a33EVxo2wJkxsl+7KJ6CAJhZeYwtzD6X6I35wtjK8p+TVCHKFNeaKAvHiw7MREJ4jyFBpOJASXFwpaVN", + "O0xTSdMvi7zSQoizxJv5ctq5ITdEd2KswLfpi1BYTIkaSYXVWolFY8pbeP8CXm8cmaE/XGtsaQB3Rq43", + "AXQIR+lqtO1KhpO7gfgqf2Jmg8hfAi4saEh6CG4XOChcWGzlpl0oniQkzGw9vSG7MFcl+0miOJVgdL0y", + "cFAzQgXigk5peWJ7bTbgmLwJKjps+mp0LH64LKHCQ7DK1196PFFEGAi6TIFiiJ89hFanZWHf6rQsJSqD", + "xv3ogUjuLV9a4ovzy5u6CRPBJzTybBdM3Pap1bacA+3lfv+iu/N/jBNd4xuIaJQZs3jMQ9KrJOPA+804", + "z4vzy/O6NWWZUKi4uqU9ZY4MD+XITNsOItZEH2CGxgRZDcahv2Ys2SS57P3EJ8tOBI7JOJ1MiBjFHuPZ", + "c/0cmReMx4oydPa0LM9quXl5aD8VPC8dDqjCExzYRJZm0PcY5yrb6BSg+d5/XG+IYcN1Ia/6qIR9x0a9", + "9tCrLPcMvTi/lCh3PnmsduXjrQ15Op8tJA1wZEY0EeyUFY1tgJyNJeTz/ENrlvTIybFXNnQXAbXn0ySF", + "a3jxpnv6+t12HJJ5p7QmcBjNeET0urcK1GLuAljz+KwSkZjXWS8MYsimF6gAq+wGNwZS4b56oKO4wtFI", + "Rlx5VvNWP0TwELXfPTeBhHoFHZSUjlL/XoBCCb8PvDdGU6S6aS9gwqr5tHTB11qyY6NRFLdXmtR3VX4l", + "ODKZqmV8znMo3MHzq/JB86u1t9cO4pv31MX2NAh+PD47MQJDwJnClBGBMvNdKVINxKFWp9XVPCrEJAYP", + "6uQ/V0et1ZjjM3RZZdA9XkpzuxNjbk1KhiZy0ZyEKMaMTohUNiWjNLOc4d1HBwOTRBaSyf6jg16vd9Mw", + "w2d5XGGjo9g2UViFiMOenH3bOdxBNGGTvXxunR+9/bU1aG2nUmxHPMDRthxTNij8O/tn/gD+MP8cU+aN", + "QmyUd0gnS/mGZfel5lnm94HeCSNBhpAcFPg7y7Gr0YM0Skf0EwmRNwxf4anWawymflu8/Tdk6uVp5qqQ", + "oVeMJWiQrUc/rbagOoEK3rFzpkzRKE+AXLadflUKq1yZ2bOU1ZMQluXyRJH5K+Bsrm+TL7GnRPjds6XD", + "uDbK3SikHqz+i9X8Qq2EKQiSXX/3Wts4SdajsF9ozGhh0yRFG/rv4Ur3zgG+xvdWnv319L//+L/y/PHf", + "dv54+e7d/8xf/PfJK/o/76Lz1/cSELs6W+ReUz5WBuyAw6mU6tEUrc6wCjyC1oxLVQM1+wQpjmL9cQ8d", + "g0I4GLIuekkVETgaoGELJ7RngdkLeDxsoTb5iANlvkKcIT2UDWzb0h+fG7OQ/viz0zm/VMcIbQSbsEDO", + "gk1lOg55jCnbGrIhs2MhtxEJfn/9V4gCnKhUEH0iWraNFmgscJBHruWTd9BnnCRftoYMNF/yUQm9gwQL", + "laWyuRngoO2qTFyBfZ2EaI6jlEirOQ9ZxnfAFKAHMbabXmYcAZt9xeJaAxSvWsNFOfLysN/xnCPS7+mD", + "jKhUhKHMCkIlIC9quxDaw36JbBz2D9eHsGQ4tAL9ALuXc8wcUja4HwaBYWpDxEczpZIGNnZNp8wdQb++", + "fXuuwaD/e4HcQDkssiM2yh9OkogSaWyHKgIZyEYtb7V8JnFzug03ZIxn8FnUIEj0GUyM3r68QIqImDJD", + "99uBBueEBnp/4OqnUqYaFSlGR8dnz7Z6DartAGyz9a84x7fZDiseZWdMq7MRZhiv4dtBpycdLYbZG5oL", + "aBB685wLFBkCk9/rAbqUpBwiCUdlvP3mJKNFbpEzVH3Y2nIjJlVKMUBvMrkQZ0vJUmxzZHBD5vcShrUO", + "GRMXtDR6p7xWiHiy+pIlbRAFhBWy/k9g4fWkYPX190Ac7jxnVVvnze520UiqJ/OjRn72dy657N1Ud71p", + "7l05ZL6QIpGl3zXPm7uL/LNlPe4jVaNa5zzSj60r3mkd787QDEv2JwUPK7rHzt7jRlVr9KxN3dpFhzaf", + "mCVlt8rF32fuWJOJcEWjyEQ5SDplOEJPUPvi9MVvpy9fbqEuev36rHoUq77wnU+DNDyH2i/OLyG3DcuR", + "8wzVB0biPHiYfKRSyeX0hEYO1tVpf7+WUvO8+R5bt5iv57zSS9vYRCbefYb+/etkAa7M2/vW5DsrJN9R", + "7l0tUfblrZXps/n5drPo7mQ5pXw4H10pyhIunvurU+A6LeqJZT2SmnSSEJ2e5+VFcmOVG76ypye7vZ2D", + "w95Ov9/b6Tcy+eFgxdxnR8fNJ+/vGmPGAI8HQTggkybz15gOLWIboQ9H13gh0dCJ5cOW0QMKCkDhulvR", + "vZE7dznT8OsSC6uCyLrUwZukCjbLAfzWxKtVBccuyqXGGguHj/76TVXJSFORwMZS2K9GN7GmExTwNAq1", + "ADbWV9focyS0aqckKq/iBrf9kl0xfs3KWzfGUU0A/kiJWKB3Z2clE7wgE1vQqsHGIQaj5hx4cqNj2F0j", + "o69dTSEdbxMpeFVSWmBht55wV7TfuThNg3UN7Hi5KOr1rVNmwK3PfsWeKhaYkMxHaeqTsPQjl6lxeXl6", + "UjpwjA92DvuHT7qH452D7n7Y3+ninb2D7u4j3J/sBY/3akpGNo+t+fpwmfINrc+MAsCDNdMkwYUDfYey", + "eJdxqlAWC6cv57EWVVFBJjZ5QGBgsMFJegRgz4F+Ei0ysXnlx+dYX1T3bQL/Wv3FxSxVWo6Cb+QsVUj/", + "C5ast2DVjtVDmDs/QK84fCNcECnjVf3FvA7BWcuvV3Wdtg0bcuGlMJklYAP0PCNaGdmzZK4tif3T0FIb", + "+QxR3Vul2Dp7WoU4sU7LgLDVaTnIQDzZcmSZXYg3aaKINz5rP8ER0LA8cidVNKKfzJXTS6dS0cCoexhO", + "s+7a2WIJJBwZHlznxzPhIJZPZx+5W/3uDLUh//DPyGqD+l9bmc+veIX2d5/sPzl4vPvkoFEWQr7A9dT4", + "GIKVlhe3ljQHSTpypXNrtn58fgnMRzM2mcZGvbd7LwR9JoIHWlykDOW1ePPJn/SeFJMvQp6Oo4K5yGZt", + "QYR/k8LJNU6uP2g0p5MJ++NTcLX7N0HjnY8Hcnfs1a6yifyi6GnRxLmkt5Fx1xTG8auRgFBC1qaQvCES", + "doAuiEKAP11NsDRHzWKMLMq5RBMLcS9i7e/t7R0+frTbCK/s6goXZwQK5PIqz+wKClcM3kTtNxcXaLuA", + "cGZMF3iZCCL15kw2pfeeIVugrF+KydTKy54PS2oElhxr7NjzuBbk76zEYjdlgQ6hUpk0s3TLvdDe2+s/", + "3n90+KjZNbYq00h8XE1h7Hs2VECQgNB56eTbYBZ/e3SO9OhigoOyirCzu7f/6ODx4Y1WpW60KiUwkzFV", + "6kYLO3x88Gh/b3enWS6Uz/Rts/xKF7ZMuzyXzoMUntPwgGKZ9HbquIVPSiwZh1YYnwuB+rsal4qR+kfd", + "v5rIfDTqDbZ/+fP/7r7/j3/zZ2eVjCWSiG5IJqDJXJFFF5yhWWAFUngqe+XQJrCQawHYJjcpgmNICguu", + "iK3ugD8WF/6on3HSxSscL+1lZ/cQiiZm/167M3+Z0yW4Lse9rgy1zWN3q4GaN4nMzjPtqYRRaSEoGLW1", + "cFoU9AvZ4ltNDEB+1qPnqWt0oMXwpkHTq2Okz7GanbIJX/YZ3USRtpFnzpeQaIESSoqgkDBKQscTMo3a", + "yqgQyxZJgsKUWMgZmVNgC3Bs/GYJVjNQAuBDyqblKP6lCZuot2YNq+sqwLz2xSamPOmPenorUoCVMdpL", + "hPP4p0YeCCpHfm1teWBBpmmEBaomBqxYslzEEWVXTUaXi3jMIxog/UHVTDLhUcSvR/qR/AX2stVod/qD", + "Ue6yr5g9zOJswIY5kMq8+RZ+0bvcqoSOgUS1bb7fhk42TSyjXj/ec60Um9j5S0Y/FhC9nGy8v9uvixSs", + "GbQUI7icd3FTnmlR1nfjXUrEUVYnzuMvNh65imWgrF+U9uvbLbh8V8VFLktYqO2MrS6ZuwzXQlJ1IwGn", + "mde56lZwq9mWJCjPvn/46PFBw6z2b1JhVvT6+AaFZR6vUFRqTuqsiTR8+OjwyZO9/UdPdm8kdzoPVM35", + "1HmhiudTKQdZkYUf9eH/brQo44PyL6nGD1VeUKm041cv6MuKq5tnM9VYM1b12cpP0plPyopNM9VhhbR0", + "VBK5ClWP22QyIWCQGxm4dfPFVKLdGq0hwAkOqFp4NGt8DQFAKHulkpXTYPTKYj0gtWPbxEpNuWQ6zgMs", + "2m5y9B9GY67gwmHj4hgyHddp56+rsxrd3ETMhRXLTwPDi8EIX5TDdQZMdI1lyVui/w4UCTuFqtZVv5x5", + "o3l/FIfrWYuUPFLBl1nmb4dSPP7KcRa0uZKQXIX4KhZafwW1RADheE0cFx6O7ElXC9ZHyVTog2WAX/fV", + "aFwsW7OyLlCpxk3OdW8+b7N63MvfGQ528/kKoRU3+bBawQPw0a7Bgjwfu1NCiRpsUlysL+54B3n4xjfw", + "VZn41q2wkWR8+/OdJOAvHcdFIa7MX12ThKNVOT/H2WvOlJjgBcgNtUL+4739fn9vt/9VST+3VfSzME6d", + "F73wnVXUnSEA0Kk4QuYzX676ci2oKffrwCSVIDgegMcrwQFBEZlAyGxWbatR78PS1KsXb3PpbLBU5jpz", + "B+Xq91sd2qIv4wyYihvH5ku5bbRcKl85rq74/EYdFN1h+Zsolnz+B93+Xrd/8HZnb/DoYLCzcxcZRRmQ", + "6rwujz/tXD+OdvFkPzpcPP5jZ/Z4uhvveUOsvqP6spXK3JVys3bvCRHVMkDV8lmSRJSRrsw8nOuDPVbQ", + "EGN3X0s3bqaRutZlKxjIRXmTRT6CVQ6cakHkTQSf2tWvVKuryz89Wb3sr3IZVhfiR7DqUgCfmi0Gsl29", + "U6esIQ+6LLzYmAutdE2v40O+ABu45t6Tq4GiD0dLRLJ0a96v4N7LHM4jqk+5oGoWr2YV2WtZchZUC/8k", + "VbgciGjvgX7Y6rSiT/tlPLe/Nw85tclGGdLYoyyy/AY2XqgDt3qXplScVthhY7Y5eE5o1lsc7RZWHcpv", + "luzX8Tmb+WPRJi9M5EiqCyQpwTR/vATWSqDjHZVLXyVENq/aVJHHTdxNsUhWlu9wuyWb7Jfr+5HdGaxM", + "3F5dWe3YNeCvFDikpqWxLZWCCi+jNokTtXBp/c4zt3WzOMKjbECvMn7LCVX9J7eRCn65Mvf7By/wXgz5", + "dJOsDfZcwoXahEu/mf+kmo9hfGm2UG05f6BSflOqFR2/Y54yNQJn1rLFXj8zjjKb7DxNq9VgtmOmtm2Z", + "heUse4JDYAArXaP5jXOtvLtVrlHj8VsZ31HYWWEl9Wdj4n2XM5pXAOhcg+Z6RgQpHAR8kOeH3xBk1m21", + "PnvYxExqhaJbrWRsim1p3VhmfVoNYDUIMtfmMntfncdwhj9mM4Doh+WS0gP7KLQ7fPEUCui9cRVt6cQN", + "AcuoNpZ6uh6LVsHEYdXyYRSxannf5n3vxbP0ZwUlrLtbFeTM5yih5jI+atJFglRQtbjQpNJmWxEsiDhK", + "DRoCDYVNwM/55JBB/+ULuPcmHiv/C62M0wAdnZ8ClsSYQagOeneGIjohwSKIiE2AXoo1Bgvd6+PTrqnc", + "kPX/gXbhCgDiuiMcnZ9CgXXbqLvV7+32oHUiTwjDCW0NWnu9HSghr8EAW9yGgjrwp3Xg63sIHPA0tJz6", + "qXlFfyVwTBR0Y/rd4whXRJgCPRKNF3nIUh7FlGAqbPRSEoGP3gi9VA8A+ReOyg9ahVIyhnvdlLdJtbDe", + "C5K8tuf8XuOHTDiT5oR3+/1K93qcV+Le/ps04lI+fyMRBODlSWJYivVyYpA9gy+d1n5/50brWVs82zft", + "JcOpmnFBPxFY5qMbAuGrJj1lxqXqmlES+2J+8QCnilfu9/f6vGQax1gsHLhyWCVc1slvRCIM5XvHril6", + "D1mVHnK45YynUQjNyRLTrETTVYwUFr3pJ4RFMKNzMmSWnZhC6FhAjYsYaSQzpvHyXTFTm9M3dIhI9ZSH", + "iwp0s+G29XBdJ7flAL5xW//MHpbU9Pf3Wi+heYAMuLdrAmGYqbwWvekacEUgMnhCP3oHbBTirikgHAuB", + "hjVZzZTdLX8QCKQA++OnTrJnyIK3zPW0AkFZEKVhLhqUu/p7S+iZLvO2ucIV8UhSL+ANC5RitrTjwYyH", + "xGSwJgs148z8nY5TplLz91jwa0mE5tS2AoaFta0sblEXOt7QGKpQmFpbes5ts8Ttz1dk8aU3ZEdh7Gqj", + "2Z5+OJLcdp0wqQRUoqyzJOCuP0e7xh52bLtTmYrvxSLZZpk8VUmqeshshChbtgNehxrqckbCIVMcfRam", + "fc7iy/bnfMYvIGITHGo8KbxitrT9mYZf6lYtR1jvfgSvehQPAgAYtjSnGbb031OBtYidyhnCASQ86B+L", + "R9o2F5sLEF+2qhAOMEMJT9JIC4OAVKaZRmkMKIGEowgpuEruWy0UwUnW7MfG8/jq/NpgHhN9UblGUPG3", + "cJn6+4dbrXU9c8rD//fF61fICET6FMoRx0P2zAhgA/R5CBHGw9Zg6GKMh63OsEXYHH6zgcjD1hf/DiUJ", + "BPFZBWABwCz1/Oa1vDAAnBJlsDzXyVHvXy8NBzO38hmWaNii4bCVieHhFkArldbf2e2CLPiLXtkvZpoO", + "DX/p9Yq7/P2zGWWgb3MSjxS/ImzY+tJBhQdTqmbpOHv2vmbDNVEZFyVShNqG+2y5cnp6hwVGbDgXZiHi", + "ltpHC4RRTgOLxocxZVh4rWq2pGR9xpCpNmhfyzHqoN/fWh+5aLfqEbBLL+q7+GVJFNu9NSnESmDLUojZ", + "nMtN1MA0dSON7LUBMegpDl2lop/y3hp5z2rbBUkOvi8yBYO+ETEG2oo4ptXzyIljK3UXgxaQnAuqiIsz", + "NpoIdeJcjrxFnaSqgi7rGPt1tyyAJUYO//Y3gH8wb96zBeZ9sql5cWQ6DboOBg8LHeGwHCJ2/PryC6K+", + "B4zrb4qUutZS94i/DwV/XhArBOZAq1CzbWgfXTTGVItICIJjaUcxL2vF9QLW1L0gTKFn8GvP/tepP5Cf", + "/yHi0w8DZEAY8SmKKCPSxoBl3g7NFC0s4SMTbJN9Z2NvghlmUyJR2/DPf/79H7Aoyqb//Ps/tGht/oLr", + "vm3yiyCF/cOMYKHGBKsPA/QbIUkXR3RO3GYg55TMiVigvb5tmA+PPFWx5ZAN2RuiUsFkllmk9wUwMQPa", + "Jk16P5SlRNpgJWhBOrEpL8Yo6tHn3V02oNzoje4se3vNDgob0FzR4QDEMFNGFcWRVcZafrOa2XPJqFa1", + "7y5Z/NfTF0U+KoO9XbPAGxIYALHv3sEDu2nUvrh4ttVDIO4brIC0JtAb8mGsJtD7SZPW0yRDUcoEBaBs", + "aFOhIUqtdfjEvtPMPGxH/KHtw3UtX+oNxMYgQgQJHQB/Kg9NjMV+uDnDsc96e+JiVuvNt1+/3+IULlSj", + "kWZ8e+fscG8Z5rb9cQ6y+9CJUdt2rsyqEpd6LN8X0m+EjRRaeme8BHFTC3ljetoxZ5OIBgp13Vqg3lFM", + "Mt2tjCAPhRy8satG2O2rWlGgyPC2Swlytawvy5XLeeDdc4/KpDdhI3nVgxzXfnKSdahzQmXA9bcFbOkG", + "OLE1mY08k93TIhats1CdwO8Zy1kpP51kzf7thdycrcpOnbIqb9gAUTypEMR7JISVerGFOiEPCZsvs1N0", + "+TkrTFnfF2r2NycFbdqs5UPzh2TXCitg01RwlnUlrEMv27fwDg/azuDZ+AUR7labhZoyo/m2zKcomJHg", + "ymwI3NWrleFT80ozXdiM90OrwqaD5A1EGHsGP2WWBtpvDqtVGu+pLaB7dwovzHAjfff2PMEWwTxAhtiU", + "sbNpm9q0WC5YsPVDOYM3wt4MsB8kdztPo8j5ROZEqLwfZpEpbH+GKKb1wr67bSv5w+Wbl13CAg5ha1nI", + "lV+qcm3sblfkNwdmtvITTZooiQAqhxj1EvU3nL+JLkRZT5R/331uu6L8++5z0xfl3/eOTGeUrTtDlv6m", + "SPOmRfAHjHxaAqdloAFpMs3m1oms2VsNpVb3/o8tuNrupDcRXTNA/5Rem0ivRXCtFGCzRrF3KMLa/pv3", + "47TJkM0HbXjkQhp/MNF1s3ZAi5GuaBKVZceIrYrLRd7zkjKUSvIAYy5phnFFPtLQoJ1fyJX8xKHu6UnH", + "tjM1TUizBJMNmbfdOjYu7dp5N2/bPorHdJryVBZzV6B7LZE22SkiZQL80OTwnD3XSuLfMZb2N8k6Ni5o", + "/8T7O1IBqgdqiLfxUa1TAtxbTZUA+z70iTWdh0zu2xvX0ciWN9mqiUN0/bqaonGpXdxyfKRvXbXKCbrU", + "6kuuMyBQIwZD9l/uk98VwfH7X1x6U9rv7x5kzwibv//FZTmxM4c3hClBiURYEHT06gS8hFPIjYfijXl+", + "X3U9piQj4KFrXP0vrTnlTtPmqpNDz5+qUyPVqQCu1apT1lnrLnWncnGijStPDt98ALdFPH6qT5tQn2Q6", + "mdCAEqby6uVL8WW2+cEDzFNj1pNUiAspceDG6lPe7m61ZJrXSdx4TFA2+ea1JleS8WHG23OTYRM6PSVn", + "hvWKyveGD/3NEufNKygPGcWMJlAF3TIh2p7Y4ul+AeE5F1dNMc9TuPXWEfD2pZPiDr9D2UQvD8qW3L+I", + "AszbhOVrpClLLhu4kEvVeO8zGtRBwmq9JsGSsmnWpPiaqhlPTbmWkf3R1H/Tt8J2wgKRJ7Cj3jd50bNv", + "QAB9xRWicRKRmEB9uK7BJugOnSYJF1lPSioLtatvRv70tSnG5pqqObY1ewfZCt9gxcu6SoNBf/m4vFQz", + "4tP1CbrZ5C4b1ZOhO2SX0lSP+WBE4Q8oI7JIcSRJRAKFrmc0mEG2rv7NlNqHRFqcJB+y8hxbA/QCbmqx", + "YAhM3pZEUBxB518emabVH+Zx/GGwXGvu3dkZfGQSdU1VuQ8D5OrLZQxC6reK2bd6FxGWCr2yOcVtjUmC", + "R5E50Q+aCxX2t2XzcvNKJkPmy9Fl5NoOSCfoQyFd90NNvq4jqC/1Kd2TvNSpr4Bl9qI4EgA4g5uEhTU2", + "Mg01f6buTt9bMrVh1rBZxh0nDS8t5iWfZtW3SqiMk6Qp+tplAhbP43gFDqN2ofq9VCFP1Z+lCokQ8LHF", + "7jrkRm0cmH8ofKURldk2iq5/AKCf165pKuB4QaWJaqEAtPnXPI5bnZZdj6ej+rdnX1cHXDaz6ZMppFj/", + "lLRvkjxdJvaF7OkK57A9d+pFbttK6IfX9yyg7tumcA/2sXwVlDlRBc6W5wXkH1TSpekyVZXFTLV73x3J", + "2lTV35KyUfkir6v/L6iimr1We4ttWEnNQOzTzEr9UO5dO806bfzUUDMNlQsUpma6SoOkH1btzAgKSllJ", + "87Ti6dfqnlnBugzM0EOVrXQI5DRv+7P78/QrxIXvhBJ2ahvE1JVGyjf9PZDcmn6OjWjuPclJlq0WBIR7", + "JMGus+SmKXAGFa3uZVTuuyDD5sJl1LhIc5TATFLXNPYnMS6ZAY2l9GuJsRM+l2yBBfJMWTeJcB1dtnJq", + "LQGuNF+6V33t9glhTWupL5YS3ifhy5WjjRG704y8GYJn2/H94CpqwIUw8XMQlveQsjoLTtKCrt1OcCpJ", + "J6MQHeeof3d2tlVHJYRaSSOEesAUotJGLg79TewFDV25/+OzE9scgEokUtZDr2MKNfivCEmgniflqUQQ", + "ANkrtnWr6S2f920jTIlFwilTa1eRv3o3i/nyVRXON0wnbV77D29Hs13RHxqRAtqhxRW7gdVapDLdDL1+", + "Seeno8x0CIDOqGOe6tGXWs1B42y5kIrExkk5SU1vUSiFYkvn2u9MsF4HUSWRvg8dCG5KiIgpdI6UQzYm", + "Ey2GJUTouaElJo1Iwd/ic+VdKJxRzXND+r4PXx50nwP3FVZ1UCs3nsNJ4hrP+fxFWa+8r17Sc3DOIbmI", + "xzyiAYoou5KoHdEro3SguUSR/mNrpXdvBN/ddmHgr79ZGtKnbMK9pRINzmbI/CNQuNMKWXPRCw+OrL0g", + "xcvi6A8ctJ+sybV0TRAcQa/VLK4YpYpG9JMhdXoQKhUNTPcpnMEOGueY+XpDdkaU0O9gQVDAo4gEyhlX", + "thPBg+1h2u/vBQmFhJA9AosDglf/OIYZj88v4T3T3KczZPofMPDbo3NENUwn2NoICgtlRF1zcYVOt1+v", + "iXe4ADD9CzsMzQZXXQv/gf90Zd88aLT2DsmaK8qTVQoQT354j7aV4H5aCx6mtQCi9rPdtKcCByAUy1mq", + "Qn7N/JYB03xWbn82f5yuy/1QOJi9c521vw9p1zbiXTeN2+CDuJR2TyExpVzvxUFheyU/0EpXGnBuCyDE", + "FLNY/FzA9GD/0bD79o3yRTh+h65JC1FXJvm7uVub5nx2DS6lsQiPh3LNDaa5nUDrzaL1KcvfXKubBakQ", + "hCkoipOLlgFOcEDVooNw5PrS2t5SmQ0p77E/FgRfaU7bG7I3Weao7W2ltauOU61QSOWVGcFqTz30ek6E", + "TMfZ4hAQJqPnAfBta9oAR4Hp6UomExIoOiem2aqs0b6ypdxlCeN8Es9Bu4cWdA9N5fDjBJxejhZW6yiF", + "BtYWsrjI3mpWyCIbtRD+UwiNWRnkPXIvjoAT3cRk55n8itbmAdhHNwvX+01/1HDucliYfxH20Tfu8oct", + "GHhRCM9pWvYiR/mHVoGisPLS3S2FuK1PhW8c03aXMWbrUuGzyTedCn/hDXN6YJW6cClwrS4H/vtDhP5m", + "46s3nQP/sHFLyxZyCXT1lKhBLvx3gYF3kwR/z/kFX5EE/11FvEIS8/1lHnxXsa42ZjOLdf2Z5n6XIa4m", + "1x1SeutCXA3Vs6bolZrTO/tOM73JjvhDi/TWnHkDgd6dw88qdg10iAKwHFuu0B9gBtLeABInauHsVXwC", + "kTl5yUVJP0F8ny+XMDNL310K31dYbG8PPRye1tprf1a/25hJOK8dfnry8EveFe9cidNsazbUxSKY0Xkp", + "hW3VDbYgSgTpJjwBS2xoAGbh4ZibwqI3/YTs8L0hezsj7l+IugIiJEQhFSRQ0QJRpjhQBDPHnyQSXKsG", + "8JyLhc/AW7y5zwWPj+xu1jBIe6esuSwPBIwXXc21unNHbVYY2b7BqXWGP9I4jYHgIcrQi6eoTT4qYepZ", + "oIlWhRCdZCAlHwNCQgk4uVVc8E6/xvZJP5HRdNxklSsqk7y2lV9QkErFY3f2pyeojVPFu1PC9Flo2X8C", + "om0i+JyGpp5wDtQ5jwxUd2oAelPLrBMukMJTaUPHc7XDrPLeBZsmXGr6iSZlWmGiJVuD1pgyDAtdWxik", + "fNFM4K6eD1MIn8svlEOn1k++Vm1kDtjERQZExTmKtNy/9ZP3PWTeVwyAcIyuxAKbVXttFhPRMFThLiq9", + "ZvEymzVuv/t+3PiFxs8P0MA+z7TUOuP694WC/c3xh00b1d894LCvF8Rp5AWDOgygR/QhzEse4AiFZE4i", + "nsRa1jTvtjqtVEStQWumVDLY3o70ezMu1eCwf9hvfXn/5f8HAAD//wD0kXu+JQEA", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 25cc3536..621afe81 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -24,6 +24,7 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/system" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" @@ -125,7 +126,15 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima meter := otel.GetMeterProvider().Meter("hypeman") tracer := otel.GetTracerProvider().Tracer("hypeman") defaultHypervisor := hypervisor.Type(cfg.Hypervisor.Default) - return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, meter, tracer), nil + level := cfg.Snapshot.CompressionDefault.Level + snapshotDefaults := instances.SnapshotPolicy{ + Compression: &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Snapshot.CompressionDefault.Enabled, + Algorithm: snapshot.SnapshotCompressionAlgorithm(strings.ToLower(cfg.Snapshot.CompressionDefault.Algorithm)), + Level: &level, + }, + } + return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, meter, tracer), nil } // ProvideVolumeManager provides the volume manager diff --git a/lib/snapshot/types.go b/lib/snapshot/types.go index ef2bd8a8..c1e2af13 100644 --- a/lib/snapshot/types.go +++ b/lib/snapshot/types.go @@ -19,15 +19,42 @@ const ( // Snapshot is a centrally stored immutable snapshot resource. type Snapshot struct { - Id string `json:"id"` - Name string `json:"name"` - Kind SnapshotKind `json:"kind"` - Metadata tags.Metadata `json:"metadata,omitempty"` - SourceInstanceID string `json:"source_instance_id"` - SourceName string `json:"source_instance_name"` - SourceHypervisor hypervisor.Type - CreatedAt time.Time `json:"created_at"` - SizeBytes int64 `json:"size_bytes"` + Id string `json:"id"` + Name string `json:"name"` + Kind SnapshotKind `json:"kind"` + Metadata tags.Metadata `json:"metadata,omitempty"` + SourceInstanceID string `json:"source_instance_id"` + SourceName string `json:"source_instance_name"` + SourceHypervisor hypervisor.Type + CreatedAt time.Time `json:"created_at"` + SizeBytes int64 `json:"size_bytes"` + CompressionState string `json:"compression_state,omitempty"` + CompressionError string `json:"compression_error,omitempty"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + CompressedSizeBytes *int64 `json:"compressed_size_bytes,omitempty"` + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes,omitempty"` +} + +// SnapshotCompressionAlgorithm defines supported compression algorithms. +type SnapshotCompressionAlgorithm string + +const ( + SnapshotCompressionAlgorithmZstd SnapshotCompressionAlgorithm = "zstd" + SnapshotCompressionAlgorithmLz4 SnapshotCompressionAlgorithm = "lz4" +) + +const ( + SnapshotCompressionStateNone = "none" + SnapshotCompressionStateCompressing = "compressing" + SnapshotCompressionStateCompressed = "compressed" + SnapshotCompressionStateError = "error" +) + +// SnapshotCompressionConfig defines requested or effective compression config. +type SnapshotCompressionConfig struct { + Enabled bool `json:"enabled"` + Algorithm SnapshotCompressionAlgorithm `json:"algorithm,omitempty"` + Level *int `json:"level,omitempty"` } // ListSnapshotsFilter contains optional filters for listing snapshots. diff --git a/openapi.yaml b/openapi.yaml index ea6a3190..3d710cb2 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -198,6 +198,8 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor + snapshot_policy: + $ref: "#/components/schemas/SnapshotPolicy" skip_kernel_headers: type: boolean description: | @@ -311,6 +313,54 @@ components: format: int64 description: Total payload size in bytes example: 104857600 + compression_state: + type: string + enum: [none, compressing, compressed, error] + description: Compression status of the snapshot payload memory file + example: compressed + compression_error: + type: string + description: Compression error message when compression_state is error + nullable: true + example: "write compressed stream: no space left on device" + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" + compressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Compressed memory payload size in bytes + example: 73400320 + uncompressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Uncompressed memory payload size in bytes + example: 4294967296 + + SnapshotCompressionConfig: + type: object + required: [enabled] + properties: + enabled: + type: boolean + description: Enable snapshot memory compression + example: true + algorithm: + type: string + enum: [zstd, lz4] + description: Compression algorithm (defaults to zstd when enabled) + example: zstd + level: + type: integer + description: Compression level for zstd only + example: 1 + + SnapshotPolicy: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" CreateSnapshotRequest: type: object @@ -326,6 +376,14 @@ components: example: pre-upgrade metadata: $ref: "#/components/schemas/MetadataTags" + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" + + StandbyInstanceRequest: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" RestoreSnapshotRequest: type: object @@ -492,6 +550,8 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor running this instance example: cloud-hypervisor + snapshot_policy: + $ref: "#/components/schemas/SnapshotPolicy" PathInfo: type: object @@ -1625,6 +1685,12 @@ paths: schema: type: string description: Instance ID or name + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/StandbyInstanceRequest" responses: 200: description: Instance in standby @@ -1632,6 +1698,12 @@ paths: application/json: schema: $ref: "#/components/schemas/Instance" + 400: + description: Invalid request payload + content: + application/json: + schema: + $ref: "#/components/schemas/Error" 404: description: Instance not found content: From 47a3197424a84b7aa8a47f60e8ffb995450d1d41 Mon Sep 17 00:00:00 2001 From: Steven Miller Date: Thu, 19 Mar 2026 17:29:37 -0400 Subject: [PATCH 2/2] Add CH snapshot compression restore coverage --- .../compression_integration_linux_test.go | 327 ++++++++++++++++++ lib/instances/restore.go | 3 +- lib/instances/snapshot_compression.go | 95 ++++- lib/instances/snapshot_compression_test.go | 28 ++ lib/instances/standby.go | 6 +- lib/snapshot/README.md | 57 +++ skills/hypeman-remote-linux-tests/SKILL.md | 107 ++++++ .../agents/openai.yaml | 4 + 8 files changed, 618 insertions(+), 9 deletions(-) create mode 100644 lib/instances/compression_integration_linux_test.go create mode 100644 skills/hypeman-remote-linux-tests/SKILL.md create mode 100644 skills/hypeman-remote-linux-tests/agents/openai.yaml diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go new file mode 100644 index 00000000..1cc14546 --- /dev/null +++ b/lib/instances/compression_integration_linux_test.go @@ -0,0 +1,327 @@ +//go:build linux + +package instances + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/resources" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/volumes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type compressionIntegrationHarness struct { + name string + hypervisor hypervisor.Type + setup func(t *testing.T) (*manager, string) + requirePrereqs func(t *testing.T) + waitHypervisorUp func(ctx context.Context, inst *Instance) error + testImplicitLZ4 bool +} + +func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "cloud-hypervisor", + hypervisor: hypervisor.TypeCloudHypervisor, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeCloudHypervisor) + }, + requirePrereqs: requireKVMAccess, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForVMReady(ctx, inst.SocketPath, 10*time.Second) + }, + testImplicitLZ4: true, + }) +} + +func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "firecracker", + hypervisor: hypervisor.TypeFirecracker, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeFirecracker) + }, + requirePrereqs: requireFirecrackerIntegrationPrereqs, + }) +} + +func TestQEMUStandbyRestoreCompressionScenarios(t *testing.T) { + t.Parallel() + + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "qemu", + hypervisor: hypervisor.TypeQEMU, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeQEMU) + }, + requirePrereqs: requireQEMUUsable, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForQEMUReady(ctx, inst.SocketPath, 10*time.Second) + }, + }) +} + +func setupCompressionTestManagerForHypervisor(t *testing.T, hvType hypervisor.Type) (*manager, string) { + t.Helper() + + tmpDir, err := os.MkdirTemp("", "hmcmp-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tmpDir) + }) + prepareIntegrationTestDataDir(t, tmpDir) + + cfg := &config.Config{ + DataDir: tmpDir, + Network: legacyParallelTestNetworkConfig(testNetworkSeq.Add(1)), + } + + p := paths.New(tmpDir) + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + systemManager := system.NewManager(p) + networkManager := network.NewManager(p, cfg, nil) + deviceManager := devices.NewManager(p) + volumeManager := volumes.NewManager(p, 0, nil) + limits := ResourceLimits{ + MaxOverlaySize: 100 * 1024 * 1024 * 1024, + MaxVcpusPerInstance: 0, + MaxMemoryPerInstance: 0, + } + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hvType, SnapshotPolicy{}, nil, nil).(*manager) + + resourceMgr := resources.NewManager(cfg, p) + resourceMgr.SetInstanceLister(mgr) + resourceMgr.SetImageLister(imageManager) + resourceMgr.SetVolumeLister(volumeManager) + require.NoError(t, resourceMgr.Initialize(context.Background())) + mgr.SetResourceValidator(resourceMgr) + + t.Cleanup(func() { + cleanupOrphanedProcesses(t, mgr) + }) + + return mgr, tmpDir +} + +func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionIntegrationHarness) { + t.Helper() + harness.requirePrereqs(t) + + mgr, tmpDir := harness.setup(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + createNginxImageAndWait(t, ctx, imageManager) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + + inst, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: fmt.Sprintf("compression-%s", harness.name), + Image: integrationTestImageRef(t, "docker.io/library/nginx:alpine"), + Size: 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Hypervisor: harness.hypervisor, + }) + require.NoError(t, err) + + deleted := false + t.Cleanup(func() { + if !deleted { + _ = mgr.DeleteInstance(context.Background(), inst.Id) + } + }) + + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, harness.waitHypervisorUp) + + inFlightCompression := &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + } + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "in-flight-zstd-19", inFlightCompression, false) + + completedCases := []struct { + name string + cfg *snapshotstore.SnapshotCompressionConfig + }{ + { + name: "zstd-1", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + }, + { + name: "zstd-19", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + }, + }, + { + name: "lz4", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + } + + for _, tc := range completedCases { + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, tc.name, tc.cfg, true) + } + + if harness.testImplicitLZ4 { + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "implicit-default-lz4", nil, true) + } + + require.NoError(t, mgr.DeleteInstance(ctx, inst.Id)) + deleted = true +} + +func runCompressionCycle( + t *testing.T, + ctx context.Context, + mgr *manager, + p *paths.Paths, + inst *Instance, + waitHypervisorUp func(ctx context.Context, inst *Instance) error, + label string, + cfg *snapshotstore.SnapshotCompressionConfig, + waitForCompression bool, +) *Instance { + t.Helper() + + markerPath := fmt.Sprintf("/tmp/%s.txt", label) + markerValue := fmt.Sprintf("%s-%d", label, time.Now().UnixNano()) + writeGuestMarker(t, ctx, inst, markerPath, markerValue) + + inst, err := mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{ + Compression: cloneCompressionConfig(cfg), + }) + require.NoError(t, err) + require.Equal(t, StateStandby, inst.State) + require.True(t, inst.HasSnapshot) + + snapshotDir := p.InstanceSnapshotLatest(inst.Id) + jobKey := mgr.snapshotJobKeyForInstance(inst.Id) + + if waitForCompression { + waitForCompressionJobCompletion(t, mgr, jobKey, 3*time.Minute) + requireCompressedSnapshotFile(t, snapshotDir, effectiveCompressionForCycle(inst.HypervisorType, cfg)) + } else { + waitForCompressionJobStart(t, mgr, jobKey, 15*time.Second) + } + + inst, err = mgr.RestoreInstance(ctx, inst.Id) + require.NoError(t, err) + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, waitHypervisorUp) + assertGuestMarker(t, ctx, inst, markerPath, markerValue) + + waitForCompressionJobCompletion(t, mgr, jobKey, 30*time.Second) + return inst +} + +func effectiveCompressionForCycle(hvType hypervisor.Type, cfg *snapshotstore.SnapshotCompressionConfig) snapshotstore.SnapshotCompressionConfig { + if cfg != nil { + normalized, err := normalizeCompressionConfig(cfg) + if err != nil { + panic(err) + } + return normalized + } + if hvType == hypervisor.TypeCloudHypervisor { + return defaultCloudHypervisorStandbyCompressionPolicy() + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false} +} + +func waitForRunningAndExecReady(t *testing.T, ctx context.Context, mgr *manager, instanceID string, waitHypervisorUp func(context.Context, *Instance) error) *Instance { + t.Helper() + + inst, err := waitForInstanceState(ctx, mgr, instanceID, StateRunning, 30*time.Second) + require.NoError(t, err) + if waitHypervisorUp != nil { + require.NoError(t, waitHypervisorUp(ctx, inst)) + } + require.NoError(t, waitForExecAgent(ctx, mgr, instanceID, 30*time.Second)) + return inst +} + +func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) +} + +func assertGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, expected string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "cat", path) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) + assert.Equal(t, expected, output) +} + +func waitForCompressionJobStart(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + mgr.compressionMu.Lock() + job := mgr.compressionJobs[key] + mgr.compressionMu.Unlock() + if job != nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("compression job %q did not start within %v", key, timeout) +} + +func waitForCompressionJobCompletion(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + require.NoError(t, mgr.waitCompressionJobContext(ctx, key)) +} + +func requireCompressedSnapshotFile(t *testing.T, snapshotDir string, cfg snapshotstore.SnapshotCompressionConfig) { + t.Helper() + require.True(t, cfg.Enabled) + + rawPath, rawExists := findRawSnapshotMemoryFile(snapshotDir) + assert.False(t, rawExists, "raw snapshot memory should be removed after compression, found %q", rawPath) + + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + require.True(t, ok, "compressed snapshot memory should exist in %s", snapshotDir) + assert.Equal(t, cfg.Algorithm, algorithm) + _, err := os.Stat(compressedPath) + require.NoError(t, err) +} diff --git a/lib/instances/restore.go b/lib/instances/restore.go index fc57ce03..6020c167 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,7 +69,8 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) - if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id)); err != nil { + waitForCompression := stored.HypervisorType == hypervisor.TypeCloudHypervisor + if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id), waitForCompression); err != nil { return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) } starter, err := m.getVMStarter(stored.HypervisorType) diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go index 8974ede5..4cab65f2 100644 --- a/lib/instances/snapshot_compression.go +++ b/lib/instances/snapshot_compression.go @@ -10,6 +10,7 @@ import ( "strings" "time" + "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/klauspost/compress/zstd" @@ -103,6 +104,55 @@ func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, overr return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil } +func defaultCloudHypervisorStandbyCompressionPolicy() snapshotstore.SnapshotCompressionConfig { + return snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + } +} + +func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { + if override != nil { + cfg, err := normalizeCompressionConfig(override) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { + cfg, err := normalizeCompressionConfig(stored.SnapshotPolicy.Compression) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if m.snapshotDefaults.Compression != nil { + cfg, err := normalizeCompressionConfig(m.snapshotDefaults.Compression) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if stored != nil && stored.HypervisorType == hypervisor.TypeCloudHypervisor { + cfg := defaultCloudHypervisorStandbyCompressionPolicy() + return &cfg, nil + } + + return nil, nil +} + func (m *manager) snapshotJobKeyForInstance(instanceID string) string { return "instance:" + instanceID } @@ -192,10 +242,32 @@ func (m *manager) waitCompressionJob(key string, timeout time.Duration) { } } -func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string) error { +func (m *manager) waitCompressionJobContext(ctx context.Context, key string) error { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job == nil { + return nil + } + + select { + case <-job.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, waitForCompression bool) error { if jobKey != "" { - m.cancelCompressionJob(jobKey) - m.waitCompressionJob(jobKey, 2*time.Second) + if waitForCompression { + if err := m.waitCompressionJobContext(ctx, jobKey); err != nil { + return err + } + } else { + m.cancelCompressionJob(jobKey) + m.waitCompressionJob(jobKey, 2*time.Second) + } } if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { @@ -269,8 +341,8 @@ func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapsho compressedPath := compressedPathFor(rawPath, cfg.Algorithm) tmpPath := compressedPath + ".tmp" + removeCompressedSnapshotArtifacts(rawPath) _ = os.Remove(tmpPath) - _ = os.Remove(compressedPath) if err := runCompression(ctx, rawPath, tmpPath, cfg); err != nil { _ = os.Remove(tmpPath) @@ -323,6 +395,9 @@ func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotst } case snapshotstore.SnapshotCompressionAlgorithmLz4: enc := lz4.NewWriter(dst) + if err := enc.Apply(lz4.CompressionLevelOption(lz4.Fast)); err != nil { + return fmt.Errorf("configure lz4 encoder: %w", err) + } if err := copyWithContext(ctx, enc, src); err != nil { _ = enc.Close() return err @@ -376,6 +451,7 @@ func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, al _ = os.Remove(tmpRawPath) return fmt.Errorf("finalize decompressed snapshot: %w", err) } + removeCompressedSnapshotArtifacts(rawPath) return nil } @@ -388,6 +464,17 @@ func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressi } } +func removeCompressedSnapshotArtifacts(rawPath string) { + for _, path := range []string{ + rawPath + ".zst", + rawPath + ".zst.tmp", + rawPath + ".lz4", + rawPath + ".lz4.tmp", + } { + _ = os.Remove(path) + } +} + func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { buf := make([]byte, 1024*1024) for { diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go index dd043f3f..bf599481 100644 --- a/lib/instances/snapshot_compression_test.go +++ b/lib/instances/snapshot_compression_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + "github.com/kernel/hypeman/lib/hypervisor" snapshotstore "github.com/kernel/hypeman/lib/snapshot" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -86,6 +87,33 @@ func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { assert.Equal(t, 2, *cfg.Level) } +func TestResolveStandbyCompressionPolicyCloudHypervisorDefault(t *testing.T) { + t.Parallel() + + m := &manager{} + + cfg, err := m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeCloudHypervisor, + }, nil) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeCloudHypervisor, + }, &snapshotstore.SnapshotCompressionConfig{Enabled: false}) + require.NoError(t, err) + assert.Nil(t, cfg) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeQEMU, + }, nil) + require.NoError(t, err) + assert.Nil(t, cfg) +} + func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { t.Parallel() diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 9b9a3b96..d218a9ce 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -62,13 +62,11 @@ func (m *manager) standbyInstance( // fails before any state transition side effects. var compressionPolicy *snapshotstore.SnapshotCompressionConfig if !skipCompression { - policy, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + policy, err := m.resolveStandbyCompressionPolicy(stored, req.Compression) if err != nil { return nil, err } - if policy.Enabled { - compressionPolicy = &policy - } + compressionPolicy = policy } // 3. Get network allocation BEFORE killing VMM (while we can still query it) diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index edcdc989..217d245f 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -31,6 +31,63 @@ Snapshots are immutable point-in-time captures of a VM that can later be: - `Stopped` snapshot from `Stopped`: - snapshot payload is copied directly +## Snapshot Compression + +Snapshot memory compression is optional and is **off by default**. + +The current exception is cloud-hypervisor standby snapshots: if no request, instance, +or server policy is set, Hypeman defaults that path to `lz4` so standby snapshots +store compressed guest memory by default on Linux. + +- Compression applies only to `Standby` snapshots, because only standby snapshots contain guest memory state. +- `Stopped` snapshots cannot use compression because they do not include resumable RAM state. +- Compression affects only the memory snapshot file, not the entire snapshot directory. + +### When compression runs + +- A standby operation can request compression explicitly. +- A snapshot create request can request compression explicitly. +- If the request does not specify compression, Hypeman falls back to: + - the instance's `snapshot_policy` + - then the server's global `snapshot.compression_default` +- Effective precedence is: + - request override + - instance default + - server default + - then the cloud-hypervisor standby fallback (`lz4`) when no other policy is set + +Compression runs **asynchronously after the snapshot is already durable on disk**. + +- This keeps the standby path fast. +- While compression is running, the snapshot remains valid and reports `compression_state=compressing`. +- Once finished, the snapshot reports `compression_state=compressed` and exposes compressed/uncompressed size metadata. +- If compression fails, the snapshot reports `compression_state=error` and keeps the error message for inspection. + +### Restore behavior with compressed snapshots + +Restore prefers correctness first and resume latency second. + +- If a restore starts while compression is still running, Hypeman will not let restore race the compressor. +- For cloud-hypervisor standby restores, Hypeman waits for the in-flight compression job to finish. +- For the other hypervisors, Hypeman cancels the in-flight compression job and waits briefly for it to stop. +- If the snapshot memory has already been compressed, Hypeman expands it back to the raw memory file before asking the hypervisor to restore. +- This means compression is never allowed to race with restore. + +In practice, the tradeoff is: + +- standby stays fast because compression is not on the hot path +- restore from a compressed standby snapshot pays decompression cost +- restore from an uncompressed standby snapshot avoids that cost entirely + +### Supported algorithms + +- `zstd` + - default when compression is enabled + - supports configurable levels +- `lz4` + - optimized for lower decompression overhead + - does not currently accept a level setting + ### Restore (in-place) - Restore always applies to the original source VM. - Source VM must not be `Running`. diff --git a/skills/hypeman-remote-linux-tests/SKILL.md b/skills/hypeman-remote-linux-tests/SKILL.md new file mode 100644 index 00000000..cb2d7c5a --- /dev/null +++ b/skills/hypeman-remote-linux-tests/SKILL.md @@ -0,0 +1,107 @@ +--- +name: hypeman-remote-linux-tests +description: Run Hypeman tests on a remote Linux host that supports the Linux hypervisors. Use when validating cloud-hypervisor, Firecracker, QEMU, embedded guest artifacts, or Linux-only integration behavior on a server over SSH. +--- + +# Hypeman Remote Linux Tests + +Use this skill to run Hypeman tests on a remote Linux server in a way that is repeatable and low-friction for any developer. + +## Quick Start + +1. Pick a short remote working directory under the remote user's home directory. +2. Ensure the remote test shell includes `/usr/sbin` and `/sbin` on `PATH`. +3. Bootstrap embedded artifacts and bundled binaries before running `go test`. +4. Run the smallest relevant test target first. +5. Treat host-environment failures separately from product failures. + +## Remote Workspace + +Prefer a short path such as `~/hm`, `~/hypeman-test`, or another short directory name. + +Cloud Hypervisor uses UNIX sockets with strict path-length limits. Long paths from default test temp directories or deeply nested clones can make the VMM fail before the actual test starts. + +If a new test creates temporary directories for guest data, prefer a short root like: + +```bash +mktemp -d /tmp/hmcmp-XXXXXX +``` + +## Remote Bootstrap + +Before running Linux hypervisor tests, make sure the remote clone has the generated guest binaries and embedded VMM assets: + +```bash +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH +make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded +``` + +Why: + +- `go:embed` patterns in the repo require the bundled binaries to exist at build time. +- `mkfs.ext4` is commonly installed in `/usr/sbin` or `/sbin`, not always on the default non-login shell `PATH`. +- Running `go test` before this bootstrap often fails during package setup, not during test execution. + +QEMU is usually provided by the host OS, so there is no matching bundled `ensure-qemu-binaries` target. If QEMU tests fail before boot, verify the system `qemu-system-*` binary separately. + +## Test Execution Pattern + +Start with one focused test: + +```bash +go test ./lib/instances -run TestCloudHypervisorStandbyRestoreCompressionScenarios -count=1 -v +``` + +Then run the sibling hypervisor tests one at a time: + +```bash +go test ./lib/instances -run TestFirecrackerStandbyRestoreCompressionScenarios -count=1 -v +go test ./lib/instances -run TestQEMUStandbyRestoreCompressionScenarios -count=1 -v +``` + +For unit-only validation: + +```bash +go test ./lib/instances -run 'TestNormalizeCompressionConfig|TestResolveSnapshotCompressionPolicyPrecedence' -count=1 +``` + +Run one hypervisor at a time when the tests boot real VMs. This avoids noisy failures from competing KVM guests and makes logs much easier to interpret. + +## Syncing Local Changes + +If the main development is happening locally and the Linux host is only for execution, sync only the files you changed rather than recloning every time. + +Typical pattern: + +```bash +rsync -az lib/instances/... remote-host:~/hypeman-test/lib/instances/ +``` + +After syncing, re-run the bootstrap command if the changes affect embedded binaries, generated assets, or files referenced by `go:embed`. + +## Common Host Failures + +Classify failures quickly: + +- Missing embedded assets: + - symptom: package setup fails with `pattern ... no matching files found` + - fix: run `make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded` +- `mkfs.ext4` not found: + - symptom: overlay or disk creation fails with `exec: "mkfs.ext4": executable file not found` + - fix: add `/usr/sbin:/sbin` to `PATH`, then retry +- Cloud Hypervisor socket path too long: + - symptom: `path must be shorter than SUN_LEN` + - fix: shorten the remote repo path and any temp/data directories used by the test +- `/tmp` or shared test-lock issues: + - symptom: test-specific lock or temp-file permission failures + - fix: prefer no-network test setup when the test does not require networking; avoid unnecessary shared lock files + +## Reporting Guidance + +When reporting results, separate: + +1. Code failures +2. Test harness issues +3. Remote host environment issues + +This matters in Hypeman because Linux integration tests often fail before product code executes if the remote machine is missing bundled binaries, system tools, or short enough paths. diff --git a/skills/hypeman-remote-linux-tests/agents/openai.yaml b/skills/hypeman-remote-linux-tests/agents/openai.yaml new file mode 100644 index 00000000..244c1d28 --- /dev/null +++ b/skills/hypeman-remote-linux-tests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Hypeman Remote Linux Tests" + short_description: "Run Hypeman tests on a remote Linux hypervisor host" + default_prompt: "Run Hypeman tests on a remote Linux host, prepare prerequisites, and report actionable failures."