diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index a6c2edcf..b24055c1 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -43,7 +43,7 @@ func newTestService(t *testing.T) *ApiService { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Initialize network manager (creates bridge for network-enabled tests) if err := networkMgr.Initialize(ctx(), nil); err != nil { diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index ba5b448a..3595b760 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -17,6 +17,7 @@ import ( "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/samber/lo" ) @@ -301,6 +302,16 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst SkipKernelHeaders: request.Body.SkipKernelHeaders != nil && *request.Body.SkipKernelHeaders, SkipGuestAgent: request.Body.SkipGuestAgent != nil && *request.Body.SkipGuestAgent, } + if request.Body.SnapshotPolicy != nil { + snapshotPolicy, err := toInstanceSnapshotPolicy(*request.Body.SnapshotPolicy) + if err != nil { + return oapi.CreateInstance400JSONResponse{ + Code: "invalid_snapshot_policy", + Message: err.Error(), + }, nil + } + domainReq.SnapshotPolicy = snapshotPolicy + } inst, err := s.InstanceManager.CreateInstance(ctx, domainReq) if err != nil { @@ -438,7 +449,19 @@ func (s *ApiService) StandbyInstance(ctx context.Context, request oapi.StandbyIn } log := logger.FromContext(ctx) - result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id) + standbyReq := instances.StandbyInstanceRequest{} + if request.Body != nil && request.Body.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.StandbyInstance400JSONResponse{ + Code: "invalid_snapshot_compression", + Message: err.Error(), + }, nil + } + standbyReq.Compression = compression + } + + result, err := s.InstanceManager.StandbyInstance(ctx, inst.Id, standbyReq) if err != nil { switch { case errors.Is(err, instances.ErrInvalidState): @@ -895,6 +918,10 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { if len(inst.Tags) > 0 { oapiInst.Tags = toOAPITags(inst.Tags) } + if inst.SnapshotPolicy != nil { + oapiPolicy, _ := toOAPISnapshotPolicy(*inst.SnapshotPolicy) + oapiInst.SnapshotPolicy = &oapiPolicy + } // Convert volume attachments if len(inst.Volumes) > 0 { @@ -929,3 +956,56 @@ func instanceToOAPI(inst instances.Instance) oapi.Instance { return oapiInst } + +func toDomainSnapshotCompressionConfig(cfg oapi.SnapshotCompressionConfig) (*snapshot.SnapshotCompressionConfig, error) { + out := &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != nil { + out.Algorithm = snapshot.SnapshotCompressionAlgorithm(*cfg.Algorithm) + } + if cfg.Level != nil { + level := *cfg.Level + out.Level = &level + } + return out, nil +} + +func toInstanceSnapshotPolicy(policy oapi.SnapshotPolicy) (*instances.SnapshotPolicy, error) { + out := &instances.SnapshotPolicy{} + if policy.Compression != nil { + compression, err := toDomainSnapshotCompressionConfig(*policy.Compression) + if err != nil { + return nil, err + } + out.Compression = compression + } + return out, nil +} + +func toOAPISnapshotCompressionConfig(cfg snapshot.SnapshotCompressionConfig) (oapi.SnapshotCompressionConfig, error) { + out := oapi.SnapshotCompressionConfig{ + Enabled: cfg.Enabled, + } + if cfg.Algorithm != "" { + algo := oapi.SnapshotCompressionConfigAlgorithm(cfg.Algorithm) + out.Algorithm = &algo + } + if cfg.Level != nil { + level := *cfg.Level + out.Level = &level + } + return out, nil +} + +func toOAPISnapshotPolicy(policy instances.SnapshotPolicy) (oapi.SnapshotPolicy, error) { + out := oapi.SnapshotPolicy{} + if policy.Compression != nil { + compression, err := toOAPISnapshotCompressionConfig(*policy.Compression) + if err != nil { + return oapi.SnapshotPolicy{}, err + } + out.Compression = &compression + } + return out, nil +} diff --git a/cmd/api/api/snapshots.go b/cmd/api/api/snapshots.go index 2609c8be..d963bc40 100644 --- a/cmd/api/api/snapshots.go +++ b/cmd/api/api/snapshots.go @@ -10,6 +10,7 @@ import ( mw "github.com/kernel/hypeman/lib/middleware" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/oapi" + "github.com/kernel/hypeman/lib/snapshot" "github.com/samber/lo" ) @@ -27,11 +28,20 @@ func (s *ApiService) CreateInstanceSnapshot(ctx context.Context, request oapi.Cr if request.Body.Name != nil { name = *request.Body.Name } + var compression *snapshot.SnapshotCompressionConfig + if request.Body.Compression != nil { + var err error + compression, err = toDomainSnapshotCompressionConfig(*request.Body.Compression) + if err != nil { + return oapi.CreateInstanceSnapshot400JSONResponse{Code: "invalid_snapshot_compression", Message: err.Error()}, nil + } + } result, err := s.InstanceManager.CreateSnapshot(ctx, inst.Id, instances.CreateSnapshotRequest{ - Kind: instances.SnapshotKind(request.Body.Kind), - Name: name, - Tags: toMapTags(request.Body.Tags), + Kind: instances.SnapshotKind(request.Body.Kind), + Name: name, + Tags: toMapTags(request.Body.Tags), + Compression: compression, }) if err != nil { log := logger.FromContext(ctx) @@ -207,6 +217,23 @@ func snapshotToOAPI(snapshot instances.Snapshot) oapi.Snapshot { SizeBytes: snapshot.SizeBytes, Name: lo.ToPtr(snapshot.Name), } + if snapshot.CompressionState != "" { + state := oapi.SnapshotCompressionState(snapshot.CompressionState) + out.CompressionState = &state + } + if snapshot.CompressionError != "" { + out.CompressionError = lo.ToPtr(snapshot.CompressionError) + } + if snapshot.Compression != nil { + compression, _ := toOAPISnapshotCompressionConfig(*snapshot.Compression) + out.Compression = &compression + } + if snapshot.CompressedSizeBytes != nil { + out.CompressedSizeBytes = snapshot.CompressedSizeBytes + } + if snapshot.UncompressedSizeBytes != nil { + out.UncompressedSizeBytes = snapshot.UncompressedSizeBytes + } if snapshot.Name == "" { out.Name = nil } diff --git a/cmd/api/config/config.go b/cmd/api/config/config.go index 086df055..e9704693 100644 --- a/cmd/api/config/config.go +++ b/cmd/api/config/config.go @@ -176,6 +176,18 @@ type HypervisorMemoryConfig struct { VZBalloonRequired bool `koanf:"vz_balloon_required"` } +// SnapshotCompressionDefaultConfig holds default snapshot compression settings. +type SnapshotCompressionDefaultConfig struct { + Enabled bool `koanf:"enabled"` + Algorithm string `koanf:"algorithm"` + Level int `koanf:"level"` +} + +// SnapshotConfig holds snapshot defaults. +type SnapshotConfig struct { + CompressionDefault SnapshotCompressionDefaultConfig `koanf:"compression_default"` +} + // GPUConfig holds GPU-related settings. type GPUConfig struct { ProfileCacheTTL string `koanf:"profile_cache_ttl"` @@ -202,6 +214,7 @@ type Config struct { Oversubscription OversubscriptionConfig `koanf:"oversubscription"` Capacity CapacityConfig `koanf:"capacity"` Hypervisor HypervisorConfig `koanf:"hypervisor"` + Snapshot SnapshotConfig `koanf:"snapshot"` GPU GPUConfig `koanf:"gpu"` } @@ -334,6 +347,14 @@ func defaultConfig() *Config { }, }, + Snapshot: SnapshotConfig{ + CompressionDefault: SnapshotCompressionDefaultConfig{ + Enabled: false, + Algorithm: "zstd", + Level: 1, + }, + }, + GPU: GPUConfig{ ProfileCacheTTL: "30m", }, @@ -446,6 +467,14 @@ func (c *Config) Validate() error { if c.Build.Timeout <= 0 { return fmt.Errorf("build.timeout must be positive, got %d", c.Build.Timeout) } + if c.Snapshot.CompressionDefault.Level < 1 { + return fmt.Errorf("snapshot.compression_default.level must be >= 1, got %d", c.Snapshot.CompressionDefault.Level) + } + switch strings.ToLower(c.Snapshot.CompressionDefault.Algorithm) { + case "", "zstd", "lz4": + default: + return fmt.Errorf("snapshot.compression_default.algorithm must be one of zstd or lz4, got %q", c.Snapshot.CompressionDefault.Algorithm) + } if c.Hypervisor.Memory.KernelPageInitMode != "performance" && c.Hypervisor.Memory.KernelPageInitMode != "hardened" { return fmt.Errorf("hypervisor.memory.kernel_page_init_mode must be one of {performance,hardened}, got %q", c.Hypervisor.Memory.KernelPageInitMode) } diff --git a/integration/systemd_test.go b/integration/systemd_test.go index ad60510d..4c552db0 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -68,7 +68,7 @@ func TestSystemdMode(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Cleanup any orphaned instances t.Cleanup(func() { diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index cbf59d09..12861a6a 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -77,7 +77,7 @@ func TestVGPU(t *testing.T) { MaxMemoryPerInstance: 0, } - instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil) + instanceManager := instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", instances.SnapshotPolicy{}, nil, nil) // Track instance ID for cleanup var instanceID string diff --git a/lib/builds/manager_test.go b/lib/builds/manager_test.go index a3765d5f..1a68ded3 100644 --- a/lib/builds/manager_test.go +++ b/lib/builds/manager_test.go @@ -108,7 +108,7 @@ func (m *mockInstanceManager) ForkSnapshot(ctx context.Context, snapshotID strin return nil, instances.ErrNotFound } -func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string) (*instances.Instance, error) { +func (m *mockInstanceManager) StandbyInstance(ctx context.Context, id string, req instances.StandbyInstanceRequest) (*instances.Instance, error) { return nil, nil } diff --git a/lib/devices/gpu_e2e_test.go b/lib/devices/gpu_e2e_test.go index 86708446..a7fe6390 100644 --- a/lib/devices/gpu_e2e_test.go +++ b/lib/devices/gpu_e2e_test.go @@ -79,7 +79,7 @@ func TestGPUPassthrough(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, // 100GB } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Discover available GPUs t.Log("Step 1: Discovering available GPUs...") diff --git a/lib/devices/gpu_inference_test.go b/lib/devices/gpu_inference_test.go index c99d1b39..f521158e 100644 --- a/lib/devices/gpu_inference_test.go +++ b/lib/devices/gpu_inference_test.go @@ -116,7 +116,7 @@ func TestGPUInference(t *testing.T) { limits := instances.ResourceLimits{ MaxOverlaySize: 100 * 1024 * 1024 * 1024, } - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Build custom CUDA+Ollama image t.Log("Step 1: Building custom CUDA+Ollama Docker image...") diff --git a/lib/devices/gpu_module_test.go b/lib/devices/gpu_module_test.go index cd9f1b76..f39d4690 100644 --- a/lib/devices/gpu_module_test.go +++ b/lib/devices/gpu_module_test.go @@ -80,7 +80,7 @@ func TestNVIDIAModuleLoading(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Find an NVIDIA GPU t.Log("Step 1: Discovering available GPUs...") @@ -326,7 +326,7 @@ func TestNVMLDetection(t *testing.T) { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 10*1024*1024*1024, nil) limits := instances.ResourceLimits{MaxOverlaySize: 10 * 1024 * 1024 * 1024} - instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil) + instanceMgr := instances.NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", instances.SnapshotPolicy{}, nil, nil) // Step 1: Check if ollama-cuda:test image exists in Docker t.Log("Step 1: Checking for ollama-cuda:test Docker image...") diff --git a/lib/forkvm/copy.go b/lib/forkvm/copy.go index 2076c0ce..931648c3 100644 --- a/lib/forkvm/copy.go +++ b/lib/forkvm/copy.go @@ -6,6 +6,7 @@ import ( "io/fs" "os" "path/filepath" + "strings" ) var ErrSparseCopyUnsupported = errors.New("sparse copy unsupported") @@ -41,6 +42,9 @@ func CopyGuestDirectory(srcDir, dstDir string) error { if d.IsDir() && shouldSkipDirectory(relPath) { return filepath.SkipDir } + if shouldSkipRegularFile(relPath) { + return nil + } dstPath := filepath.Join(dstDir, relPath) info, err := d.Info() @@ -85,3 +89,7 @@ func CopyGuestDirectory(srcDir, dstDir string) error { func shouldSkipDirectory(relPath string) bool { return relPath == "logs" } + +func shouldSkipRegularFile(relPath string) bool { + return strings.HasSuffix(relPath, ".lz4.tmp") || strings.HasSuffix(relPath, ".zst.tmp") +} diff --git a/lib/forkvm/copy_test.go b/lib/forkvm/copy_test.go index 762499e4..67c1bbe9 100644 --- a/lib/forkvm/copy_test.go +++ b/lib/forkvm/copy_test.go @@ -20,6 +20,8 @@ func TestCopyGuestDirectory(t *testing.T) { require.NoError(t, os.WriteFile(filepath.Join(src, "overlay.raw"), []byte("overlay"), 0644)) require.NoError(t, os.WriteFile(filepath.Join(src, "logs", "app.log"), []byte("hello"), 0644)) require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "config.json"), []byte(`{}`), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp"), []byte("partial"), 0644)) + require.NoError(t, os.WriteFile(filepath.Join(src, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp"), []byte("partial"), 0644)) require.NoError(t, os.Symlink("metadata.json", filepath.Join(src, "meta-link"))) require.NoError(t, CopyGuestDirectory(src, dst)) @@ -28,6 +30,8 @@ func TestCopyGuestDirectory(t *testing.T) { assert.FileExists(t, filepath.Join(dst, "config.ext4")) assert.FileExists(t, filepath.Join(dst, "overlay.raw")) assert.FileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "config.json")) + assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.lz4.tmp")) + assert.NoFileExists(t, filepath.Join(dst, "snapshots", "snapshot-latest", "memory-ranges.zst.tmp")) assert.NoFileExists(t, filepath.Join(dst, "logs", "app.log")) assert.FileExists(t, filepath.Join(dst, "meta-link")) diff --git a/lib/instances/compression_integration_linux_test.go b/lib/instances/compression_integration_linux_test.go new file mode 100644 index 00000000..205a0ade --- /dev/null +++ b/lib/instances/compression_integration_linux_test.go @@ -0,0 +1,321 @@ +//go:build linux + +package instances + +import ( + "context" + "fmt" + "os" + "testing" + "time" + + "github.com/kernel/hypeman/cmd/api/config" + "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/images" + "github.com/kernel/hypeman/lib/network" + "github.com/kernel/hypeman/lib/paths" + "github.com/kernel/hypeman/lib/resources" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/kernel/hypeman/lib/system" + "github.com/kernel/hypeman/lib/volumes" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type compressionIntegrationHarness struct { + name string + hypervisor hypervisor.Type + setup func(t *testing.T) (*manager, string) + requirePrereqs func(t *testing.T) + waitHypervisorUp func(ctx context.Context, inst *Instance) error + testImplicitLZ4 bool +} + +func TestCloudHypervisorStandbyRestoreCompressionScenarios(t *testing.T) { + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "cloud-hypervisor", + hypervisor: hypervisor.TypeCloudHypervisor, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeCloudHypervisor) + }, + requirePrereqs: requireKVMAccess, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForVMReady(ctx, inst.SocketPath, 10*time.Second) + }, + testImplicitLZ4: true, + }) +} + +func TestFirecrackerStandbyRestoreCompressionScenarios(t *testing.T) { + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "firecracker", + hypervisor: hypervisor.TypeFirecracker, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeFirecracker) + }, + requirePrereqs: requireFirecrackerIntegrationPrereqs, + }) +} + +func TestQEMUStandbyRestoreCompressionScenarios(t *testing.T) { + runStandbyRestoreCompressionScenarios(t, compressionIntegrationHarness{ + name: "qemu", + hypervisor: hypervisor.TypeQEMU, + setup: func(t *testing.T) (*manager, string) { + return setupCompressionTestManagerForHypervisor(t, hypervisor.TypeQEMU) + }, + requirePrereqs: requireQEMUUsable, + waitHypervisorUp: func(ctx context.Context, inst *Instance) error { + return waitForQEMUReady(ctx, inst.SocketPath, 10*time.Second) + }, + }) +} + +func setupCompressionTestManagerForHypervisor(t *testing.T, hvType hypervisor.Type) (*manager, string) { + t.Helper() + + tmpDir, err := os.MkdirTemp("", "hmcmp-") + require.NoError(t, err) + t.Cleanup(func() { + _ = os.RemoveAll(tmpDir) + }) + prepareIntegrationTestDataDir(t, tmpDir) + + cfg := &config.Config{ + DataDir: tmpDir, + Network: legacyParallelTestNetworkConfig(testNetworkSeq.Add(1)), + } + + p := paths.New(tmpDir) + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + + systemManager := system.NewManager(p) + networkManager := network.NewManager(p, cfg, nil) + deviceManager := devices.NewManager(p) + volumeManager := volumes.NewManager(p, 0, nil) + limits := ResourceLimits{ + MaxOverlaySize: 100 * 1024 * 1024 * 1024, + MaxVcpusPerInstance: 0, + MaxMemoryPerInstance: 0, + } + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hvType, SnapshotPolicy{}, nil, nil).(*manager) + + resourceMgr := resources.NewManager(cfg, p) + resourceMgr.SetInstanceLister(mgr) + resourceMgr.SetImageLister(imageManager) + resourceMgr.SetVolumeLister(volumeManager) + require.NoError(t, resourceMgr.Initialize(context.Background())) + mgr.SetResourceValidator(resourceMgr) + + t.Cleanup(func() { + cleanupOrphanedProcesses(t, mgr) + }) + + return mgr, tmpDir +} + +func runStandbyRestoreCompressionScenarios(t *testing.T, harness compressionIntegrationHarness) { + t.Helper() + harness.requirePrereqs(t) + + mgr, tmpDir := harness.setup(t) + ctx := context.Background() + p := paths.New(tmpDir) + + imageManager, err := images.NewManager(p, 1, nil) + require.NoError(t, err) + createNginxImageAndWait(t, ctx, imageManager) + + systemManager := system.NewManager(p) + require.NoError(t, systemManager.EnsureSystemFiles(ctx)) + + inst, err := mgr.CreateInstance(ctx, CreateInstanceRequest{ + Name: fmt.Sprintf("compression-%s", harness.name), + Image: integrationTestImageRef(t, "docker.io/library/nginx:alpine"), + Size: 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 10 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: false, + Hypervisor: harness.hypervisor, + }) + require.NoError(t, err) + + deleted := false + t.Cleanup(func() { + if !deleted { + _ = mgr.DeleteInstance(context.Background(), inst.Id) + } + }) + + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, harness.waitHypervisorUp) + + inFlightCompression := &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + } + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "in-flight-zstd-19", inFlightCompression, false) + + completedCases := []struct { + name string + cfg *snapshotstore.SnapshotCompressionConfig + }{ + { + name: "zstd-1", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(1), + }, + }, + { + name: "zstd-19", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(19), + }, + }, + { + name: "lz4", + cfg: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + } + + for _, tc := range completedCases { + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, tc.name, tc.cfg, true) + } + + if harness.testImplicitLZ4 { + inst = runCompressionCycle(t, ctx, mgr, p, inst, harness.waitHypervisorUp, "implicit-default-lz4", nil, true) + } + + require.NoError(t, mgr.DeleteInstance(ctx, inst.Id)) + deleted = true +} + +func runCompressionCycle( + t *testing.T, + ctx context.Context, + mgr *manager, + p *paths.Paths, + inst *Instance, + waitHypervisorUp func(ctx context.Context, inst *Instance) error, + label string, + cfg *snapshotstore.SnapshotCompressionConfig, + waitForCompression bool, +) *Instance { + t.Helper() + + markerPath := fmt.Sprintf("/tmp/%s.txt", label) + markerValue := fmt.Sprintf("%s-%d", label, time.Now().UnixNano()) + writeGuestMarker(t, ctx, inst, markerPath, markerValue) + + inst, err := mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{ + Compression: cloneCompressionConfig(cfg), + }) + require.NoError(t, err) + require.Equal(t, StateStandby, inst.State) + require.True(t, inst.HasSnapshot) + + snapshotDir := p.InstanceSnapshotLatest(inst.Id) + jobKey := mgr.snapshotJobKeyForInstance(inst.Id) + + if waitForCompression { + waitForCompressionJobCompletion(t, mgr, jobKey, 3*time.Minute) + requireCompressedSnapshotFile(t, snapshotDir, effectiveCompressionForCycle(inst.HypervisorType, cfg)) + } else { + waitForCompressionJobStart(t, mgr, jobKey, 15*time.Second) + } + + inst, err = mgr.RestoreInstance(ctx, inst.Id) + require.NoError(t, err) + inst = waitForRunningAndExecReady(t, ctx, mgr, inst.Id, waitHypervisorUp) + assertGuestMarker(t, ctx, inst, markerPath, markerValue) + + waitForCompressionJobCompletion(t, mgr, jobKey, 30*time.Second) + return inst +} + +func effectiveCompressionForCycle(hvType hypervisor.Type, cfg *snapshotstore.SnapshotCompressionConfig) snapshotstore.SnapshotCompressionConfig { + if cfg != nil { + normalized, err := normalizeCompressionConfig(cfg) + if err != nil { + panic(err) + } + return normalized + } + if hvType == hypervisor.TypeCloudHypervisor { + return defaultCloudHypervisorStandbyCompressionPolicy() + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false} +} + +func waitForRunningAndExecReady(t *testing.T, ctx context.Context, mgr *manager, instanceID string, waitHypervisorUp func(context.Context, *Instance) error) *Instance { + t.Helper() + + inst, err := waitForInstanceState(ctx, mgr, instanceID, StateRunning, 30*time.Second) + require.NoError(t, err) + if waitHypervisorUp != nil { + require.NoError(t, waitHypervisorUp(ctx, inst)) + } + require.NoError(t, waitForExecAgent(ctx, mgr, instanceID, 30*time.Second)) + return inst +} + +func writeGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, value string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "sh", "-c", fmt.Sprintf("printf %q > %s && sync", value, path)) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) +} + +func assertGuestMarker(t *testing.T, ctx context.Context, inst *Instance, path string, expected string) { + t.Helper() + output, exitCode, err := execCommand(ctx, inst, "cat", path) + require.NoError(t, err) + require.Equal(t, 0, exitCode, output) + assert.Equal(t, expected, output) +} + +func waitForCompressionJobStart(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + mgr.compressionMu.Lock() + job := mgr.compressionJobs[key] + mgr.compressionMu.Unlock() + if job != nil { + return + } + time.Sleep(100 * time.Millisecond) + } + t.Fatalf("compression job %q did not start within %v", key, timeout) +} + +func waitForCompressionJobCompletion(t *testing.T, mgr *manager, key string, timeout time.Duration) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + require.NoError(t, mgr.waitCompressionJobContext(ctx, key)) +} + +func requireCompressedSnapshotFile(t *testing.T, snapshotDir string, cfg snapshotstore.SnapshotCompressionConfig) { + t.Helper() + require.True(t, cfg.Enabled) + + rawPath, rawExists := findRawSnapshotMemoryFile(snapshotDir) + assert.False(t, rawExists, "raw snapshot memory should be removed after compression, found %q", rawPath) + + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + require.True(t, ok, "compressed snapshot memory should exist in %s", snapshotDir) + assert.Equal(t, cfg.Algorithm, algorithm) + _, err := os.Stat(compressedPath) + require.NoError(t, err) +} diff --git a/lib/instances/create.go b/lib/instances/create.go index d423dc0e..800ee437 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -341,6 +341,7 @@ func (m *manager) createInstance( Cmd: req.Cmd, SkipKernelHeaders: req.SkipKernelHeaders, SkipGuestAgent: req.SkipGuestAgent, + SnapshotPolicy: cloneSnapshotPolicy(req.SnapshotPolicy), } // 12. Ensure directories @@ -540,6 +541,11 @@ func validateCreateRequest(req *CreateInstanceRequest) error { if err := tags.Validate(req.Tags); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } + if req.SnapshotPolicy != nil && req.SnapshotPolicy.Compression != nil { + if _, err := normalizeCompressionConfig(req.SnapshotPolicy.Compression); err != nil { + return err + } + } // Validate volume attachments if err := validateVolumeAttachments(req.Volumes); err != nil { diff --git a/lib/instances/firecracker_test.go b/lib/instances/firecracker_test.go index ebec9e68..0e3e8dc9 100644 --- a/lib/instances/firecracker_test.go +++ b/lib/instances/firecracker_test.go @@ -49,7 +49,7 @@ func setupTestManagerForFirecrackerWithNetworkConfig(t *testing.T, networkCfg co MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeFirecracker, SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -212,7 +212,7 @@ func TestFirecrackerStandbyAndRestore(t *testing.T) { writeGuestFile(firstFilePath, firstFileContents) firstStandbyStart := time.Now() - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) firstStandbyDuration := time.Since(firstStandbyStart) t.Logf("first standby (full snapshot expected) took %v", firstStandbyDuration) @@ -231,7 +231,7 @@ func TestFirecrackerStandbyAndRestore(t *testing.T) { require.NoError(t, err, "restored instances should keep the retained snapshot base for the next diff snapshot") secondStandbyStart := time.Now() - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) secondStandbyDuration := time.Since(secondStandbyStart) t.Logf("second standby (diff snapshot expected) took %v", secondStandbyDuration) @@ -281,7 +281,7 @@ func TestFirecrackerStopClearsStaleSnapshot(t *testing.T) { require.Equal(t, StateRunning, inst.State) // Establish a realistic standby/restore lifecycle first. - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) require.Equal(t, StateStandby, inst.State) require.True(t, inst.HasSnapshot) @@ -397,7 +397,7 @@ func TestFirecrackerNetworkLifecycle(t *testing.T) { require.Equal(t, 0, exitCode) require.Contains(t, output, "Connection successful") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/fork.go b/lib/instances/fork.go index d68b8c22..203d1a64 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -56,7 +56,7 @@ func (m *manager) forkInstance(ctx context.Context, id string, req ForkInstanceR log.InfoContext(ctx, "fork from running requested; transitioning source to standby", "source_instance_id", id, "hypervisor", source.HypervisorType) - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, "", fmt.Errorf("standby source instance: %w", err) } @@ -421,7 +421,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe if _, err := m.startInstance(ctx, forkID, StartInstanceRequest{}); err != nil { return nil, fmt.Errorf("start forked instance for standby transition: %w", err) } - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) } case StateStandby: switch target { @@ -436,7 +436,7 @@ func (m *manager) applyForkTargetState(ctx context.Context, forkID string, targe case StateRunning: switch target { case StateStandby: - return returnWithReadiness(m.standbyInstance(ctx, forkID)) + return returnWithReadiness(m.standbyInstance(ctx, forkID, StandbyInstanceRequest{}, false)) case StateStopped: return returnWithReadiness(m.stopInstance(ctx, forkID)) } diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 8305a0c5..70cf368a 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -34,7 +34,7 @@ type Manager interface { DeleteSnapshot(ctx context.Context, snapshotID string) error ForkInstance(ctx context.Context, id string, req ForkInstanceRequest) (*Instance, error) ForkSnapshot(ctx context.Context, snapshotID string, req ForkSnapshotRequest) (*Instance, error) - StandbyInstance(ctx context.Context, id string) (*Instance, error) + StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) RestoreInstance(ctx context.Context, id string) (*Instance, error) RestoreSnapshot(ctx context.Context, id string, snapshotID string, req RestoreSnapshotRequest) (*Instance, error) StopInstance(ctx context.Context, id string) (*Instance, error) @@ -88,6 +88,9 @@ type manager struct { egressProxy *egressproxy.Service egressProxyServiceOptions egressproxy.ServiceOptions egressProxyMu sync.Mutex + snapshotDefaults SnapshotPolicy + compressionMu sync.Mutex + compressionJobs map[string]*compressionJob // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter @@ -101,7 +104,7 @@ var platformStarters = make(map[hypervisor.Type]hypervisor.VMStarter) // NewManager creates a new instances manager. // If meter is nil, metrics are disabled. // defaultHypervisor specifies which hypervisor to use when not specified in requests. -func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, meter metric.Meter, tracer trace.Tracer, memoryPolicy ...guestmemory.Policy) Manager { +func NewManager(p *paths.Paths, imageManager images.Manager, systemManager system.Manager, networkManager network.Manager, deviceManager devices.Manager, volumeManager volumes.Manager, limits ResourceLimits, defaultHypervisor hypervisor.Type, snapshotDefaults SnapshotPolicy, meter metric.Meter, tracer trace.Tracer, memoryPolicy ...guestmemory.Policy) Manager { // Validate and default the hypervisor type if defaultHypervisor == "" { defaultHypervisor = hypervisor.TypeCloudHypervisor @@ -134,6 +137,8 @@ func NewManager(p *paths.Paths, imageManager images.Manager, systemManager syste defaultHypervisor: defaultHypervisor, now: time.Now, guestMemoryPolicy: policy, + snapshotDefaults: snapshotDefaults, + compressionJobs: make(map[string]*compressionJob), } // Initialize metrics if meter is provided @@ -275,11 +280,11 @@ func (m *manager) ForkSnapshot(ctx context.Context, snapshotID string, req ForkS } // StandbyInstance puts an instance in standby (pause, snapshot, delete VMM) -func (m *manager) StandbyInstance(ctx context.Context, id string) (*Instance, error) { +func (m *manager) StandbyInstance(ctx context.Context, id string, req StandbyInstanceRequest) (*Instance, error) { lock := m.getInstanceLock(id) lock.Lock() defer lock.Unlock() - return m.standbyInstance(ctx, id) + return m.standbyInstance(ctx, id, req, false) } // RestoreInstance restores an instance from standby diff --git a/lib/instances/manager_darwin_test.go b/lib/instances/manager_darwin_test.go index a7557b20..7d966e02 100644 --- a/lib/instances/manager_darwin_test.go +++ b/lib/instances/manager_darwin_test.go @@ -57,7 +57,7 @@ func setupVZTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, MaxMemoryPerInstance: 0, } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) resourceMgr := resources.NewManager(cfg, p) resourceMgr.SetInstanceLister(mgr) @@ -470,7 +470,7 @@ func TestVZStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Putting instance in standby...") - inst, err = mgr.StandbyInstance(ctx, inst.Id) + inst, err = mgr.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/manager_test.go b/lib/instances/manager_test.go index 3003781c..70d7cbfb 100644 --- a/lib/instances/manager_test.go +++ b/lib/instances/manager_test.go @@ -55,7 +55,7 @@ func setupTestManager(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -1228,7 +1228,7 @@ func TestStorageOperations(t *testing.T) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", nil, nil).(*manager) + manager := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, "", SnapshotPolicy{}, nil, nil).(*manager) // Test metadata doesn't exist initially _, err := manager.loadMetadata("nonexistent") @@ -1348,7 +1348,7 @@ func TestStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/network_test.go b/lib/instances/network_test.go index d8537e80..a3febb0b 100644 --- a/lib/instances/network_test.go +++ b/lib/instances/network_test.go @@ -134,7 +134,7 @@ func TestCreateInstanceWithNetwork(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/qemu_test.go b/lib/instances/qemu_test.go index 6a58abee..155435b8 100644 --- a/lib/instances/qemu_test.go +++ b/lib/instances/qemu_test.go @@ -54,7 +54,7 @@ func setupTestManagerForQEMU(t *testing.T) (*manager, string) { MaxVcpusPerInstance: 0, // unlimited MaxMemoryPerInstance: 0, // unlimited } - mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, nil, nil).(*manager) + mgr := NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, hypervisor.TypeQEMU, SnapshotPolicy{}, nil, nil).(*manager) // Set up resource validation using the real ResourceManager resourceMgr := resources.NewManager(cfg, p) @@ -841,7 +841,7 @@ func TestQEMUStandbyAndRestore(t *testing.T) { // Standby instance t.Log("Standing by instance...") - inst, err = manager.StandbyInstance(ctx, inst.Id) + inst, err = manager.StandbyInstance(ctx, inst.Id, StandbyInstanceRequest{}) require.NoError(t, err) assert.Equal(t, StateStandby, inst.State) assert.True(t, inst.HasSnapshot) diff --git a/lib/instances/resource_limits_test.go b/lib/instances/resource_limits_test.go index d34b9c45..32cad66c 100644 --- a/lib/instances/resource_limits_test.go +++ b/lib/instances/resource_limits_test.go @@ -176,7 +176,7 @@ func createTestManager(t *testing.T, limits ResourceLimits) *manager { deviceMgr := devices.NewManager(p) volumeMgr := volumes.NewManager(p, 0, nil) - return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", nil, nil).(*manager) + return NewManager(p, imageMgr, systemMgr, networkMgr, deviceMgr, volumeMgr, limits, "", SnapshotPolicy{}, nil, nil).(*manager) } func TestResourceLimits_StructValues(t *testing.T) { diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 369525fe..6020c167 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -69,6 +69,10 @@ func (m *manager) restoreInstance( // 3. Get snapshot directory snapshotDir := m.paths.InstanceSnapshotLatest(id) + waitForCompression := stored.HypervisorType == hypervisor.TypeCloudHypervisor + if err := m.ensureSnapshotMemoryReady(ctx, snapshotDir, m.snapshotJobKeyForInstance(id), waitForCompression); err != nil { + return nil, fmt.Errorf("prepare standby snapshot memory: %w", err) + } starter, err := m.getVMStarter(stored.HypervisorType) if err != nil { return nil, fmt.Errorf("get vm starter: %w", err) diff --git a/lib/instances/snapshot.go b/lib/instances/snapshot.go index fcf65e79..69b116db 100644 --- a/lib/instances/snapshot.go +++ b/lib/instances/snapshot.go @@ -91,7 +91,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps if err := ensureGuestAgentReadyForForkPhase(ctx, &inst.StoredMetadata, "before running snapshot"); err != nil { return nil, err } - if _, err := m.standbyInstance(ctx, id); err != nil { + if _, err := m.standbyInstance(ctx, id, StandbyInstanceRequest{}, true); err != nil { return nil, fmt.Errorf("standby source instance: %w", err) } restoreSource = true @@ -138,9 +138,27 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone + effectiveCompression, err := m.resolveSnapshotCompressionPolicy(stored, req.Compression) + if err != nil { + return nil, err + } + if effectiveCompression.Enabled { + rec.Snapshot.Compression = cloneCompressionConfig(&effectiveCompression) + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateCompressing + } if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } + if effectiveCompression.Enabled { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForSnapshot(snapshotID), + OwnerID: stored.Id, + SnapshotID: snapshotID, + SnapshotDir: snapshotGuestDir, + Policy: effectiveCompression, + }) + } cu.Release() log.InfoContext(ctx, "snapshot created", "instance_id", id, "snapshot_id", snapshotID, "kind", req.Kind) return &rec.Snapshot, nil @@ -170,6 +188,7 @@ func (m *manager) createSnapshot(ctx context.Context, id string, req CreateSnaps return nil, err } rec.Snapshot.SizeBytes = sizeBytes + rec.Snapshot.CompressionState = snapshotstore.SnapshotCompressionStateNone if err := m.saveSnapshotRecord(rec); err != nil { return nil, err } @@ -221,6 +240,9 @@ func (m *manager) restoreSnapshot(ctx context.Context, id string, snapshotID str return nil, err } + m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) + m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := m.replaceInstanceWithSnapshotPayload(snapshotID, id); err != nil { return nil, err } @@ -335,6 +357,9 @@ func (m *manager) forkSnapshot(ctx context.Context, snapshotID string, req ForkS }) defer cu.Clean() + m.cancelCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID)) + m.waitCompressionJob(m.snapshotJobKeyForSnapshot(snapshotID), 2*time.Second) + if err := forkvm.CopyGuestDirectory(m.paths.SnapshotGuestDir(snapshotID), dstDir); err != nil { if errors.Is(err, forkvm.ErrSparseCopyUnsupported) { return nil, fmt.Errorf("fork from snapshot requires sparse-capable filesystem (SEEK_DATA/SEEK_HOLE unsupported): %w", err) @@ -461,6 +486,12 @@ func validateCreateSnapshotRequest(req CreateSnapshotRequest) error { if req.Kind != SnapshotKindStandby && req.Kind != SnapshotKindStopped { return fmt.Errorf("%w: kind must be one of %s, %s", ErrInvalidRequest, SnapshotKindStandby, SnapshotKindStopped) } + if req.Kind == SnapshotKindStopped && req.Compression != nil && req.Compression.Enabled { + return fmt.Errorf("%w: compression is only supported for standby snapshots", ErrInvalidRequest) + } + if _, err := normalizeCompressionConfig(req.Compression); err != nil { + return err + } if err := tags.Validate(req.Tags); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } diff --git a/lib/instances/snapshot_compression.go b/lib/instances/snapshot_compression.go new file mode 100644 index 00000000..4cab65f2 --- /dev/null +++ b/lib/instances/snapshot_compression.go @@ -0,0 +1,497 @@ +package instances + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "time" + + "github.com/kernel/hypeman/lib/hypervisor" + "github.com/kernel/hypeman/lib/logger" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4/v4" +) + +const ( + defaultSnapshotCompressionZstdLevel = 1 + maxSnapshotCompressionZstdLevel = 19 +) + +type compressionJob struct { + cancel context.CancelFunc + done chan struct{} +} + +type compressionTarget struct { + Key string + OwnerID string + SnapshotID string + SnapshotDir string + Policy snapshotstore.SnapshotCompressionConfig +} + +func cloneCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) *snapshotstore.SnapshotCompressionConfig { + if cfg == nil { + return nil + } + cloned := *cfg + if cfg.Level != nil { + v := *cfg.Level + cloned.Level = &v + } + return &cloned +} + +func cloneSnapshotPolicy(policy *SnapshotPolicy) *SnapshotPolicy { + if policy == nil { + return nil + } + return &SnapshotPolicy{ + Compression: cloneCompressionConfig(policy.Compression), + } +} + +func normalizeCompressionConfig(cfg *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if cfg == nil || !cfg.Enabled { + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil + } + + normalized := snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: cfg.Algorithm, + } + switch normalized.Algorithm { + case "": + normalized.Algorithm = snapshotstore.SnapshotCompressionAlgorithmZstd + case snapshotstore.SnapshotCompressionAlgorithmZstd, snapshotstore.SnapshotCompressionAlgorithmLz4: + default: + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + + switch normalized.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + if level < 1 || level > maxSnapshotCompressionZstdLevel { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: invalid zstd level %d (must be between 1 and %d)", ErrInvalidRequest, level, maxSnapshotCompressionZstdLevel) + } + normalized.Level = &level + case snapshotstore.SnapshotCompressionAlgorithmLz4: + if cfg.Level != nil { + return snapshotstore.SnapshotCompressionConfig{}, fmt.Errorf("%w: lz4 does not support level", ErrInvalidRequest) + } + } + return normalized, nil +} + +func (m *manager) resolveSnapshotCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (snapshotstore.SnapshotCompressionConfig, error) { + if override != nil { + return normalizeCompressionConfig(override) + } + if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { + return normalizeCompressionConfig(stored.SnapshotPolicy.Compression) + } + if m.snapshotDefaults.Compression != nil { + return normalizeCompressionConfig(m.snapshotDefaults.Compression) + } + return snapshotstore.SnapshotCompressionConfig{Enabled: false}, nil +} + +func defaultCloudHypervisorStandbyCompressionPolicy() snapshotstore.SnapshotCompressionConfig { + return snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + } +} + +func (m *manager) resolveStandbyCompressionPolicy(stored *StoredMetadata, override *snapshotstore.SnapshotCompressionConfig) (*snapshotstore.SnapshotCompressionConfig, error) { + if override != nil { + cfg, err := normalizeCompressionConfig(override) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if stored != nil && stored.SnapshotPolicy != nil && stored.SnapshotPolicy.Compression != nil { + cfg, err := normalizeCompressionConfig(stored.SnapshotPolicy.Compression) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if m.snapshotDefaults.Compression != nil { + cfg, err := normalizeCompressionConfig(m.snapshotDefaults.Compression) + if err != nil { + return nil, err + } + if !cfg.Enabled { + return nil, nil + } + return &cfg, nil + } + + if stored != nil && stored.HypervisorType == hypervisor.TypeCloudHypervisor { + cfg := defaultCloudHypervisorStandbyCompressionPolicy() + return &cfg, nil + } + + return nil, nil +} + +func (m *manager) snapshotJobKeyForInstance(instanceID string) string { + return "instance:" + instanceID +} + +func (m *manager) snapshotJobKeyForSnapshot(snapshotID string) string { + return "snapshot:" + snapshotID +} + +func (m *manager) startCompressionJob(ctx context.Context, target compressionTarget) { + if target.Key == "" || !target.Policy.Enabled { + return + } + + m.compressionMu.Lock() + if _, exists := m.compressionJobs[target.Key]; exists { + m.compressionMu.Unlock() + return + } + jobCtx, cancel := context.WithCancel(context.Background()) + job := &compressionJob{ + cancel: cancel, + done: make(chan struct{}), + } + m.compressionJobs[target.Key] = job + m.compressionMu.Unlock() + + go func() { + defer func() { + m.compressionMu.Lock() + delete(m.compressionJobs, target.Key) + m.compressionMu.Unlock() + close(job.done) + }() + + log := logger.FromContext(ctx) + rawPath, ok := findRawSnapshotMemoryFile(target.SnapshotDir) + if !ok { + if _, _, found := findCompressedSnapshotMemoryFile(target.SnapshotDir); found && target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, nil, nil) + } + return + } + + uncompressedSize, compressedSize, err := compressSnapshotMemoryFile(jobCtx, rawPath, target.Policy) + if err != nil { + if errors.Is(err, context.Canceled) { + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateNone, "", nil, nil, nil) + } + return + } + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateError, err.Error(), &target.Policy, nil, nil) + } + log.WarnContext(ctx, "snapshot compression failed", "snapshot_dir", target.SnapshotDir, "error", err) + return + } + + if target.SnapshotID != "" { + _ = m.updateSnapshotCompressionMetadata(target.SnapshotID, snapshotstore.SnapshotCompressionStateCompressed, "", &target.Policy, &compressedSize, &uncompressedSize) + } + }() +} + +func (m *manager) cancelCompressionJob(key string) { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job != nil { + job.cancel() + } +} + +func (m *manager) waitCompressionJob(key string, timeout time.Duration) { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job == nil { + return + } + + timer := time.NewTimer(timeout) + defer timer.Stop() + select { + case <-job.done: + case <-timer.C: + } +} + +func (m *manager) waitCompressionJobContext(ctx context.Context, key string) error { + m.compressionMu.Lock() + job := m.compressionJobs[key] + m.compressionMu.Unlock() + if job == nil { + return nil + } + + select { + case <-job.done: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (m *manager) ensureSnapshotMemoryReady(ctx context.Context, snapshotDir, jobKey string, waitForCompression bool) error { + if jobKey != "" { + if waitForCompression { + if err := m.waitCompressionJobContext(ctx, jobKey); err != nil { + return err + } + } else { + m.cancelCompressionJob(jobKey) + m.waitCompressionJob(jobKey, 2*time.Second) + } + } + + if _, ok := findRawSnapshotMemoryFile(snapshotDir); ok { + return nil + } + compressedPath, algorithm, ok := findCompressedSnapshotMemoryFile(snapshotDir) + if !ok { + return nil + } + return decompressSnapshotMemoryFile(ctx, compressedPath, algorithm) +} + +func (m *manager) updateSnapshotCompressionMetadata(snapshotID, state, compressionError string, cfg *snapshotstore.SnapshotCompressionConfig, compressedSize, uncompressedSize *int64) error { + rec, err := m.loadSnapshotRecord(snapshotID) + if err != nil { + return err + } + rec.Snapshot.CompressionState = state + rec.Snapshot.CompressionError = compressionError + rec.Snapshot.Compression = cloneCompressionConfig(cfg) + rec.Snapshot.CompressedSizeBytes = compressedSize + rec.Snapshot.UncompressedSizeBytes = uncompressedSize + + if state == snapshotstore.SnapshotCompressionStateCompressed { + sizeBytes, sizeErr := snapshotstore.DirectoryFileSize(m.paths.SnapshotGuestDir(snapshotID)) + if sizeErr == nil { + rec.Snapshot.SizeBytes = sizeBytes + } + } + return m.saveSnapshotRecord(rec) +} + +func findRawSnapshotMemoryFile(snapshotDir string) (string, bool) { + for _, candidate := range snapshotMemoryFileCandidates(snapshotDir) { + if st, err := os.Stat(candidate); err == nil && st.Mode().IsRegular() { + return candidate, true + } + } + return "", false +} + +func findCompressedSnapshotMemoryFile(snapshotDir string) (string, snapshotstore.SnapshotCompressionAlgorithm, bool) { + for _, raw := range snapshotMemoryFileCandidates(snapshotDir) { + zstdPath := raw + ".zst" + if st, err := os.Stat(zstdPath); err == nil && st.Mode().IsRegular() { + return zstdPath, snapshotstore.SnapshotCompressionAlgorithmZstd, true + } + lz4Path := raw + ".lz4" + if st, err := os.Stat(lz4Path); err == nil && st.Mode().IsRegular() { + return lz4Path, snapshotstore.SnapshotCompressionAlgorithmLz4, true + } + } + return "", "", false +} + +func snapshotMemoryFileCandidates(snapshotDir string) []string { + return []string{ + filepath.Join(snapshotDir, "memory-ranges"), + filepath.Join(snapshotDir, "memory"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory-ranges"), + filepath.Join(snapshotDir, "snapshots", "snapshot-latest", "memory"), + } +} + +func compressSnapshotMemoryFile(ctx context.Context, rawPath string, cfg snapshotstore.SnapshotCompressionConfig) (int64, int64, error) { + rawInfo, err := os.Stat(rawPath) + if err != nil { + return 0, 0, fmt.Errorf("stat raw memory snapshot: %w", err) + } + uncompressedSize := rawInfo.Size() + + compressedPath := compressedPathFor(rawPath, cfg.Algorithm) + tmpPath := compressedPath + ".tmp" + removeCompressedSnapshotArtifacts(rawPath) + _ = os.Remove(tmpPath) + + if err := runCompression(ctx, rawPath, tmpPath, cfg); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, err + } + if err := os.Rename(tmpPath, compressedPath); err != nil { + _ = os.Remove(tmpPath) + return 0, 0, fmt.Errorf("finalize compressed snapshot: %w", err) + } + + compressedInfo, err := os.Stat(compressedPath) + if err != nil { + return 0, 0, fmt.Errorf("stat compressed snapshot: %w", err) + } + if err := os.Remove(rawPath); err != nil { + return 0, 0, fmt.Errorf("remove raw memory snapshot: %w", err) + } + return uncompressedSize, compressedInfo.Size(), nil +} + +func runCompression(ctx context.Context, srcPath, dstPath string, cfg snapshotstore.SnapshotCompressionConfig) error { + src, err := os.Open(srcPath) + if err != nil { + return fmt.Errorf("open source snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(dstPath) + if err != nil { + return fmt.Errorf("create compressed snapshot: %w", err) + } + defer dst.Close() + + switch cfg.Algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + level := defaultSnapshotCompressionZstdLevel + if cfg.Level != nil { + level = *cfg.Level + } + enc, err := zstd.NewWriter(dst, zstd.WithEncoderLevel(zstd.EncoderLevelFromZstd(level))) + if err != nil { + return fmt.Errorf("create zstd encoder: %w", err) + } + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close zstd encoder: %w", err) + } + case snapshotstore.SnapshotCompressionAlgorithmLz4: + enc := lz4.NewWriter(dst) + if err := enc.Apply(lz4.CompressionLevelOption(lz4.Fast)); err != nil { + return fmt.Errorf("configure lz4 encoder: %w", err) + } + if err := copyWithContext(ctx, enc, src); err != nil { + _ = enc.Close() + return err + } + if err := enc.Close(); err != nil { + return fmt.Errorf("close lz4 encoder: %w", err) + } + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, cfg.Algorithm) + } + return nil +} + +func decompressSnapshotMemoryFile(ctx context.Context, compressedPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) error { + rawPath := strings.TrimSuffix(strings.TrimSuffix(compressedPath, ".zst"), ".lz4") + tmpRawPath := rawPath + ".tmp" + _ = os.Remove(tmpRawPath) + + src, err := os.Open(compressedPath) + if err != nil { + return fmt.Errorf("open compressed snapshot: %w", err) + } + defer src.Close() + + dst, err := os.Create(tmpRawPath) + if err != nil { + return fmt.Errorf("create decompressed snapshot file: %w", err) + } + defer dst.Close() + + var reader io.Reader + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmZstd: + dec, err := zstd.NewReader(src) + if err != nil { + return fmt.Errorf("create zstd decoder: %w", err) + } + defer dec.Close() + reader = dec + case snapshotstore.SnapshotCompressionAlgorithmLz4: + reader = lz4.NewReader(src) + default: + return fmt.Errorf("%w: unsupported compression algorithm %q", ErrInvalidRequest, algorithm) + } + + if err := copyWithContext(ctx, dst, reader); err != nil { + _ = os.Remove(tmpRawPath) + return err + } + if err := os.Rename(tmpRawPath, rawPath); err != nil { + _ = os.Remove(tmpRawPath) + return fmt.Errorf("finalize decompressed snapshot: %w", err) + } + removeCompressedSnapshotArtifacts(rawPath) + return nil +} + +func compressedPathFor(rawPath string, algorithm snapshotstore.SnapshotCompressionAlgorithm) string { + switch algorithm { + case snapshotstore.SnapshotCompressionAlgorithmLz4: + return rawPath + ".lz4" + default: + return rawPath + ".zst" + } +} + +func removeCompressedSnapshotArtifacts(rawPath string) { + for _, path := range []string{ + rawPath + ".zst", + rawPath + ".zst.tmp", + rawPath + ".lz4", + rawPath + ".lz4.tmp", + } { + _ = os.Remove(path) + } +} + +func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) error { + buf := make([]byte, 1024*1024) + for { + if err := ctx.Err(); err != nil { + return err + } + n, readErr := src.Read(buf) + if n > 0 { + if _, writeErr := dst.Write(buf[:n]); writeErr != nil { + return fmt.Errorf("write compressed stream: %w", writeErr) + } + } + if readErr != nil { + if errors.Is(readErr, io.EOF) { + return nil + } + return fmt.Errorf("read snapshot stream: %w", readErr) + } + } +} diff --git a/lib/instances/snapshot_compression_test.go b/lib/instances/snapshot_compression_test.go new file mode 100644 index 00000000..bf599481 --- /dev/null +++ b/lib/instances/snapshot_compression_test.go @@ -0,0 +1,147 @@ +package instances + +import ( + "errors" + "testing" + + "github.com/kernel/hypeman/lib/hypervisor" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNormalizeCompressionConfig(t *testing.T) { + t.Parallel() + + cfg, err := normalizeCompressionConfig(nil) + require.NoError(t, err) + assert.False(t, cfg.Enabled) + + cfg, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }) + require.NoError(t, err) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 1, *cfg.Level) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(25), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) + + _, err = normalizeCompressionConfig(&snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + Level: intPtr(1), + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestResolveSnapshotCompressionPolicyPrecedence(t *testing.T) { + t.Parallel() + + m := &manager{ + snapshotDefaults: SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(2), + }, + }, + } + + stored := &StoredMetadata{ + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmLz4, + }, + }, + } + + cfg, err := m.resolveSnapshotCompressionPolicy(stored, &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(4), + }) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 4, *cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(stored, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg, err = m.resolveSnapshotCompressionPolicy(&StoredMetadata{}, nil) + require.NoError(t, err) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmZstd, cfg.Algorithm) + require.NotNil(t, cfg.Level) + assert.Equal(t, 2, *cfg.Level) +} + +func TestResolveStandbyCompressionPolicyCloudHypervisorDefault(t *testing.T) { + t.Parallel() + + m := &manager{} + + cfg, err := m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeCloudHypervisor, + }, nil) + require.NoError(t, err) + require.NotNil(t, cfg) + assert.True(t, cfg.Enabled) + assert.Equal(t, snapshotstore.SnapshotCompressionAlgorithmLz4, cfg.Algorithm) + assert.Nil(t, cfg.Level) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeCloudHypervisor, + }, &snapshotstore.SnapshotCompressionConfig{Enabled: false}) + require.NoError(t, err) + assert.Nil(t, cfg) + + cfg, err = m.resolveStandbyCompressionPolicy(&StoredMetadata{ + HypervisorType: hypervisor.TypeQEMU, + }, nil) + require.NoError(t, err) + assert.Nil(t, cfg) +} + +func TestValidateCreateRequestSnapshotPolicy(t *testing.T) { + t.Parallel() + + req := &CreateInstanceRequest{ + Name: "compression-test", + Image: "docker.io/library/alpine:latest", + SnapshotPolicy: &SnapshotPolicy{ + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + Algorithm: snapshotstore.SnapshotCompressionAlgorithmZstd, + Level: intPtr(0), + }, + }, + } + err := validateCreateRequest(req) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} + +func TestValidateCreateSnapshotRequestRejectsStoppedCompression(t *testing.T) { + t.Parallel() + + err := validateCreateSnapshotRequest(CreateSnapshotRequest{ + Kind: SnapshotKindStopped, + Compression: &snapshotstore.SnapshotCompressionConfig{ + Enabled: true, + }, + }) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrInvalidRequest)) +} diff --git a/lib/instances/snapshot_integration_scenario_test.go b/lib/instances/snapshot_integration_scenario_test.go index ab30cb56..fb2147e5 100644 --- a/lib/instances/snapshot_integration_scenario_test.go +++ b/lib/instances/snapshot_integration_scenario_test.go @@ -68,8 +68,7 @@ func runStandbySnapshotScenario(t *testing.T, mgr *manager, tmpDir string, cfg s source, err = waitForInstanceState(ctx, mgr, sourceID, StateRunning, 20*time.Second) requireNoErr(err) require.Equal(t, StateRunning, source.State) - - _, err = mgr.StandbyInstance(ctx, sourceID) + _, err = mgr.StandbyInstance(ctx, sourceID, StandbyInstanceRequest{}) requireNoErr(err) snapshot, err := mgr.CreateSnapshot(ctx, sourceID, CreateSnapshotRequest{ diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 51ef7b85..d218a9ce 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -11,6 +11,7 @@ import ( "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" + snapshotstore "github.com/kernel/hypeman/lib/snapshot" "go.opentelemetry.io/otel/trace" ) @@ -20,6 +21,8 @@ func (m *manager) standbyInstance( ctx context.Context, id string, + req StandbyInstanceRequest, + skipCompression bool, ) (*Instance, error) { start := time.Now() log := logger.FromContext(ctx) @@ -55,6 +58,17 @@ func (m *manager) standbyInstance( return nil, fmt.Errorf("%w: standby is not supported for instances with vGPU attached (driver limitation)", ErrInvalidState) } + // Resolve/validate compression policy early so invalid request/config + // fails before any state transition side effects. + var compressionPolicy *snapshotstore.SnapshotCompressionConfig + if !skipCompression { + policy, err := m.resolveStandbyCompressionPolicy(stored, req.Compression) + if err != nil { + return nil, err + } + compressionPolicy = policy + } + // 3. Get network allocation BEFORE killing VMM (while we can still query it) // This is needed to delete the TAP device after VMM shuts down var networkAlloc *network.Allocation @@ -163,6 +177,16 @@ func (m *manager) standbyInstance( // Return instance with derived state (should be Standby now) finalInst := m.toInstance(ctx, meta) + + if compressionPolicy != nil { + m.startCompressionJob(ctx, compressionTarget{ + Key: m.snapshotJobKeyForInstance(stored.Id), + OwnerID: stored.Id, + SnapshotDir: snapshotDir, + Policy: *compressionPolicy, + }) + } + log.InfoContext(ctx, "instance put in standby successfully", "instance_id", id, "state", finalInst.State) return &finalInst, nil } diff --git a/lib/instances/types.go b/lib/instances/types.go index 8dc48837..add0df64 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -136,6 +136,9 @@ type StoredMetadata struct { SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + // Snapshot policy defaults for this instance. + SnapshotPolicy *SnapshotPolicy + // Shutdown configuration StopTimeout int // Grace period in seconds for graceful stop (0 = use default 5s) @@ -216,6 +219,7 @@ type CreateInstanceRequest struct { Cmd []string // Override image cmd (nil = use image default) SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + SnapshotPolicy *SnapshotPolicy // Optional snapshot policy defaults for this instance } // StartInstanceRequest is the domain request for starting a stopped instance @@ -249,9 +253,15 @@ type ListSnapshotsFilter = snapshot.ListSnapshotsFilter // CreateSnapshotRequest is the domain request for creating a snapshot. type CreateSnapshotRequest struct { - Kind SnapshotKind // Required: Standby or Stopped - Name string // Optional: unique per source instance - Tags tags.Tags // Optional user-defined key-value tags + Kind SnapshotKind // Required: Standby or Stopped + Name string // Optional: unique per source instance + Tags tags.Tags // Optional user-defined key-value tags + Compression *snapshot.SnapshotCompressionConfig // Optional compression override +} + +// StandbyInstanceRequest is the domain request for putting an instance into standby. +type StandbyInstanceRequest struct { + Compression *snapshot.SnapshotCompressionConfig // Optional compression override } // RestoreSnapshotRequest is the domain request for restoring a snapshot in-place. @@ -267,6 +277,11 @@ type ForkSnapshotRequest struct { TargetHypervisor hypervisor.Type // Optional, allowed only for Stopped snapshots } +// SnapshotPolicy defines default snapshot behavior for an instance. +type SnapshotPolicy struct { + Compression *snapshot.SnapshotCompressionConfig +} + // AttachVolumeRequest is the domain request for attaching a volume (used for API compatibility) type AttachVolumeRequest struct { MountPath string diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index 161067ad..e52a314a 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -129,6 +129,14 @@ const ( RestoreSnapshotRequestTargetHypervisorVz RestoreSnapshotRequestTargetHypervisor = "vz" ) +// Defines values for SnapshotCompressionState. +const ( + SnapshotCompressionStateCompressed SnapshotCompressionState = "compressed" + SnapshotCompressionStateCompressing SnapshotCompressionState = "compressing" + SnapshotCompressionStateError SnapshotCompressionState = "error" + SnapshotCompressionStateNone SnapshotCompressionState = "none" +) + // Defines values for SnapshotSourceHypervisor. const ( CloudHypervisor SnapshotSourceHypervisor = "cloud-hypervisor" @@ -137,6 +145,12 @@ const ( Vz SnapshotSourceHypervisor = "vz" ) +// Defines values for SnapshotCompressionConfigAlgorithm. +const ( + Lz4 SnapshotCompressionConfigAlgorithm = "lz4" + Zstd SnapshotCompressionConfigAlgorithm = "zstd" +) + // Defines values for SnapshotKind. const ( SnapshotKindStandby SnapshotKind = "Standby" @@ -372,7 +386,8 @@ type CreateInstanceRequest struct { // When true, DKMS (Dynamic Kernel Module Support) will not work, // preventing compilation of out-of-tree kernel modules (e.g., NVIDIA vGPU drivers). // Recommended for workloads that don't need kernel module compilation. - SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SkipKernelHeaders *bool `json:"skip_kernel_headers,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // Tags User-defined key-value tags. Tags *Tags `json:"tags,omitempty"` @@ -448,6 +463,8 @@ type CreateInstanceRequestNetworkEgressEnforcementMode string // CreateSnapshotRequest defines model for CreateSnapshotRequest. type CreateSnapshotRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + // Kind Snapshot capture kind Kind SnapshotKind `json:"kind"` @@ -794,7 +811,8 @@ type Instance struct { OverlaySize *string `json:"overlay_size,omitempty"` // Size Base memory size (human-readable) - Size *string `json:"size,omitempty"` + Size *string `json:"size,omitempty"` + SnapshotPolicy *SnapshotPolicy `json:"snapshot_policy,omitempty"` // StartedAt Start timestamp (RFC3339) StartedAt *time.Time `json:"started_at"` @@ -999,6 +1017,16 @@ type RestoreSnapshotRequestTargetHypervisor string // Snapshot defines model for Snapshot. type Snapshot struct { + // CompressedSizeBytes Compressed memory payload size in bytes + CompressedSizeBytes *int64 `json:"compressed_size_bytes"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + + // CompressionError Compression error message when compression_state is error + CompressionError *string `json:"compression_error"` + + // CompressionState Compression status of the snapshot payload memory file + CompressionState *SnapshotCompressionState `json:"compression_state,omitempty"` + // CreatedAt Snapshot creation timestamp CreatedAt time.Time `json:"created_at"` @@ -1025,17 +1053,48 @@ type Snapshot struct { // Tags User-defined key-value tags. Tags *Tags `json:"tags,omitempty"` + + // UncompressedSizeBytes Uncompressed memory payload size in bytes + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes"` } +// SnapshotCompressionState Compression status of the snapshot payload memory file +type SnapshotCompressionState string + // SnapshotSourceHypervisor Source instance hypervisor at snapshot creation time type SnapshotSourceHypervisor string +// SnapshotCompressionConfig defines model for SnapshotCompressionConfig. +type SnapshotCompressionConfig struct { + // Algorithm Compression algorithm (defaults to zstd when enabled) + Algorithm *SnapshotCompressionConfigAlgorithm `json:"algorithm,omitempty"` + + // Enabled Enable snapshot memory compression + Enabled bool `json:"enabled"` + + // Level Compression level for zstd only + Level *int `json:"level,omitempty"` +} + +// SnapshotCompressionConfigAlgorithm Compression algorithm (defaults to zstd when enabled) +type SnapshotCompressionConfigAlgorithm string + // SnapshotKind Snapshot capture kind type SnapshotKind string +// SnapshotPolicy defines model for SnapshotPolicy. +type SnapshotPolicy struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // SnapshotTargetState Target state when restoring or forking from a snapshot type SnapshotTargetState string +// StandbyInstanceRequest defines model for StandbyInstanceRequest. +type StandbyInstanceRequest struct { + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` +} + // Tags User-defined key-value tags. type Tags map[string]string @@ -1273,6 +1332,9 @@ type CreateInstanceSnapshotJSONRequestBody = CreateSnapshotRequest // RestoreInstanceSnapshotJSONRequestBody defines body for RestoreInstanceSnapshot for application/json ContentType. type RestoreInstanceSnapshotJSONRequestBody = RestoreSnapshotRequest +// StandbyInstanceJSONRequestBody defines body for StandbyInstance for application/json ContentType. +type StandbyInstanceJSONRequestBody = StandbyInstanceRequest + // StartInstanceJSONRequestBody defines body for StartInstance for application/json ContentType. type StartInstanceJSONRequestBody StartInstanceJSONBody @@ -1456,8 +1518,10 @@ type ClientInterface interface { RestoreInstanceSnapshot(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) - // StandbyInstance request - StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) + // StandbyInstanceWithBody request with any body + StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) + + StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) // StartInstanceWithBody request with any body StartInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) @@ -1936,8 +2000,20 @@ func (c *Client) RestoreInstanceSnapshot(ctx context.Context, id string, snapsho return c.Client.Do(req) } -func (c *Client) StandbyInstance(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*http.Response, error) { - req, err := NewStandbyInstanceRequest(c.Server, id) +func (c *Client) StandbyInstanceWithBody(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequestWithBody(c.Server, id, contentType, body) + if err != nil { + return nil, err + } + req = req.WithContext(ctx) + if err := c.applyEditors(ctx, req, reqEditors); err != nil { + return nil, err + } + return c.Client.Do(req) +} + +func (c *Client) StandbyInstance(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*http.Response, error) { + req, err := NewStandbyInstanceRequest(c.Server, id, body) if err != nil { return nil, err } @@ -3358,8 +3434,19 @@ func NewRestoreInstanceSnapshotRequestWithBody(server string, id string, snapsho return req, nil } -// NewStandbyInstanceRequest generates requests for StandbyInstance -func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) { +// NewStandbyInstanceRequest calls the generic StandbyInstance builder with application/json body +func NewStandbyInstanceRequest(server string, id string, body StandbyInstanceJSONRequestBody) (*http.Request, error) { + var bodyReader io.Reader + buf, err := json.Marshal(body) + if err != nil { + return nil, err + } + bodyReader = bytes.NewReader(buf) + return NewStandbyInstanceRequestWithBody(server, id, "application/json", bodyReader) +} + +// NewStandbyInstanceRequestWithBody generates requests for StandbyInstance with any type of body +func NewStandbyInstanceRequestWithBody(server string, id string, contentType string, body io.Reader) (*http.Request, error) { var err error var pathParam0 string @@ -3384,11 +3471,13 @@ func NewStandbyInstanceRequest(server string, id string) (*http.Request, error) return nil, err } - req, err := http.NewRequest("POST", queryURL.String(), nil) + req, err := http.NewRequest("POST", queryURL.String(), body) if err != nil { return nil, err } + req.Header.Add("Content-Type", contentType) + return req, nil } @@ -4298,8 +4387,10 @@ type ClientWithResponsesInterface interface { RestoreInstanceSnapshotWithResponse(ctx context.Context, id string, snapshotId string, body RestoreInstanceSnapshotJSONRequestBody, reqEditors ...RequestEditorFn) (*RestoreInstanceSnapshotResponse, error) - // StandbyInstanceWithResponse request - StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + // StandbyInstanceWithBodyWithResponse request with any body + StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) + + StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) // StartInstanceWithBodyWithResponse request with any body StartInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StartInstanceResponse, error) @@ -5049,6 +5140,7 @@ type StandbyInstanceResponse struct { Body []byte HTTPResponse *http.Response JSON200 *Instance + JSON400 *Error JSON404 *Error JSON409 *Error JSON500 *Error @@ -5770,9 +5862,17 @@ func (c *ClientWithResponses) RestoreInstanceSnapshotWithResponse(ctx context.Co return ParseRestoreInstanceSnapshotResponse(rsp) } -// StandbyInstanceWithResponse request returning *StandbyInstanceResponse -func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { - rsp, err := c.StandbyInstance(ctx, id, reqEditors...) +// StandbyInstanceWithBodyWithResponse request with arbitrary body returning *StandbyInstanceResponse +func (c *ClientWithResponses) StandbyInstanceWithBodyWithResponse(ctx context.Context, id string, contentType string, body io.Reader, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstanceWithBody(ctx, id, contentType, body, reqEditors...) + if err != nil { + return nil, err + } + return ParseStandbyInstanceResponse(rsp) +} + +func (c *ClientWithResponses) StandbyInstanceWithResponse(ctx context.Context, id string, body StandbyInstanceJSONRequestBody, reqEditors ...RequestEditorFn) (*StandbyInstanceResponse, error) { + rsp, err := c.StandbyInstance(ctx, id, body, reqEditors...) if err != nil { return nil, err } @@ -7201,6 +7301,13 @@ func ParseStandbyInstanceResponse(rsp *http.Response) (*StandbyInstanceResponse, } response.JSON200 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 400: + var dest Error + if err := json.Unmarshal(bodyBytes, &dest); err != nil { + return nil, err + } + response.JSON400 = &dest + case strings.Contains(rsp.Header.Get("Content-Type"), "json") && rsp.StatusCode == 404: var dest Error if err := json.Unmarshal(bodyBytes, &dest); err != nil { @@ -11262,7 +11369,8 @@ func (response RestoreInstanceSnapshot501JSONResponse) VisitRestoreInstanceSnaps } type StandbyInstanceRequestObject struct { - Id string `json:"id"` + Id string `json:"id"` + Body *StandbyInstanceJSONRequestBody } type StandbyInstanceResponseObject interface { @@ -11278,6 +11386,15 @@ func (response StandbyInstance200JSONResponse) VisitStandbyInstanceResponse(w ht return json.NewEncoder(w).Encode(response) } +type StandbyInstance400JSONResponse Error + +func (response StandbyInstance400JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(400) + + return json.NewEncoder(w).Encode(response) +} + type StandbyInstance404JSONResponse Error func (response StandbyInstance404JSONResponse) VisitStandbyInstanceResponse(w http.ResponseWriter) error { @@ -12906,6 +13023,13 @@ func (sh *strictHandler) StandbyInstance(w http.ResponseWriter, r *http.Request, request.Id = id + var body StandbyInstanceJSONRequestBody + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + sh.options.RequestErrorHandlerFunc(w, r, fmt.Errorf("can't decode JSON body: %w", err)) + return + } + request.Body = &body + handler := func(ctx context.Context, w http.ResponseWriter, r *http.Request, request interface{}) (interface{}, error) { return sh.ssi.StandbyInstance(ctx, request.(StandbyInstanceRequestObject)) } @@ -13374,217 +13498,222 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XLbOpLoq6B0Z2vkGUmWP+I42jq117GTHO+JE9849t6do1wZIiEJYxLgAUA5Sip/", - "5wHmEedJbqEB8EugRDu2E28yNXUikyA+Gt2N7kZ/fG4FPE44I0zJ1uBzSwYzEmP4eaAUDmYXPEpj8o78", - "kRKp9ONE8IQIRQk0innK1CjBaqb/CokMBE0U5aw1aJ1iNUPXMyIImkMvSM54GoVoTBB8R8JWp0U+4jiJ", - "SGvQ2oyZ2gyxwq1OSy0S/UgqQdm09aXTEgSHnEULM8wEp5FqDSY4kqRTGfZEd42wRPqTLnyT9TfmPCKY", - "tb5Aj3+kVJCwNfi9uIwPWWM+/jsJlB78YI5phMcROSJzGpBlMASpEISpUSjonIhlUBya99ECjXnKQmTa", - "oTZLowjRCWKckY0SMNichlRDQjfRQ7cGSqTEA5kQ5jSioWcHDo+ReY2Oj1B7Rj6WB9l+Ot5v1XfJcEyW", - "O/01jTHrauDqabn+oW2x79e7vp4pj+N0NBU8TZZ7Pn57cnKO4CViaTwmotjj/nbWH2WKTInQHSYBHeEw", - "FERK//rdy+Lc+v1+f4C3B/1+r++b5ZywkItakJrXfpBu9UOyostGILX9L4H0zcXx0fEBOuQi4QLDt0sj", - "VRC7CJ7iuopoU94VH/4/T2kULmP9WD8mYkSZVJjV4OCxfanBxSdIzQiy36GLE9SecIFCMk6nU8qmG03w", - "XTOsiCgSjrBaHg6mimwbyhlSNCZS4ThpdVoTLmL9USvEinT1m0YDCoLXDKdbNBpsmdRSs5OjWNb17pog", - "ylBMo4hKEnAWyuIYlKm93frFFAiGCME9HOqFfoxiIiWeEtTWbFPzboakwiqViEo0wTQiYaM98iGCWczf", - "+RjRkDBFJ7RM3wadungcbG3veHlHjKdkFNKpPYnK3R/Bc41iuh+FoLV/IZrQFs3WAUMKMlke7yWwbhhE", - "kAkRROP4Vw6XCD4nTFOLHu9PMG7rf23mR/SmPZ83AZinefMvndYfKUnJKOGSmhkucS77RqMRgBrBF/45", - "w6tVe13AKKmwWE0f0OIOKNHMrxFszkzTL52WwtO1n7zXbaq8E1ijHbLEBWpZ5Is5YR4hKeBM2Rdl6Lzm", - "UxRRRpBtYfdC80Q9wC8RB5Z4R3DIwL9M/Hret2Be5kFNb/pdp0VYGmtgRnxahOaMYKHGpATMmiPMdpTP", - "rhb8pyXyqZxVWJLRag5yShkjIdItLWGbliiVIKkuLR+o6Iqq0ZwI6aU5mNZvVCHborariAdXExqR0QzL", - "mZkxDkOgVxydllbikdZK4i9ONBN0HYIUIZHi6OzXg+0ne8gO4IGh5KkIzAyWV1L4Wndv2iKFxRhHkRc3", - "6tHt5mf0Mob4MeAsI4y6syfDQIeYhtO17G7q7jutJJUz8wt4t54VnH2aDWj0ivTvD55FHwKTMFpCrc7k", - "lwHfJmaz0TTiGqYLlDL6R1oSsHvoWOsKCumDgoYk7CAMLzTLxqni3SlhRGg+hSaCxyBtFYRg1Ca9aa+D", - "hlou7GopuIu3u/1+tz9slcXYaLc7TVINCqwUEXqC/+933P100P1bv/vsQ/5z1Ot++OuffAjQVDJ3UqFd", - "Z9vRfge5yRbF9epE14nyt+b+xen7OI7Z6mPNJ26604fHy4KDWWvIgysiepRvRnQssFhssillHwcRVkSq", - "8spXt71TWMA6VgCBTTWYbgiGitIDaNyO+DURgebAEdGIJzuaCVMlOwhrvRmYF9Kn5L+jADNNC0a44AIR", - "FqJrqmYIQ7sytOJFFye0S81UW51WjD++JmyqZq3B3s4Snmskb9sf3Q9/cY82/sOL6iKNiAfJ3/FUUTZF", - "8Nqc6jMqUT4Hqki8dkccdNMIxLyYsmPz2VY2EywEXnz9DruFrNppo8zVbnUQeyT/t3MiBA3dqXp4coTa", - "Eb0iFt2RSBkapv3+TgAN4CexTwIex5iF5tlGD72NqdKnWZof0sYa1Ctu9+8tEsw4yBlRxPWCMlDXCDE5", - "DANBQD/B0cpjeBWIvcA6zPpdPrR/5VJ1Y8zwlIA2aRuiseBXRE8UJTyiASUSXZGFFlIWaKo77c6ppJp8", - "CJujOTZGg96QvZ9xSUwT90orIgGhc4JiHlyhJMIBmXFQxOc4SonsoOuZlhg0MxYER/YxEiTGlA3ZTE9S", - "BjwhodYhTDNYGrokbH6JYpwAlWJBgERRjBURFEf0EwkRN5/EJKT6gBoyAniNEqxJNgi40Kev3luCg1kB", - "Cn+W6NLIG5fQ/SVlGisvDV31hqy4859bb8/fP397/uZo9Pb0xZuD49FvL/5bPzYftQa/f24Z+2YmaDwn", - "WBCB/vQZ1vvFSKchEa1B6yBVMy7oJ2Ns+dJpaRhIjV84oT2eEIZpL+Bxq9P6S/HPD18+OHlKD0XYXJOB", - "Z2JfvLKMOQo9HOXIGfMksgYiEO0wmGqBw7w6Pd/Uh2uCpVQzwdPprEwY9mS/EUmEVF6NKB+NE9+cqLxC", - "x5tvkZY7UEQ1gWZyxla/f/J8Uw5b+o8n7o+NHjoyVAvT1yyECyv+yJlGHy2EA8ocnp4jHEU8sCaQidaV", - "JnSaChL2KpY36N3HnwlTYpFw6tPBKswpb7rMo7rd/O0NWNHmmLJNqbehG9wM7oA3t9YEXrA5FZzFWhub", - "Y0H1MSvLtPLm7dGL0Ys3F62B5uNhGlij4unbd+9bg9ZOv99v+RBUY9AaHvjq9PwQdsqQjUqidDqS9JNH", - "EjjI1odiEnNhNGD7DWrPyoKCoVsEmzNs7bx6bpBr6xXglduUkEpo7XoxHZcxZvvVcx+2zBYJEXMqfWay", - "X7N3bucLx7ph92XclkTMiciQFrC4V1A/goinYbcwZKc1oYIEAmu0a3Vaf5BYy+HzTxp18rl7vvNbrxrJ", - "n2sESxwllJEVkuV3IuFdc3EVcRx2t+5YwGNE6b6Xl/jGvCjvr8UJkqFEq7NkjWDhNQ3VbBTya6an7OGr", - "9g3KGmfM9aNeCY7+9Y9/XpzkatLWq3FiOe3W9pOv5LQV3qq79ppAsoWkiX8Z54l/ERcn//rHP91Kvu0i", - "jCByK6HO7v8L0wOwbI3rYema0lgzy2D5rxlRMyIKp7dDFv3I6MPwOXK4V1hKyTxavNNcYtR8TkSEFwXG", - "a+fU2uoD96vMSlAFtGq/02z0CumP17Bh3Zs75F9VdfTtvp/ReiblmdNzzSvsudBkJtlEtrZP7M/t5SnV", - "zOiKJiOQmkd4mplsV902n13RxIri8IXZxigyjCBMQXgfc656Q/ZfM8IQ7B1sMPlIAuB5UmGFDk6PJbqm", - "UQQGHmAqy0eLFuxztmKaS6X/K1LWQeNUaWmdK4Ks3gSDpDAXaDwmKGXYXWdXZGe7wCpeWbBcEcFINDKy", - "sWwIGfMRsh/VAgeWOsFSEWG4fZqU4XX028kZah8tGI5pgH4zvZ7wMI0IOksTzQ82ytDrDFkiyFyrEGwK", - "xkZqx+UTxFPV5ZOuEoS4KcbQWWYis3et81en5/a2Xm70huwd0YAlLCQhzNmdOBKpGVYo5OzPmmJJWO62", - "OH4F6H5avokq32nNgyQt78h2dTfewH26XvucCpXiSLPKkjTovV43jhseqd/4hRS1D8u2MuTEqnwv2tTe", - "YXoGL45lmdhvtjCCTmOzRUETXzJgODXxc7PJrun/mLmJrDTb5JriV4x1Zjqpgsj23XEruwWUjjOYlGGF", - "7wY8B7KgWdeaxUMiFWUGnXRbZAU6idqXWhm3eKzV78sOuvxL6YEmXacZaPHgGhloADtg+lGx/6pNYa22", - "31ynq2wOlrffjwNZ62eE5ltICcykPhq1iJSQHvoVeDBSJE40I2JTRCWShneSEDF+/e+IG5nEfTpkemrS", - "eGlYcGQ2H0mnjLLphpbS9bmCw9AYhiapSoVuN6cyh2YZdZzxpbqA92Z2xLDTOJX6QA2iNCTo0hloLsti", - "3bL5Zlmjs/acJQXFgAQUE9DV1GacKj28XnCMVTDTcOKpMm5bdumyPIGykWjddaadS3bRdYv9P8vYRRmo", - "1lxQYfx6cfaKBax6BfNinRXPyhl+C+MVWcCWO2siXrInFg2JfnOfIJJHc2JPzaIpcoyDK3OUGM8Ja4U0", - "9kRrQtTkXyFRr3Ft3VZoeDUGf1nSX0YlsODaxeYYY4V3Y75dZFxIL86M19F6rSQAfNAcBgikqcuOUXUI", - "GBAQ08gSoZAKEqil7imbDhl4cFzaJz3b26Umci1i+IjQp6t4RbmCsmK+KW0tKuysk9qgG700HlOlSNgp", - "ywZXhCRy/aK0dGztzh7juCDXgjpGZu09YUPpirAJFwGJrYz/dXrfi0JnXi3sZl0sO1QY+BbmbPEJ4SSJ", - "KAmN947ZD7CSSrtPYCKteuyGFaXLXOCXh7zEUXSJ2rbRBhJEr0W6vWKc5cj+/vDUoUB26Xxx0tEYqbnA", - "5UypZKT/I0eaii+rndlvHYXr7vSZJNF+H9Sj3d0du6vWZmYmXOm2bB7zOiXUb80Zw4mccVV7r3VFWbgO", - "UVwnv+m2tUaxTKCRtvl928USQbppMhUYHFPv0ip269tGgGY9513jc+5zLsygGqRS8bjgYojaFccIWnah", - "KANrzqNuiBUGC2JDM6eZ7rK7brwwXRkdqs4AMpqOPd429JPmlmhKp3i8UGWz/Vbfp6l97dWvm4tvW+rc", - "3o3mR8KR4qsdf+kEubZN/PzgHBgpPppPqKfn7DjKvUaoREHFx97qo7qLbhJQq8WDbBLMjF+mAQIIexcn", - "xSuz3pB14dgcoKNsgKzbrEsMMiEOzYVFm4vCJCg4e6HxYgNhdHHSQ++z2f5ZIq1ozImLA5hhicaEMJSC", - "xRdOsa45Q4sTSCUcdqr6uTVZmJCBDbgZ5PZdD/26SEiMrflHk0KMFQ3AwWhMK+uBY8RslL2KxaxofGpk", - "LFrlLv2OTKlUouIsjdrvXh7u7Ow8q5oNt590+1vdrSfvt/qDvv7/35r7Vd99VISvr4Myb7EuW0Xuc3h+", - "fLRtbZTlcdSnXfxs/+NHrJ7t0Wv57FM8FtO/7+AHiZvws7Kj3NcMtVNJRNexSY1VPg+zgiNXjQfZrR3D", - "7snPK3dbXdXWQOK9bnkfASE+V2Pr6HrzkI0qw1zrrFxY3LIGvkhAX8yppCB5WZ/AgHq9H4+ovHouCL4K", - "+TXznNsxnhI5MueZ340glca3hXy0VgnBuZpIc11ZtlZu7T7d3d/Z293v9z1xEMsIzwM6CvQJ1GgCbw+P", - "UYQXRCD4BrXhnilE44iPy4j+ZGdv/2n/2dZ203mYm5VmcMgUJvcValuI/NXF1Lk3pUltbz/d29nZ6e/t", - "be82mpW18zaalLMJl0SSpztPd7f2t3cbQcEniL9wcSlV3/nQ5zGg9R5zx9eVCQnohAYIIluQ/gC1YzjC", - "SHZJVKbJMQ5H1ujhPzsUppFc6ahgBrMtjYEsTiNFk4iYd7AhjWzIsPIj6MnnBEIZI2KUhe3coCcbzbP2", - "Yt6tJWuCSlFZJdCdUAlSSC48URKFA0Oha/kc7GY+sQ91eGDX0BAbXmvVqRuROYmKSGCOLj3ZmAuCMjwx", - "m1ZaFWVzHNFwRFmSelGiFpQvUwGyqOkU4TFPlbndgw0rDgK+wqB7TDS7bqafvuTiaq3XpT6JRyJlTHez", - "1ppzAAbwiTWxwCmOkf3aOfYXhL7sFs7cVdr3Er0zXxjLTv44SRWiTHGtnbJwvOjASNYCxJAgUnHgpNbQ", - "Z7tpKl365RYwcjqvCzNezjsfyOWkOzG39HerYYspUSOpsForsWhMeQ/tz6B5Yydu/eFaA0gDuDNy/RBA", - "By/3rkbbrmQ4uR+Ir/IBy2wNeSM4hQUNSQ8BdYEziouqq1DameJJQsLM/tMbsjNDKtkjaW4+9IcGDmpG", - "qEBc0CktD1w2jN2nM9lNUNFh063RsfjhsoQKL8Frop7o8UQRYSDoAoaLUT92E1qdloV9q9OynKgMGvfQ", - "A5Hcw3Fpiq9Oz2/qEpYIPqGRZ7nggmDfWs3MOUu93u2fdbf+j3F81PgGIhplxm0h5iHpVWLyoX2zk+fV", - "6flp3ZyyhAioOLulNWWOJh7OkfkjOIjYyyB7m2g1GIf++mDJBsll72c+WXYicEzG6WRCxCj2GNde6vfI", - "NDAeRZShk+dleVbLzU215tPS5oDaPMGBjWdvBn2PQa6yjE4Bmh/82/WOmGO4LgpOb5WwbWwgXA+9yVJQ", - "oFen5xLlzkEeS115e2vd1E9nC0kDHJkeTVArZUUDGyBnYwn5NP/QmiI9cnLslQ0dIaD2fJqkQIZn77rH", - "by8245DMO6U5gUPPjEdEz3ujwC3mLhYu96kvMYl5naXDIIZsSkAFWGUU3BhIBXr1QEdxhaORjLjPyeK9", - "fongJWpfvDSxSnoGHZSUtlI/L0ChhN97XorRHKlu2DMYsGoyLRG4V3csZ24x5pXC8kqD+kjlV4Ijk7Cm", - "jM95WLXbeH5V3mh+tZZ6bSe+cY+dP3aDmKnDkyMjMAScKUwZESgmCtv0OAXXFBCHWp1WV59RISYxeLhN", - "/n21V0qNCb4YBFVrxD1cynZxLwbcmijtd8Z1IEQxZnRCpLJR2qWR5QxvP9kbmFwSIZnsPtnr9Xo3DQ15", - "kceCNNqKTeM5X4gS6cnZ1+3DPUSANFnL59bpwftfW4PWZirFZsQDHG3KMWWDwt/Zn/kL+GH+HFPmjRxp", - "lH6ETpbSjpSvNPWZZZ4P9EqYdeXSuMRBgV97xVSjz4BHAoSreaN0FZ5q/cRg3NeG4946YUeeNUoVEnUU", - "HTkbJO2gn1ZbQp1gBG3smClTNMrzmSzbQG+VkUauDNpfCthPCMvC9KPI/Ao4m2uq8MXslxi4e/dV9wfW", - "O2UUUg8m/5fV9oxzAwQzrae31iZOkvVo6xcUM/7XNFeJjSj2nETfnOvf5o6tPPrb6X/+8X/l6dO/b/3x", - "+uLiv+ev/vPoDf3vi+j07VcFLq0OJv+mEeF3FgQOF0ulSPCmqHSCVeARqGZcqhoI2zdIceNn2UOHoPgN", - "hqyLXlNFBI4GaNiquPYOW6hNPuJAma8QZ0h3ZQMMNvTHp8b8oz/+7HTLL9U+QhtJIOyGZAFEMh2HPMaU", - "bQzZkNm+kFuIhDt9/StEAU5UKojePS3DRgs0FjjIIwjywTvoM06SLxtDBhou+aiEXkGChcqyX7gRACns", - "rIzPgG1OQhePbTTkIcvOpSwc29hoepkRBGzzVU9JP1C86gsX5QiY/b4vcB28tfRGRlQqAg7VGWZrNMrc", - "yNB+v8Qq9vv7/bUCfoZDK9APKGE5N6VDyga0ZBAYhjaMGzzLGtjSNW8yNIJ+ff/+VINB/3uGXEc5LLIt", - "Nkqe8d2TxkaoIlnw2tto+SNC9O42XJAxksFnUYNgnRfGrfP96zOkiIido3070OCc0ECvD67/qZSpRkWK", - "0cHhyYuNXoPkmgDbbP4r9vF9tsJqUIY1mtXZAjOM1/DtoOMjcKu1FJoLcOBW85ILFBkGk9P1AJ1LUvZR", - "ha0yt/pmJ6NFbnkzJ8CwteF6TKqcYoDeZXIjzqaSOUjmyOC6zOkSurUXL8bnZ6n3ij8teDNZvciyNvDw", - "wSpz7tYnbj0rWE3+HogDzVt/7IJN82a0XTSG6sH8qJHv/b1LKzs31VFvmhehHLpYCHvNUiM0z2lwH7kB", - "lvW1j1SNai/hkX5tr9ydVnJxgmZYsj8reFnRTbZ2njZKUqlHbXp9Xby45hMzpYyqXBxkdu1qIkKvaBQZ", - "bwZJpwxH6Blqnx2/+u349esN1EVv355Ut2LVF779aZAiwaH2q9NziFLBcuRugOqdHnHuOEw+Uqnkcpho", - "o4vU1SkZfi2lTfDG3W7cYS4Fd/u8tIyHyJLwLd36vr8MDStzKnxtYgQr7N5TXoRa5urLKVDms+bx3WY4", - "uJfplGJ2fPyhKBM4n+tbpxTotKjH3/RAahZIQnR8mmcWzI1SrvvKmp5t97b29ntb/X5vq9/ERBfjYMXY", - "JweHzQfvbxtDxACPB0E4IJOvMBFaxDbCG46u8UKioROvhy0jzxcE+QLZWhG80fXrcuaG2yVqqAoU61Ix", - "3CT1QrOcCivSA5+VEwM3ltGe/O2rcgiTpiezdV2wX41uYrwmKOBpFGo5aKwpz6hVJLTanyQqz7kMxHrO", - "rhi/ZuWlGxumpt8/UiIW6OLkpGTxFmRiU8o2WDi4PNTsA09utA3ba0TltbO5ZXqDh0hpUOWahdPqzhMY", - "FE1uzoXSYGgD01suPXqvvSkzW6PxZMWaKkaTkMxHaeoTivQrFzhxfn58VEIOjPe29vv7z7r746297m7Y", - "3+rirZ297vYT3J/sBE93apK6N3d7ub0nS5ma6wOVAPBggDRxaOFA01vmijJOFcrc1DQhH2rpEhXEWBOW", - "AzaBY0YVZD6kbKq7ARXdSrkmLtIkZ6SMKgjEhywulOklgy1Ed2KdjwboFbSFVziGcCE3Ca3blM0AOFwY", - "M6hmDG7oBP5aPeWzWaq02AXfyFmqkP4Llq3BYLWN1V0YHjNAbzh8I5yPKONVtcU0B9+r5eZVFadtvYKc", - "9ygMZhnmAL3MmGTGZi1bbUtifxrebR2bwWl7o+Q6Z3e8pbEl37mCV1inZSDa6rQcoMB7bNmPzM7LGyJR", - "REXf/QDBEbDQ3E8nVTSyuQVgJVQqGhilD8Pm1lGyTYNFwpE5wetu+4zzhz3ls48co7g4QW2IRvwrsjqh", - "/msjuxksUuXu9rPdZ3tPt5/tNYo5yCe4nsEfgmvS8uTWcvsgSUeuXkbN0g9Pz+Hs0+eqTGOj5Nu1F1w8", - "E8EDLWxShvICHPngz3rPiqEWIU/HUcFoZOOywJ+/SbWUmuutP2g0p5MJ++NTcLX9d0HjrY97cnvs1c2y", - "gfyC7HHR0Lmk9ZFx16Qu9HvDA0IJWRsw8o5IWAE6IwoB/nQRDuCQzjyKLMq5sBILcS9i7e7s7Ow/fbLd", - "CK/s7AqEMwL1c3mWJ3YGBRKDlqj97uwMbRYQzvTp3CwhLQOzApyfzpDNYtwveWBq1WfHhyU18lKONbbv", - "eVwL8gsrBNlFWaCDY1QmIC1RuRfaOzv9p7tP9p80I2OrcI3Ex9UcxqXCMOCx2UOKO98G4/j7g1OkexcT", - "HJQVjK3tnd0ne0/3bzQrdaNZQeYbk7HiBhPbf7r3ZHdne6tZ5JPPAG5j+koEW+ZdHqLzIIVnNzygWGa9", - "nbrTwid4LntjrnQAzT1Kq+6DN/EXzmO+qYReacFVFbW1XFaUcQtxyxtNzBx+FqnHqavCpSXQpq68qz13", - "T7GaHbMJ9yT0uYG+af2hnOU70XKQhJojIWGUhI53ZYqnFa3AwyqSBIUpsZAzopLAFuDY3PJA4h7mZDLK", - "pmXf8qUBm2iBZg6rI/xhXNuwicFK+v1y3osUYGVMzBLh3EOnkb2cypFfUVnuWJBpGmGBqu7qK6YsF3FE", - "2VWT3uUiHvOIBkh/ULUmTHgU8euRfiV/gbVsNFqd/mCUXzBXrANmcta9wGxIZdx8Cb/oVW5UnJvg5N80", - "329CmcUm9j/vrdNLrTsZj+5zRj8WEL0cAru73a/ze6vptOTxthwNcFPeblHWR/HOUf8gy1Trud0090cV", - "pbgsB5fW61stXFCu8vJblgRQ25kUXYhxGa6FUN9GB3GzO9Kq8dzNZlOSoDz67v6Tp3sNY62/StReUYju", - "KwTrebxCoK7ZqZMmUtv+k/1nz3Z2nzzbvpF85O5Zavan7q6luD+VhNQVme1JH/53o0mZmxb/lGpuW8oT", - "KiWXvvWEvqwg3TzGpkbrXlUENt9Jp+aXBfBmIu4KaemgJHIV6ie0yWRCAkXnZGTg1s0nU/HNajSHACc4", - "oGrh0QDxtcnTmTWpxIo06L0yWQ9Ibd823E9zLpmOc3eAthsc/cVodhVc2G+cskGm4zot8m11VKND2txs", - "FQtFAwNBng22eid/nQETXWNZulTQvwNItJfXx6jePpkWzQv5OVzPavnl9+q+eCd/3b7i9le2s6B1lITk", - "KsRXHaH1JKglgsZ5fD0nsq800Hqfjgp/sAfg7b4ajYvJVFZmqyllXslP3ZuP26yyx/J35gS7+XgFB4Kb", - "fFjNKwH4aOdgQZ733SmhRA02KS7WpwG8h+hwY9K+VXy4tYY/SIi4fXwvYeFL23FW8IJq7vPnvvKXYy7d", - "Y+51+zvd/t77rZ3Bk73B1tZ9BChkdxh1ptynn7aun0bbeLIb7S+e/rE1ezrdjne8Xh/3kH6yUkShko3S", - "riEhopoRpJpJR5KIMtKV2fXH+ovoFaFHxiiX4AUIeSs0spuoAa6w6QqqPSsvski8WOXAqSa0fwj/NDv7", - "lbpMdfrHR6unfav7hOpE/AhWnQrgU7PJQMDc1p1mGgW7KlCPF5A1i/KhTOk+voTEH1ZwsN8s4dZxKusW", - "bmeYZ6dwxOTuF0tYk79eApSPxa5OoFE5hMwdaTFfSeaSerfZM97bLa1zYC4kddl+slfO6nLQ/ZvJ4oJG", - "vcHmL3/9390Pf/mTP5NXSXWURHRDMgGJ+Yosuia1uEauXjn61RTskwrb/FeK4Bi4XXBFDHeN8cfifJ/0", - "M1v24g2Ol5YAqkZMWfb32gX5i+ouIZpxNqlLzRprCvEkzKKmUr4NvUeFxqhN4kQtXHips6lv3Mz55SDr", - "sKbC5p067vef3UWY4fnKuMIfMDFw0TfJTWitV9LS/tcG8/iNckdVH2Fj+bbJDss+rZUUblJ16212MU+Z", - "GoHpedm+pt8Zs7YNpJum1YwCmzFTmzZsdznak+AQcpCvvMjIqcw56nTho/X2+ZW3hoWVFWZSvzfGMW05", - "Wm4FgE41aK5nRJDCRsAHeezhDUFmjczrI9OMY46WRLvVbJgmYYugYLW2ADKA1SDILiKWbztW+9ae4I/Z", - "CCCBYrkkLcM6CmVOXz2HJEzvXFZEOnFdwDSqheier8eiJvUYljejiFXL6zbtvYRnedUK7ldHWxXkzMco", - "oeYyPmo2R4JUULU402zIRgBANZeD1KAh8CdYBDzOB4foTCieS+3NaeXmVmtjNEAHp8e2lguDsxxdnKCI", - "TkiwCCJig+uWHNpAn357eNw1UcFZjS89PFUAEJdV++D0GJL0CmnG7fe2e1AyFcoDJbQ1aO30tiBlsQYD", - "LHETkjnAT3vdpukQTr3j0J7Oz00T/ZXAMVFQce13z7WVIsIkh5DgsICnBcEmwVRYySaJ4DLNaGRUfwv+", - "xI7BD8wp0TEAx00dc6VaWNMiSd7abf2g0UEmnEmzodv9vsncyZQ9DnCevHXz79LcfeXjNpIyADwe59ol", - "kc9JOhbkXzqt3f7WjeazNt+qb9hzhm1RIALTfHJDINxq0GNm7jtczVliG+Z0BihUpLDfTaHqNI6xWDhw", - "5bBKuKwT0YhEGDI+mswkf+fjHrKqH4QDyhlPoxDqDSYmp71moxgpLHrTTwiLYEbnZMjs6WFy52IB4dIx", - "0qeGsVuVScMMbXbfsB0i1XMeLirQzbrb1N2BtFUGcDXsSJIRuF+P6tIOZXaThDIG6Uvzqs8u/8YSRzf5", - "pqFkuq/UFcNM5emLTaLpKwLuZRP60dthIz9JzfBgWwjUNcjC77c3/De0EE3md244yt4hC97yIad1BFug", - "K5ME3KUBFmMcRd6sS9OIj3Fk83FfEY/g9ApaWKAUA+/ckct4SEwQVbJQM87M73ScMpWa32PBryUR+mC2", - "wdQW1q4ckUFdKIxAYwhoNqla9JibZoqbn6/I4ktvyA7C2KXhsWU6cSS5TVSeFazKCs8OWW24X43d5NAW", - "LjFJgot5Vc00eaqSVPWQWQhRNgIcmkPaXTkj4ZApjj4LU2Vh8WXzcz7iF5CoCQ41nhSamCVtfqbhl7pZ", - "yxHWqx9BU49OQgAAw5Y+XYYt/XsqsJaoUwmV/YmEOpHT4pa2DWFzAdLKRhXCAWYo4Ulqa8ERZPOvl/qA", - "bBo4ipACUnLfahkIdrJmPfay3Zca0t60m6vRChlBksgCMfV39/30JEkgiE/t/s+zt28QHFV6D0yzPMIT", - "YGTKEmalUfXovSF7gYOZrRcGzv/DFg2HrUzmDTdgrqm0VwHdLghev+ip/WKG6dDwl15Pd2VkugH6/bPp", - "ZaBpKYlHil8RNmx96aDCiylVs3ScvfvgB2jdheVZiRGgtuH9Gy4XEhT4yo9Bc25gFiJueW20QBjlHKio", - "3Y8pw2JlIicP6C0EtYKJp7IIjM9DMPEMW4OhM/IMW51hi7A5PLOWoGHrix8CNvFYvae5yWVlm+VItNfv", - "b6z3JLLw9YjQpYaa/L4sSV/bdyZ4WKFrWfAwi3NhMnoHTVYyI249gOTzHGe1GX+KeGtEPKtPF4Q3+L54", - "Dhj0jYixHVckMK2AR04CW6mdGLSAODHQOJzfn1E4qJPgcuQtqh9VJXNZrdito7IAphg5/Nt9APyDcfPM", - "/jDus4caF0emBpXLc/240BE2yyFix68RvyLqe8C4/kOxUleA5Bvi72PBn1fEyn050CrcbBOKwBfNLdXY", - "Z0FwLG0vprHWVc9gTt0zwhR6AU979l+n8UCo6GXEp5cDZEAY8SmKKCPS+mRkdxj6ULSwhI9MqsbsO5vt", - "NJhhNiUStc35+a9//BMmRdn0X//4p5amzS8g903j7w+RkJczgoUaE6wuB+g3QpIujuicuMVArBKZE7FA", - "O31p68rqV57cqXLIhuwdUalgMvP01+sCmJgObSkPvR7KUiKRBBBCcbqJdUH/JS8666dlA8oHpejOks5l", - "V1BYgD4VHQ6ATyE14aBW/2r5rWdmzSX7WdWCu2TTX89fFPmoDPZ2zQRvyGAAxD66gxd20ah9dvZio4dA", - "xzBYAWEGIDHn3VjhufeTJ63nSYajlBkKQNnwpkLa/Fr775Ft08wAbHv8kSzAdXUA6k3AxuRBBAkdvH7q", - "Ck3MwX64OdOwzz575MoG1htob7/e4hDOT7ORInx3++xwbxnmtn5mDrJvoQKjti1nlqWwLBXp/FZI/yCn", - "RqG2a3Z0IG4SZz6YWnbI2SSigUJdNxfIkhGTTFUrI8hjYQfv7KwRduuqBvQWz7fNUnxK7UmXharkR979", - "nx6VQW9yjORBxzmu/TxJ1qHOEZUB198WsKUb4MQm8DTiS0anRSxaZ5A6gufZkbNSXDrKqj5bgnw405Qd", - "OmXVs+EBmOJRhSF+Q0ZYSUpYCNN/TNh8nu2iK5G8wnL1faFm/+GkoIe2YvnQ/DGZscIK2DQXnGWlqurQ", - "yxazuseNtiN4Fn5GhKNqM1GT4C5flvkUBTMSXJkF2UreqySCY1fsu4nqa/r7kTRfU0XsBhKLBflPEaWB", - "spvDapWCe2wzNd6ffgsj3Ei9vbt7XotgHiCDs8nYWaxNEkQsFyzY+KGueh/kNKtWC39ElHSaRpG78ZgT", - "ofJaasUzYPMzuCWtl+0dta08Ds7fve4SFnDwQ8t8qPxClCtxdLcSvtkws5SfaNJEJwRQOcSoF6C/Yv+N", - "uyDK8uX/2/ZLmzH/37Zfmpz5/7ZzYLLmb9wbsvQfijU/tMT9iJFPC9y0DDRgTaYQ0ToJNWvVUEh17X8o", - "OdUWtbuJpJrB9aew2kRYLYJrpbya1Re8R4nVlmL7NlcyGbL5oA2vnH/iDyapPqyVz2JkoWp/6drDppzk", - "Ii9/Zmt+Pz4HSpphXPHYaGiuzgly5fHhUPf4qGMr25l6dFmAyAMZr908Hly4teM+vOX6IB7TacpTWYw9", - "gUKGRNpgpYiUGfBjE7vz47lW8P6OsbT/kEfHg8vVP/H+niT+6oYa5m1uoNbJ/K5VU5nftoeSgaYahYld", - "e+eqXNg0Khs1ToWuDkxTNC6VLFp2dvTNy6eLoHOtqOTqAgINYjBk/6H1j98VwfGHX1yQTNrvb+/Bc8Lm", - "H35xcTLsxKEKYUpQIhEWBB28OYJrvylEr0MytDwkrzoPk+LM1Ia2ZUv/xylI+c1ncw3JYeFPDamRhlQA", - "12oNKauicp8qkhnkm+lIDt98ALepNX5qSQ+hJcl0MqEBJUzlGYCXnMRsAvFHGFvG7P1QwbmjdNA21pLy", - "0karBdA87d2DO/Zkgz+8cuQy7D1OH3luomJCp47kh2G9PvK94UP/YZnzw+shjxnFjMBfBd0yI9qc2ATE", - "fgHhJRdXTTHPk4fzzhHw7qWT4gq/Q9lET48Uqhx+QxEFDm/jW6+Rpiy5PABBLiVX/ZYunQ4SVrk1QZGU", - "TfM6l1TNeGqyqozsQ5OVTVOFrSYDIk9ge/3W7EWP/gAC6BuuEI2TiMQEsrZ1DTZBcdE0SbjI6o9RWUhF", - "fDP2p8mm6GBrktvYKsAdZBM2g7HObVgb7PbL2+XlmhGfrg+qzQZ3EaSeqNohO5cmyculEYUvUcZkkeJI", - "kogECl3PaDCDCFv9DPo3Abg4SS6zlBobrlhqMbMIDN6WRFAcQZVHHpl6pZfzOL4cLGeAuzg5gY9McK3J", - "9XY5QC7rW3ZASN2qGDGrVxFhqdAbGwfc1pgkeBSZHb3Up1BhfRs2ljZPeTJkvrhaRq5th3SCLgshtpc1", - "MbaOob7mU/mt5KVOfaIqsxbFkQDAGdwkLGzVGXZo5I+u3er3fflTGkb6mmncc6Dv0mRe82mWJKuEyjhJ", - "mqKvnSZg8TyOV+AwaheSmUsV8lT9VaqQCAEfW+yuQ27UxoH5Q+ErjajMliJz6eAB/bzmS5O1xgsqzVQL", - "+aTNX/M4bnVadj6e6rlfHzFd7XDZzKZ3phAW/VPSvknAc5nZFyKeKyeHrVtRL3Lbchw/vL7nyl1/YzT8", - "BvaxfBaUOVEF9javI/64IidNpZaqLGaS5/toJCv1Uk8lZaPyWZ6m/3+gimrWWq3P88BKagZin2ZWKm/x", - "zbXTrNrGTw0101C5QGFqhqvUu/lh1c6MoaCUlTRPK57eVvfMksxlYIY6hGzlhUDO8zY/u5/HtxAXvhNO", - "2Kmt+lKXzihf9PfAcmtqojXiud9ITrLHakFA+IYs2FVne2gOnEFFq3sZl/su2LAhuIwbF3kOVN6nrvDi", - "T2ZcMgMaS+ltmbETPpdsgQX2TFk3iXAdX7Zyai0DtlWgfnh9LddVfnCNLeBCGNcxcEZ7TKGLhTvDgurZ", - "TnAqSScjmI67t744OdmoIxqhVpKM+D4utG8nOVTKcsahvy6yoKFLUn94cmRT2lOJRMp66G1MIXP8FSEJ", - "pKSkPJUIfAB7xXpjdVXQsoJihCmxSDhlau0s8qb3M5kvt0rS/cB8ygZv//BmJVto97ExKeAd+vS2C1it", - "VClTZs97TeeurSgzmfW18IHHPNW9L9VDQxMaEbmQisTmzm6SRkBEkN7DZn+13xnftQ6iSkL18A74+iRE", - "xFRKypkcsjGZaKkkIUKPDQUnaUQK1w++m60zhTOueWpY3/dxtQUl0uA2B6s6qJWro+EkcdXRfNcnWUG3", - "W0/pJdxVIbmIxzyiAYoou5KoHdErI4OjuUSR/rGx8rJrBN/ddW7b21OWhvQxm3Bv+j+Dsxky/wgc7rjC", - "1txl/qNja69IkVgc/4GN9rM1uZavCYIjKAKaudmiVNGIfjKsTndCpaKBqZmEM9hBuRczXm/ITogSug0W", - "BAU8ikignK1hMxE82Bym/f5OkFCIh9ghMDlgePWvYxjx8PQc2pmSNJ0h039Ax+8PThHVMJ1gqzIXJmoL", - "26Pjzbdrrv/PAEz/g/Uxs8BVZOHf8J83uzf3oaylIVlDojxZpQDx5Ic3GFgJ7qe14HFaC8CJPVtNeypw", - "AEKxnKUq5NfMbxkwFVLl5mfz43hdKITCwezClYr+PqRdWy123TBugY+CKO2aQmLSk34Te70t6PtI0zlp", - "wLklgBBTDOrwnwKmUPiPht13f1lXhON3eFNnIepS/343tPXQJ5+dg4vwK8LjsZC5wTS3EihZWbQ+ZeGM", - "a3WzIBWCMAWpYHLRMsAJDqhadBCOXDVVWx4psyHlheDHguArfdL2huxdFkhpyzNp7arjVCsUUnllerDa", - "Uw+9nRMh03E2OQSMyeh5AHxbUDXAUWAqkZLJhASKzokpESprtK9sKveZljcfxLPR7qUF3WNTOfw4AbuX", - "o4XVOkqecrXpG86yVs3SN2S9FrxhCp4iK32eR66hqYJ/E5OdZ/ArWusWb1/dzHvtN/1Rw7HLXlL+SdhX", - "X7nKHyUr3lnBOaVp0occwx9b/oXCzEukWnLwWh8I3tij6z49rNYFgmeDP3Qg+JnXyeeRpaPCJbetugjw", - "7w8R+g/rXfzQEeCPG7e0KCGXQFfPiRpEgn8XGHg/IeDf2Lv+FiHg35W/J4Twfju/++/K09N6LGaenj+D", - "vO/TwdNEekNAa52Dp+F61vK8UlG6sG2aqUm2xx9JgrfGyhvI7w7sP1O2NVAZCsByp3CF3QDvlxbhSZyo", - "hbNG8Qn43eQ5BSX9BN57vsC5zOh8f/Fqt7DH3h16ODyttcb+TPX2YAbfPB/28dHjz+9WpLnSwbKpT50u", - "FsGMzkvxWqso2IIoEaSb8ATsrKEBmIWHO8sUFr3pJ2S77w3Z+xlxfyHqsmWQEIVUkEBFC0SZ4sARzBh/", - "lkhwrQnAey4WPvNtkXJfCh4f2NWsOQ8tTVljWO7mFy+6IVa4O3fcZoUJ7SuurE7wRxqnMTA8RBl69Ry1", - "yUclTPIGNNGaD6KTDKTkY0BIKAEnN4oT3urXWDbpJzKajpvMckUajrc2zQkKUql47Pb++Ai1cap4d0qY", - "3gst6k9Akk0En9PQ5MjNgTrnkYHqVg1Ab2p31UKF9QfPlQszuW8iwzQ5kKafaFJmC8btsTVojSnDMLm1", - "CS/KNGU8cPV4mIIfXE47DnNaP4+wapVtjYlayXFAVJyjSEv0Gz+Pucd8zBU9GdyZVjrtmmUxbebc0NDn", - "4D4ymGaOLw9rtr74fu7jC1WJH6HpfJ4ppHVm8+8LBfsPdz48tLn84hH7b70iTvkumMqhA92jD2Fe8wBH", - "KCRzEvEk1mKladvqtFIRtQatmVLJYHMz0u1mXKrBfn+/3/ry4cv/DwAA//+gy9hgdyoBAA==", + "H4sIAAAAAAAC/+x9a3PbOpbgX0Fxe6rtbkmWH3EcTd2adewk13PjxBvHmZ2+ysoQCUlokwAvAMpRUvna", + "P6B/Yv+SLRwAfAmUaCd24km6um5kEsTj4OC8cB6fgpAnKWeEKRkMPgUynJEEw89DpXA4e8fjLCFvyB8Z", + "kUo/TgVPiVCUQKOEZ0yNUqxm+q+IyFDQVFHOgkFwhtUMXc+IIGgOvSA541kcoTFB8B2Jgk5APuAkjUkw", + "CLYSprYirHDQCdQi1Y+kEpRNg8+dQBAccRYvzDATnMUqGExwLEmnNuyp7hphifQnXfgm72/MeUwwCz5D", + "j39kVJAoGPxeXsb7vDEf/52ESg9+OMc0xuOYHJM5DckyGMJMCMLUKBJ0TsQyKI7M+3iBxjxjETLt0AbL", + "4hjRCWKckc0KMNicRlRDQjfRQwcDJTLigUwEcxrRyLMDRyfIvEYnx2hjRj5UB9l5PD4ImrtkOCHLnf6a", + "JZh1NXD1tFz/0Lbc98s9X8+UJ0k2mgqepcs9n7w+Pb1A8BKxLBkTUe7xYCfvjzJFpkToDtOQjnAUCSKl", + "f/3uZXlu/X6/P8A7g36/1/fNck5YxEUjSM1rP0i3+xFZ0WUrkNr+l0D66t3J8ckhOuIi5QLDt0sj1RC7", + "DJ7yuspoU90VH/4/zWgcLWP9WD8mYkSZVJg14OCJfanBxSdIzQiy36F3p2hjwgWKyDibTimbbrbBd02w", + "YqJINMJqeTiYKrJtKGdI0YRIhZM06AQTLhL9URBhRbr6TasBBcFrhtMtWg22fNQys5OjRDb17pogylBC", + "45hKEnIWyfIYlKn9vebFlA4MEYJ7KNQz/RglREo8JWhDk01NuxmSCqtMIirRBNOYRK32yIcIZjF/52NE", + "I8IUndDq+Tbo1MXjcHtn10s7Ejwlo4hOLSeqdn8MzzWK6X4Ugtb+heiDtmi3DhhSkMnyeM+BdMMggkyI", + "IBrHv3C4VPA5Yfq06PH+BOMG/2urYNFblj9vATDPiuafO8EfGcnIKOWSmhkuUS77RqMRgBrBF/45w6tV", + "e13CKKmwWH0+oMVXOIlmfq1gc26afu4ECk/XfvJWt6nTTiCNdsgKFWgkkc/mhHmEpJAzZV9UofOST1FM", + "GUG2hd0LTRP1AL/EHEjiV4JDDv7lw6/nfQviZR409KbfdQLCskQDM+bTMjRnBAs1JhVgNrAw21Exu0bw", + "n1WOT41XYUlGqynIGWWMREi3tAfbtESZBEl1aflwiq6oGs2JkN4zB9P6jSpkWzR2FfPwakJjMpphOTMz", + "xlEE5xXHZ5WVeKS1iviLU00EXYcgRUikODr/9XDn0T6yA3hgKHkmQjOD5ZWUvtbdm7ZIYTHGcezFjWZ0", + "uzmPXsYQPwac5wejiffkGOgQ01C6wO6m7r4TpJmcmV9Au/WsgPdpMqDRK9a/33sWfQREwmgJjTqTXwZ8", + "nZrNRtOYa5guUMboH1lFwO6hE60rKKQZBY1I1EEYXmiSjTPFu1PCiNB0Ck0ET0DaKgnBaIP0pr0OGmq5", + "sKul4C7e6fb73f4wqIqx8V53mmYaFFgpIvQE/9/vuPvxsPu3fvfJ++LnqNd9/9c/+RCgrWTupEK7zg13", + "9jvITbYsrtcnuk6UvzX1L0/fR3HMVp9oOnHTnT46WRYczFojHl4R0aN8K6ZjgcVii00p+zCIsSJSVVe+", + "uu1XhQWsYwUQ2FSD6YZgqCk9gMYbMb8mItQUOCYa8WRHE2GqZAdhrTcD8UKaS/47CjHTZ8EIF1wgwiJ0", + "TdUMYWhXhVay6OKUdqmZatAJEvzhJWFTNQsG+7tLeK6RfMP+6L7/i3u0+R9eVBdZTDxI/oZnirIpgteG", + "q8+oRMUcqCLJ2h1x0M1iEPMSyk7MZ9v5TLAQePHlO+wWsmqnjTLXuNVh4pH8X8+JEDRyXPXo9BhtxPSK", + "WHRHImNomPX7uyE0gJ/EPgl5kmAWmWebPfQ6oUpzs6xg0sYa1Ctv9+8BCWcc5Iw45npBOagbhJgChqEg", + "oJ/geCUbXgViL7CO8n6XmfavXKpughmeEtAmbUM0FvyK6ImilMc0pESiK7LQQsoCTXWn3TmVVB8fwuZo", + "jo3RoDdkb2dcEtPEvdKKSEjonKCEh1cojXFIZhwU8TmOMyI76HqmJQZNjAXBsX2MBEkwZUM205OUIU9J", + "pHUI0wyWhi4Jm1+iBKdwSrEgcERRghURFMf0I4kQN58kJKKaQQ0ZAbxGKdZHNgy50NxX7y3B4awEhT9L", + "dGnkjUvo/pIyjZWX5lz1hqy885+C1xdvn76+eHU8en327NXhyei3Z/+tH5uPgsHvnwJj38wFjacECyLQ", + "nz7Bej8b6TQiIhgEh5macUE/GmPL506gYSA1fuGU9nhKGKa9kCdBJ/hL+c/3n987eUoPRdhcHwPPxD57", + "ZRnDCj0U5dgZ8ySyBiIQ7TCYaoHCvDi72NLMNcVSqpng2XRWPRiWs9/oSERUXo0oH41T35yovEInW6+R", + "ljtQTPUBzeWM7X7/9OmWHAb6j0fuj80eOjanFqavSQgXVvyRM40+WggHlDk6u0A4jnloTSATrStN6DQT", + "JOrVLG/Qu48+E6bEIuXUp4PViFPRdJlGdbvF2xuQoq0xZVtSb0M3vBncAW9urQk8Y3MqOEu0NjbHgmo2", + "K6tn5dXr42ejZ6/eBQNNx6MstEbFs9dv3gaDYLff7wc+BNUYtIYGvji7OIKdMsdGpXE2HUn60SMJHObr", + "QwlJuDAasP0GbcyqgoI5twg2ZxjsvnhqkGv7BeCV25SISmjtejEdVzFm58VTH7bMFikRcyp9ZrJf83du", + "50ts3ZD7Km5LIuZE5EgLWNwrqR9hzLOoWxqyE0yoIKHAGu2CTvAHSbQcPv+oUaeYu+c7v/Wqlfy5RrDE", + "cUoZWSFZficS3jUXVzHHUXf7Kwt4jCjd9/ISX5kX1f21OEFylAg6S9YIFl3TSM1GEb9mesoeumrfoLxx", + "Tlw/6JXg+F//+Oe700JN2n4xTi2l3d559IWUtkZbdddeE0i+kCz1L+Mi9S/i3em//vFPt5JvuwgjiNxK", + "qLP7/8z0ACRb43pUuaY01swqWP5rRtSMiBL3dsiiHxl9GD5HDvdKS6mYR8t3mkuEms+JiPGiRHjtnILt", + "PlC/2qwEVXBW7XeajF4h/fEaMqx7c0z+RV1H3+n7Ca1nUp45PdW0wvKFNjPJJ7K9c2p/7ixPqWFGVzQd", + "gdQ8wtPcZLvqtvn8iqZWFIcvzDbGsSEEUQbC+5hz1Ruy/5oRhmDvYIPJBxICzZMKK3R4diLRNY1jMPAA", + "UVlmLVqwL8iKaS6V/q/IWAeNM6Wlda4IsnoTDJLBXKDxmKCMYXedXZOd7QLreGXBckUEI/HIyMayJWTM", + "R8h+1AgcWOoES0WEofZZWoXX8W+n52jjeMFwQkP0m+n1lEdZTNB5lmp6sFmFXmfIUkHmWoVgUzA2Ujsu", + "nyCeqS6fdJUgxE0xgc5yE5m9a52/OLuwt/Vyszdkb4gGLGERiWDOjuNIpGZYoYizP+sTS6Jqt+Xxa0D3", + "n+VOIBlO5YyrEWh+i3XU6dw2PzOtb2QL6ATzMM2qW7pT385XcCGvgTenQmU41rS2Ik567+eN54dHbTCO", + "JWX1xdK9HLuxql6stjWYmJ7BDWRZqPbbPYyk1NruUVLllywgTs/81G6ya/o/YW4iK+0+har5BWOdm07q", + "ILJ9d9zKbgGlkxwmVVjhrwOeQ1lSzRvt6hGRijKDTrotshKhRBuXWpu3eKz198sOuvxL5YE++0610PLF", + "NTLQAHrC9KNy/3WjxFpzQXulsLY5WN5+Pw5lo6MSmm8jJTCTmrdqGSslPfQrEHGkSJJqSsamiEokDfEl", + "EWL8+t8RN0KN+3TI9NSkcfOw4MiNRpJOGWXTTS3ma8aEo8hYliaZyoRuN6eygGYVdZz1pr6At2Z2xNDj", + "JJOaI4dxFhF06Sw8l1W5cNn+s6wSWoPQkoZjQAKaDSh7aivJlB5eLzjBKpxpOPFMGb8vu3RZnUDVyrTu", + "PtTOJb8pu8X+n+fkogpUa2+oEX69OHtHA2bBkn2yyQxoBRW/ifKKLGDLnTkSLxkky5ZIv71QEMnjObFs", + "t2zLHOPwyrAS43phzZjGIGltkPr4146o1zq3bis0vFqDv6oqLKMSmIDtYguMsdK/sf8uciqkF2fG62jF", + "WBIAPqgeAwTi2GXH6EoELBCIaWSJUUQFCdVS95RNhwxcQC7tk57t7VIfci2j+A6hT9nxyoIlbcd8U9la", + "VNpZJ/ZBN3ppPKFKkahTlQ2uCEnl+kVp8doarj3WdUGuBXWEzBqMopbiGWETLkKSWCXhyxTHZ6XOvGrc", + "zbpY9sgw8C3N2eITwmkaUxIZ9x+zH2BmlXafwMZad/mNalqb8QCoDnmJ4/gSbdhGm0gQvRbp9opxViD7", + "26MzhwL5rfW7047GSE0FLmdKpSP9HznSp/iy3pn91p1w3Z3mSRId9EG/2tvbtbtqjW5mwrVuq/Y1r1dD", + "89Y48bv5Yown+iw6N5E2ovxR8UlhSb2iLGrbwW+6baN1LheMnKZx1wa6VJBulk4FBg/Zr2meu/W1J0Cz", + "mYKvcX73eTnmUA0zqXhS8nVEGzUPDVr15agCa87jboQVBlNmS3urme6y33CyMF0ZXazJEjOajj1uP/Sj", + "prpoSqd4vFDV+4Ptvk/j+9I7aDcX37Y0+d8bDZJEI8VXeyDTCXJt2zgcAj8ZKT6aT6in55ytFe4rVKKw", + "5uxv9VrdRTcNqTUngIwTzoyDqAECCI3vTst3d70h6wL7HaDjfIC827xLDLIljszNyQYXpUlQ8DpD48Um", + "wujdaQ+9zWf7Z4m0wjInLiBhhiUaE8JQBqZn4IZdw4vLE8gkME1V/9zaTkzswiZcUXL7rod+XaQkwdYO", + "pY9CghUNwdNpTGvrAXZkNsreCWNWtoK1slqt8tt+Q6ZUKlHz2kYbb54f7e7uPqnbL3cedfvb3e1Hb7f7", + "g77+/9/aO3h//fAMX1+HVdpifcfK1Ofo4uR4xxpLq+Ooj3v4ycGHD1g92afX8snHZCymf9/F9xLA4Sdl", + "x4XTG9rIJBFdRyY1Vvlc3UoeZQ2ubLf2ULsjh7PCf3ZVWwOJt7rlXUSm+HyercftzWNH6gRzrdd0aXHL", + "mvwiBb2zOCUlCc46J4bU64Z5TOXVU0HwVcSvmYdvJ3hK5MjwM78/QyaNkw35YK0bgnM1kebetGr13N57", + "vHewu7930O97AjKWEZ6HdBRqDtRqAq+PTlCMF0Qg+AZtwIVXhMYxH1cR/dHu/sHj/pPtnbbzMFc87eCQ", + "K17uK7RhIfJXF9zn3lQmtbPzeH93d7e/v7+z12pW1l7calLOtlwRSR7vPt7bPtjZawUFn0D/zAXI1AX4", + "yOe6oPUnc9nYlSkJ6YSGCEJskP4AbSTAwkh+W1U9k2McjazxxM87FKaxXOkxYQazLY2hLcliRdOYmHew", + "Ia1s0bDyY+jJ541CGSNilMcP3aAnG1a01kPArSVvgirhYRXQnVIJUkghPFESRwNzQtfSOdjNYmLvm/DA", + "rqElNrzUqlM3JnMSl5HAsC492YQLgnI8MZtWWRVlcxzTaERZmnlRohGUzzMBsqjpFOExz5S5ZoQNKw8C", + "Tsuge0w0uW6n5z7n4mqt+6fmxCORMaa7WWsVOgRD+sSaaoCLY2S/dhEGJaEvvw40l6b2vURvzBfGQlQ8", + "TjOFKFNca6csGi86MJK1JDEkiFQcKKk1GNpu2kqXfrkFjKXO/cOMV9DOe/J96U6Mu8DX1bDFlKiRVFit", + "lVg0pryF9ufQvLU3uf5wrSGlBdwZub4PoIO7fVejbVcynN4NxFc5o+W2hqIRcGFBI9JDcLrAK8aF99VO", + "2rniaUqi3P7TG7Jzc1TyR9LcoOgPDRzUjFCBuKBTWh24amC7S6+2m6Ciw6Zbo2P5w2UJFV6C+0bzoccT", + "RYSBoItcLocf2U0IOoGFfdAJLCWqgsY99ECkcLVcmuKLs4ub+qalgk9o7Fku+ELYt1Yzc15bL/f6593t", + "/2M8MDW+gYhGmfGfSHhEerXkANC+Hed5cXZx1jSnPDMDKs9uaU25x4uHcuR+DQ4i9lLJ3kpaDcahv2Ys", + "+SCF7P3EJ8tOBE7IOJtMiBglHuPac/0emQbGtYkydPq0Ks9qubmt1nxW2RxQmyc4tIH17aDvMcjVltEp", + "QfO9f7veEMOGm8Lx9FYJ28ZG5PXQqzwXBnpxdiFR4aXksdRVt7fRX/5stpA0xLHp0UTXUlY2sAFytpaQ", + "z4oPrSnSIycnXtnQHQS0MZ+mGRzD8zfdk9fvtpKIzDuVOYFn0YzHRM97s0Qt5i4or3DurxCJeZOlwyCG", + "bHuASrDKT3BrIJXOqwc6iiscj2TMfc4ab/VLBC/RxrvnJmhKz6CD0spW6uclKFTwe997YjRFahr2HAas", + "m0wrB9yrO1ZTyBjzSml5lUF9R+VXgmOTOaeKz0V8t9t4flXdaH619vTaTnzjnjjH8BbBW0enx0ZgCDlT", + "mDIiUEIUtnl6Si4uIA4FnaCreVSESQKudpN/X+3d0mCCL0djNRpxj5bSbtyJAbchXPyNcUGIUIIZnRCp", + "bLh4ZWQ5wzuP9gcmqUVEJnuP9nu93k1jVJ4VQSmttmLLuPCXwlV6cvZl+3AHoSht1vIpODt8+2swCLYy", + "KbZiHuJ4S44pG5T+zv8sXsAP8+eYMm8IS6s8KHSylP+keqWpeZZ5PtArYdYlTOMSBwV+7RVTgz4Dng0Q", + "N+cNF1Z4qvUTg3FfGhd868whRfoqVcoYUnYIbZE9hH5cbQl1ghG0sWNmTNG4SKyybAO9VWocuTJ7wFLm", + "gJSwPF9AHJtfIWdzfSp8yQMqBNy9+6L7A+vlMoqoB5P/y2p7xkkCoqrWn7dgC6fperT1C4o5/WubNMWG", + "Nns40Ten+re5Y6uO/nr6n3/8X3n2+O/bf7x89+6/5y/+8/gV/e938dnrL4qgWh3V/k1D079aNDpcLFVC", + "0tui0ilWoUegmnGpGiBs3yDFjb9mDx2B4jcYsi56SRUROB6gYVBzER4GaIN8wKEyXyHOkO7KRjps6o/P", + "jPlHf/zJ6Zaf631ENqRB2A3JI5lkNo54ginbHLIhs30htxAJd/r6V4RCnKpMEL17WoaNF2gscFiEMhSD", + "d9AnnKafN4cMNFzyQQm9ghQLlafhcCMAUthZGZ8B25xELjDcaMhDlvOlPC7c2Gh6uREEbPN1j0s/ULzq", + "CxfVUJyDvi+CHry+9EbGVCoCjtk5Zms0yt3R0EG/QioO+gf9tQJ+jkMr0A9OwnKSTIeULc6SQWAY2hBu", + "8FBrYUvXtMmcEfTr27dnGgz633PkOipgkW+xUfKMD6A0NkIVy5L332bgM32b3W25IGMkg8/iFlFDz4x7", + "6NuX50gRkTiH/Y1Qg3NCQ70+uP6nUmYaFSlGh0enzzZ7LbJ8Amzz+a/Yx7f5CuvBHdZo1mQLzDFew7eD", + "To7BPdee0EKAA7ea51yg2BCY4lwP0IUkVV9X2Cpzq292Ml4UljfDAYbBpusxrVOKAXqTy404n0ruaFkg", + "g+uyOJfQrb14MT4/S73X/HLBm8nqRZa0gYcPVrmTuOa4zaRg9fH3QBzOvPXrLtk0b3a2y8ZQPZgfNYq9", + "v3NpZfemOupNEzRUYyhL8bd5job2yRXuIknBsr72gapR4yU80q/tlbvTSt6dohmW7M8KXtZ0k+3dx62y", + "ZepR215fly+u+cRMKT9VLiAzv3Y1oalXNI6NN4OkU4Zj9ARtnJ+8+O3k5ctN1EWvX5/Wt2LVF779aZGr", + "waH2i7MLiHbBcuRugJqdHnHhOEw+UKnkcrxqq4vU1bkhfq3kb/AGAG9+xaQO7vZ5aRn3ka7hW7r1fX+p", + "IlYmd/jSDA1W2L2jBA2NxNWX3KBKZ83jr5tq4U6mU4n98dGHskzgfK5vndugE1CPv+mh1CSQROjkrEhx", + "WBilXPe1NT3Z6W3vH/S2+/3edr+NiS7B4YqxTw+P2g/e3zGGiAEeD8JoQCZfYCK0iG2ENxxf44VEQyde", + "DwMjz5cE+dKxtSJ4q+vX5RQSt8sYURco1uWEuEkOiHbJHb40on5VouPzaorj1kLeo799UTZk0pa1W98H", + "+9XoJtZvgkKexZEWpMb66Bq9jERWfZREFdmj4bRfsCvGr1l16cYIqgnAHxkRC/Tu9LRiMhdkYpPjtlg4", + "+Ew07ANPb7QNO2tk7bWzuWWehfvIrVAnuyV299UzKZRtds4H02BoC9tdIX56780pM1uj8WTFmmpWl4jM", + "R1nmk6r0Kxd5cXFxclxBDoz3tw/6B0+6B+Pt/e5e1N/u4u3d/e7OI9yf7IaPdxvS07f3m7m9K0z1NDdH", + "OgHgwYJpAtmigT5vuS/LOFMo93PTB/lIi6eoJAebuB4wKpwwqiCHI2VT3Q3o+FZMNgGaJs0kZVRBRgDI", + "R0OZXjIYU3Qn1ntpgF5AW3iFE4g3cpPQylHVjoCjhbGjasLghk7hr9VTPp9lSstt8I2cZQrpv2DZGgxW", + "XVndhaExA/SKwzfCOZkyXtd7THNw3lpuXteRNqxbkXM/hcEswRyg5zmRzMmsJasbktifhnZbz2jw+t6s", + "+N7ZHQ80thQ7V3Ir6wQGokEncIAC97NlRzQ7L2+MRRkVfRcMBMdAQgtHn0zR2CY5gJVQqWhotEYMm9t0", + "km1CLxKNjAjQdF1ovEesmJB/5AjFu1O0AeGMf0VWqdR/beZXi+VTubfzZO/J/uOdJ/utghaKCa4n8Efg", + "27Q8ubXUPkyzkav80bD0o7ML4H2ar8osMVYCu/aSj2gqeKilVcpQUUqkGPxJ70k5ViPi2TguWZ1sYBcE", + "BLSp+9JwP/YHjed0MmF/fAyvdv4uaLL9YV/ujL3KXT6QXxI+KVtKl9RGMu6aJIx+d3pAKCEbI07eEAkr", + "QOdEIcCfLsIhMOncJcminItLsRD3Itbe7u7uweNHO63wys6udHBGoL8uz/LUzqB0xKAl2nhzfo62Sghn", + "+nR+mpAfglkBzn/OkM3H3K+4cGrdadeHJQ3yUoE1tu950gjyd1YIsouyQAfPqlxAWjrlXmjv7vYf7z06", + "eNTuGFuNbSQ+rKYwLieHAY9NY1Le+Q2wrr89PEO6dzHBYVVD2d7Z3Xu0//jgRrNSN5oVpOAxqTNuMLGD", + "x/uP9nZ3ttuFTvks6DYosHJgq7TLc+g8SOHZDQ8olklvp4lb+ATPZXfOlR6khUtq3f/wJg7HRdA4ldAr", + "Lfm6og0tl5Vl3FLg82YbO4mfROpxmuqJaQm0rS/watffM6xmJ2zCPZmFbqBvWocqZzpPtRwEWTBQRBgl", + "kaNdueJpRStw0YolQVFGLOSMqCSwBTg210SQQYg5mYyyadU5fWnANlqgmcPqFAEwrm3YxuIl/Y49b0UG", + "sDI2aolw4eLTyuBO5civqCx3LMg0i7FAdX/3FVOWiySm7KpN73KRjHlMQ6Q/qFsTJjyO+fVIv5K/wFo2", + "W61OfzAqbqhr1gEzOeufYDakNm6xhF/0Kjdr3lHA+bfM91tQMLKNAdF7bfVc607GJfyC0Q8lRK/G0O7t", + "9Jsc5xo6rbjMLYcT3JS2W5T1nXjn6X+Y59z1XI+aC6iaUlyVgyvr9a0WbjhXuQkuSwJow9kkXYxyFa6l", + "WOFWjLjdJWvd+u5msyVJWB197+DR4/2WwdpfJGqvKKn3BYL1PFkhUDfs1Gkbqe3g0cGTJ7t7j57s3Eg+", + "chc1DfvTdFlT3p9aau2azPaoD/+70aTMVY1/Sg3XNdUJVdJk33pCn1cc3SJIp0HrXlXOtthJp+ZXBfB2", + "Iu4KaemwInKVKkFskMmEhIrOycjArVtMpubc1WoOIU5xSNXCowHia5MwNG9SCzZp0Xttsh6Q2r5tvKCm", + "XDIbF/4EG25w9Bej2dVw4aB1zgeZjZu0yNf1UY0OaZPE1SwULQwERVra+qX+dQ5MdI1l5VJB/w4h419R", + "6aN+fWVatC9J6HA9r0pYXMz7Aqb8FQjL21/bzpLWURGS6xBfxUKbj6CWCFonFPZwZF+Ro/VOITX6YBng", + "7b4ajcvZWFamu6mkbim47s3HbVejZPk7w8FuPl7JA+EmH9YTUwA+2jlYkBd9dyoo0YBNiov1+QjvILzc", + "mLRvFWBureH3EmNuH99JXPnSdpyX3Kj8CSFJNFoVAnOUN3MmrxQvQG5oFPIf7+71+7s7/VvFwHytPJWl", + "fpoum0vfWUXdGQIAnco95FfLy8lMrgU15Q4cmKQSBCcDuKhJcUhQTCbgIZonkWpVYrwy9OrJ2xAx61OU", + "3/i4jXI1jawObdGXcQZMxfVjw4fcMgIXoVZ1Iyu/v1GhcrdZ/lrllavx/W5/t9vff7u9O3i0P9jevoug", + "mRxITbcDjz9uXz+Od/BkLz5YPP5je/Z4upPsej2R7iAlaq3CSC1Dql1DSkQ9S009u5MkMWWkK/MbtfW+", + "DStogbHzrj3/N9MsXdXfFYzgvLrIMj/AqgBOvVjDffhM2tmvVI/r0z85Xj3tW11R1SfiR7D6VACf2k0G", + "gji3vzRiMGMt+c5FqWFrzrPy2nQd7/H5k8DR9u5yA8R9+FwhjJUT9n4Fx17mah7xfMoFVbNkNXvIm+Xx", + "R1D15KNU0bKPnj0z+mXQCeKPe9UzYZ+398a08TQ5gtmtLLP5FnZdSGm2epUm65lW0mFhkFG7TJTWWxnt", + "ElZtym+W1DfxNhvcYtGmyLHjyK9zcqjAtHi9BNaaD+AdZfVeJTi2T0BUk8GNi0g531Pu0v91sw/ZL9eX", + "Br4zWL21BLApBKWUlmvn0X41L9dh928mDxca9QZbv/z1f3ff/+VP/lyMFfIoiehGZAImiyuy6JoiE5oU", + "96r5C0ztV6mwzWCoCE5ANgiviJFFEvyhPN9H/fwycfEKJ0tLAFtPQln+99oF+euzL4HRePs1JddO9A55", + "Uh5SqbTka5OnoFJjtEGSVC1cggB3qbl5M+/Dw7zDhmLNXzX0qv/kawSKX6yMDP8BU7uXnUPdhNa6hS7t", + "f2M4pv9W5Lge5WGuHm262mpUQi0Jp1Td5kuThGdMjeDub/mCQ78z94o2FHqa1XPCbCVMbdnEC8vx+gRH", + "wDtX3iQXp8x5SnbrDLfhgnSl20ZpZaWZNO+N8QxejndeAaAzDZrrGRGktBHwQRE9fkOQ2Vu+9bHFxjNS", + "623dej5jk3JLULg2tAAygNUgyG+ClyWj1dERp/hDPgJIzVgu6ZawjlLF7BdPIY3eG5fXlk5cFzCNek3T", + "p+uxqE1lnuXNKGPV8rpNe+/Bs7RqBfVrOls15CzGqKDmMj5qMkfCTFC1ONdkyMZwQV2vw8ygIdAnWAQ8", + "LgaH+Hqow06t60rNdYYwImiIDs9ObFUvBrwcvTtFMZ2QcBHGxIZHL3kUg0Hz9dFJ1+R1yMtF6uGpAoC4", + "ugiHZyeQZl0YCSno93Z6UH0bCsWlNBgEu71tSDqvwQBL3IJ0PPDT+jvocwhc7ySy3PmpaaK/EjghCop3", + "/u7xG1BEmPQ+EjzG8LQk2KSYCivZpDF4MxhVgepvIaDDEfiB4RIdA3DcVjGVamHvdkj62m7re40OMuVM", + "mg3d6feNIMmUZQe4SL+99XdpBMti3FZSBoDHE92wJPI5SceC/HMn2Otv32g+azNm+4a9YNiWhyMwzUc3", + "BMKtBj1h5sLZlS8ntmFxzgCFyifs9/d6v2SWJFgsHLgKWKVcNoloRCIMOXtNbqm/83EPWUMJBHTLGc/i", + "CErXpqYqiSajGCksetOPCItwRudkyCz3MNnPsYCEFwnSXMNcHFSPhhna7L4hO0Sqpzxa1KCbd7eluwNp", + "qwrgeuCoJCOIfxk1JY7LrYwpZQwSUEtio2zzDErLtl2oGCBD7i2VQBhmqkhAb0oFXBHw753QD94OWzmq", + "a4IH20KgMk2eQGVn0+8iA/HAfu+y4/wdsuCtMjmtI9hSjbkk4G5tsRjjOPbmzZvGfIxjW1HhingEpxfQ", + "wgKlHDrtWC7jETFhsOlCzTgzv7NxxlRmfo8Fv5ZEaMZs02FYWLvCdAZ1obQNTSAlhUm2pcfcMlPc+nRF", + "Fp97Q3YYJS6Rmq34jGPJbamJvHRhXsN8yBoDthusjEe29JRJ817OjG2myTOVZqqHzEKIsjk8oDkkTpcz", + "Eg2Z4uiTMHVyFp+3PhUjfgaJmuBI40mpiVnS1icafW6atRxhvfoRNPXoJAQAMAw0dxkG+vdUYC1RZ3KG", + "cAhhC/pheUs3zMHmAqSVzTqEQ8xQytPMVgUlyFbQqPQB+ZBwHCMFR8l9q2Ug2MmG9VhvJ19yX+vqZHxT", + "ascI0vyWDlN/78B/niQJBfGp3f95/voVAlal98A0K2L0AUamQG1eZVuP3huyZzic2cqREH01DGg0DHKZ", + "N9qEuWbS3sV2uyB4/aKn9osZpkOjX3o93ZWR6Qbo90+ml4E+S2kyUvyKsGHwuYNKL6ZUzbJx/u69H6BN", + "HiPnFUKANgzt33TZ7KDUY8EGDd/ALELc0tp4gTAqKFBZux9ThsXKVHwe0FsIagUTT2UZGJ+GYOIZBoOh", + "M/IMg84wIGwOz6wlaBh89kPApo5sDvUx2QhtswKJ9vv9zfWunBa+HhG60lAfv89L0tfOVxM8rNC1LHiY", + "xbk4Rb2DJq+kEbfuQfJ5ivMqvT9FvDUintWnS8IbfF/mAwZ9Y2Ks1zUJTCvgsZPAVmonBi0gUBc0Dud4", + "bRQO6iS4AnnL6kddyVxWK/aaTlkIU4wd/u3dA/7BuEVtFhj3yX2Ni2NTRdBVKnhY6Aib5RCx49eIXxD1", + "PWBc/75IqSsh9Q3x96Hgzwti5b4CaDVqtkXm7hbEH14CrkfS9mIaa131HObUPSdMoWfwtGf/dRoPxOpf", + "xnx6OUAGhDGfopgyIq1TXH6HoZmihSV8ZLyP8u+sM1I4w2xKJNow/PNf//gnTIqy6b/+8U8tTZtfcNy3", + "TMAVhKJfzggWakywuhyg3whJuzimc+IWA8GiZE7EAu32pa0wrl95sl/LIRuyN0Rlgsk81EqvC2BiOrTF", + "mPR6KMuItN5bUF50YmOAfinKj/vPsgHlvZ7ozvJVuFlBaQGaKzocAKduauLxrf4V+K1nZs0V+1ndgrtk", + "019PXxT5oAz2ds0Eb0hgAMS+cwcv7KLRxvn5s80eAh3DYAXEeYHEXHRjhefeT5q0niYZilIlKABlQ5tK", + "hU8a7b/Htk07A7Dt8UeyADdVcmk2ARuTBxEkcvD6qSu0MQf74eZMwz777LHz2W020N5+veUhnNtKK0X4", + "6+2zw71lmNsKyAXIvoUKjDZsQco8CXGlzPK3Qvp74Rql6tw560DcpD6+N7XsiLNJTEOFum4ukKYoIbmq", + "VkWQh0IO3thZI+zWVc+oUOZvW5UAwUZOl8cKFizv7rlHbdCbsJEi60OBaz85yTrUOaYy5PrbErZ0Q5za", + "FMxGfMnPaRmL1hmkjuF5znJWikvHed1+eyDvzzRlh85YnTfcA1E8rhHEb0gIa2llS3lSHhI2X+S76OKT", + "Vliuvi/U7N+fFHTfViwfmj8kM1ZUA5umgrO82GATetlyhHe40XYEz8LPiXCn2kzUZBgtlmU+ReGMhFdm", + "QXAhvVr3PTFN2qm+pr8fSfM1dSBvILFYkP8UUVoouwWsVim4JzZV7t3ptzDCjdTbr3fPaxHMA2RwNhk7", + "i7XJQovlgoWbP9RV771wMwPsB8nMzrI4djcecyJUUQ2zzAO2PoFb0nrZ3p22lezg4s3LLmEhBz+03IfK", + "L0S5InVfV8I3G2aW8hNN2uiEACqHGM0C9Bfsv3EXRHnFk3/beW5rnvzbznNT9eTfdg9N3ZPNO0OW/n2R", + "5vuWuB8w8mmBm1aBBqTJlJJbJ6HmrVoKqa79DyWn2rKkN5FUc7j+FFbbCKtlcK2UV/MKsXcosdpimt/m", + "SiZHNh+04ZXzT/zBJNX7tfJZjHQpoaisXnvYnL9cFAUsKUOZJA/QgZLmGFdmGy3N1cWBXMk+HOqeHHds", + "bVJTUTQPELkn47Wbx70Lt3bc+7dcHyZjOs14JsuxJ1CKlkgbrBSTKgF+aGJ3wZ4bBe/vGEv798k67l2u", + "/on3dyTx1zfUEG9zA7VO5net2sr8tj0UfTXlgEzs2htXZsgmctlscCp0hbjaonGlZtyys6NvXj5dBF1o", + "RaVQFxBoEIMh+w+tf/yuCE7e/+KCZLJ+f2cfnhM2f/+Li5Nhpw5VCFOCEomwIOjw1TFc+00heh2yURYh", + "efV5mByTprq/LTz9P05BKm4+22tIDgt/akitNKQSuFZrSHkZq7tUkarZlu5dR3L45gO4Ta3xU0u6Dy1J", + "ZpMJDSlhqkjBvuQkZis4PMDYMmbvh0rOHRVG21pLKmrLrRZAiySR9+7Ykw9+/8qRy0f5MH3kuYmKiZw6", + "UjDDZn3ke8OH/v0S5/vXQx4yihmBvw66ZUK0NbEZ4P0CwnMurtpinidr7VdHwK8vnZRX+B3KJnp6pFRm", + "9huKKMC8jW+9Rpqq5HIPB3IpFfG3dOl0kLDKrQmKpGxaFBqmasYzk1VlZB+arGz6VNhyXiDyhLbXb01e", + "9Oj3IIC+4grRJI1JQiBrW9dgE1R3ztKUi7wAJJWlxN03I3/62JQdbE1yG1uGvYNsenMw1rkN2wC7/fJ2", + "ealmzKfrg2rzwV0EqSeqdsgupEnycmlE4UuUE1mkOJIkJqFC1zMaziDCVj8z9QIg+BWn6WWeUmPTVasu", + "ZxaBwTckERTHUGaXx6Zg9OU8SS4Hyxng3p2ewkcmuNbkerscIJf1LWcQUrcqR8zqVcRYKvTKxgFvaEwS", + "PI7Njl5qLlRa36aNpS1SngyZL66WkWvbIZ2gy1KI7WVDjK0jqC/5VH4reanTnKjKrEVxJABwBjcJi4Im", + "ww6N/dG12/2+L39Ky0hfM407DvRdmsxLPs2TZFVQGadpW/S10wQsnifJChxGG6XU/1JFPFN/lSoiQsDH", + "FrubkBtt4ND8ofCVRlRma0G64gmAfl7zpcla4wWVJqqljNbmr3mSBJ3AzsdTvvzLI6brHS6b2fTOlMKi", + "f0raNwl4rhL7UsRzjXPYwkHNIreth/TD63sWUN/apvAN7GPFLChzogrsLS8y4j+oyElTKqsui5n0/b4z", + "ktfaaj4lVaPyeVEo4H+gimrWWi+Qds9Kag5in2ZWKQbzzbXTvHTITw0111C5QFFmhqtVh/ph1c6coKCM", + "VTRPK57eVvfMk8zlYIZCsGzlhUBB87Y+uZ8ntxAXvhNK2GmseNOUzqhY9PdAchuKUraiud9ITrJstSQg", + "fEMS7Mpj3jcFzqGi1b2cyn0XZNgcuJwal2mOEphJ6irf/iTGFTOgsZTelhg74XPJFlgiz5R10xg30WUr", + "pzYS4Fo1qW+qr319QthQK+uzpYTfkvAVytG9EbuTnLwZgmfrC/7gKmrIhTC+cuB995BiNUuXpCVdeyPF", + "mSSdnEJ03EX9u9PTzSYqIdRKGiHUA6YQtbp4SeSvxC9o5LLyH50e2xz+VCKRsR56nVBIlX9FSAo5OCnP", + "JAKnx165wFpT2be8ghphSixSTplaO4ui6d1M5vOtspLfM5200eo/vB3NlnZ/aEQKaIcWV+wCVmuRytQV", + "9N5Luns6ykwpASj1OuaZ7n2pABxU/5YLqUhiLiknmSmWCvlMbLpb+51x1usgqiTS56EDzk0pEQmFUphy", + "yMZkosWwlAg9NtT4pDEp3bf4rvLOFc6p5pkhfd/HXR7UhIPrK6yaoFYtB4fT1JWD890X5RXsbj2l53A5", + "h+QiGfOYhiim7EqijZheGaUDzSWK9Y/Nlbd7I/juayfzvf3J0pA+YRPuzXdocDZH5h+Bwp3UyJrzXnhw", + "ZO0FKR8WR39go/1kTa6la4LgGKqe5n7FKFM0ph8NqdOdUKloaIpE4Rx2UN/GjNcbslOihG6DBUEhj2MS", + "Kmdc2UoFD7eGWb+/G6YUAkB2CUwOCF7z6wRGPDq7gHamBk9nyPQf0PHbwzNENUwn2NoIShNlRF1zcYVO", + "tl6v8Xc4BzD9D74wNAtcdSz8G/7zKvvmTqONZ0g2HFGerlKAePrD32hbCe6nteBhWgvAaz9fzcZU4BCE", + "YjnLVMSvmd8yYErCyq1P5sfJutgPhcPZO1cb+/uQdm153HXDuAU+iENp1xQRk4/1m1xQ2ArGDzR/lQac", + "WwIIMeUoFj8XMJXRfzTs/vpG+TIcv8OrSQtRl+v4uzlb98357BxcSGMZHg/lmBtMcyuBGp1l61Mev7lW", + "NwszIQhTkPumEC1DnOKQqkUH4diVj7X1oHIbUlH5fiwIvtKctjdkb/LIUVuPSmtXHadaoYjKK9OD1Z56", + "6PWcCJmN88khIExGzwPg2wqyIY5DU3qVTCYkVHROTE1U2aB95VO5yzzExSCejXYvLegemsrhxwnYvQIt", + "rNZRcQ1szFdxnrdql68i77Xk/lNyjVnp5D1yDU3Z/5uY7DyDX9HGOAD76mbuer/pj1qOXXUL80/CvvrC", + "Vf4oaQDPS944bbNcFBj+0BJOlGZeOaoVj7b1ke+tXdju0qVsXeR7Pvh9R76fe72aHlj+LVzxU2sKef/+", + "EKF/v+7U9x3y/rBxS4sScgl0zZSoRej7d4GBdxPz/o3DCW4R8/5dObhCzPK3CzT4rlxbrYtm7tr6M6r9", + "Lj1aTWg7RPA2ebQaqmctzysVpXe2TTs1yfb4I0nw1lh5A/ndgf1njroWKkMJWI4L18gN0H5pEZ4kqVo4", + "axSfgN9NkURR0o/gveeLFMyNzncXoHcLe+zXQw+Hp43W2J+57e7N4FskAD85fvgJ7cpnrsJYtjTX6WIR", + "zui8EqC26gRbEKWCdFOegp01MgCz8HC8TGHRm35EtvvekL2dEfcXoi49CIlQRAUJVbxAlCkOFMGM8WeJ", + "BNeaALznYuEz35ZP7nPBk0O7mjX80J4pawwr3PySRTfCCnfnjtqsMKF9wZXVKf5AkywBgocoQy+eog3y", + "QQmTrQJNtOaD6CQHKfkQEhJJwMnN8oS3+w2WTfqRjKbjNrNckXfktc3rgsJMKp64vT85Rhs4U7w7JUzv", + "hRb1JyDJpoLPaWSSAhdAnfPYQHW7AaA3tbtqocL6gxfKhZncN5Fh2jCk6UeaVsmCcXsMBsGYMgyTW5vh", + "o3qmjAeuHg9T8IMrzo7DnOAnC6uXFdeYqJUcB0TFOYq1RL/5k809ZDZX9mRwPK3C7dqlbW3n3NDS5+Au", + "Urbmji/3a7Z+9/3cx5fKMD9A0/k8V0ibzObfFwr2748/3Le5/N0D9t96QZzyXTKVQwe6Rx/CvOQhjlFE", + "5iTmaaLFStM26ASZiINBMFMqHWxtxbrdjEs1OOgf9IPP7z///wAAAP//XjwQ6bMxAQA=", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/providers/providers.go b/lib/providers/providers.go index 815f9c3b..55b527e5 100644 --- a/lib/providers/providers.go +++ b/lib/providers/providers.go @@ -25,6 +25,7 @@ import ( "github.com/kernel/hypeman/lib/paths" "github.com/kernel/hypeman/lib/registry" "github.com/kernel/hypeman/lib/resources" + "github.com/kernel/hypeman/lib/snapshot" "github.com/kernel/hypeman/lib/system" "github.com/kernel/hypeman/lib/vm_metrics" "github.com/kernel/hypeman/lib/volumes" @@ -126,13 +127,21 @@ func ProvideInstanceManager(p *paths.Paths, cfg *config.Config, imageManager ima meter := otel.GetMeterProvider().Meter("hypeman") tracer := otel.GetTracerProvider().Tracer("hypeman") defaultHypervisor := hypervisor.Type(cfg.Hypervisor.Default) + level := cfg.Snapshot.CompressionDefault.Level + snapshotDefaults := instances.SnapshotPolicy{ + Compression: &snapshot.SnapshotCompressionConfig{ + Enabled: cfg.Snapshot.CompressionDefault.Enabled, + Algorithm: snapshot.SnapshotCompressionAlgorithm(strings.ToLower(cfg.Snapshot.CompressionDefault.Algorithm)), + Level: &level, + }, + } memoryPolicy := guestmemory.Policy{ Enabled: cfg.Hypervisor.Memory.Enabled, KernelPageInitMode: guestmemory.KernelPageInitMode(cfg.Hypervisor.Memory.KernelPageInitMode), ReclaimEnabled: cfg.Hypervisor.Memory.ReclaimEnabled, VZBalloonRequired: cfg.Hypervisor.Memory.VZBalloonRequired, } - return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, meter, tracer, memoryPolicy), nil + return instances.NewManager(p, imageManager, systemManager, networkManager, deviceManager, volumeManager, limits, defaultHypervisor, snapshotDefaults, meter, tracer, memoryPolicy), nil } // ProvideVolumeManager provides the volume manager diff --git a/lib/snapshot/README.md b/lib/snapshot/README.md index edcdc989..c537d42b 100644 --- a/lib/snapshot/README.md +++ b/lib/snapshot/README.md @@ -31,6 +31,65 @@ Snapshots are immutable point-in-time captures of a VM that can later be: - `Stopped` snapshot from `Stopped`: - snapshot payload is copied directly +## Snapshot Compression + +Snapshot memory compression is optional and is **off by default**. + +The current exception is cloud-hypervisor standby snapshots: if no request, instance, +or server policy is set, Hypeman defaults that path to `lz4` so standby snapshots +store compressed guest memory by default on Linux. + +- Compression applies only to `Standby` snapshots, because only standby snapshots contain guest memory state. +- `Stopped` snapshots cannot use compression because they do not include resumable RAM state. +- Compression affects only the memory snapshot file, not the entire snapshot directory. + +### When compression runs + +- A standby operation can request compression explicitly. +- A snapshot create request can request compression explicitly. +- If the request does not specify compression, Hypeman falls back to: + - the instance's `snapshot_policy` + - then the server's global `snapshot.compression_default` +- Effective precedence is: + - request override + - instance default + - server default + - then the cloud-hypervisor standby fallback (`lz4`) when no other policy is set + +Compression runs **asynchronously after the snapshot is already durable on disk**. + +- This keeps the standby path fast. +- Standby can return successfully while compression is still running in the background. +- That means a later restore can arrive before compression has finished. +- While compression is running, the snapshot remains valid and reports `compression_state=compressing`. +- Once finished, the snapshot reports `compression_state=compressed` and exposes compressed/uncompressed size metadata. +- If compression fails, the snapshot reports `compression_state=error` and keeps the error message for inspection. + +### Restore behavior with compressed snapshots + +Restore prefers correctness first and resume latency second. + +- If a restore starts while compression is still running, Hypeman will not let restore race the compressor. +- For cloud-hypervisor standby restores, Hypeman waits for the in-flight compression job to finish. +- For the other hypervisors, Hypeman cancels the in-flight compression job and waits briefly for it to stop. +- If the snapshot memory has already been compressed, Hypeman expands it back to the raw memory file before asking the hypervisor to restore. +- This means compression is never allowed to race with restore. + +In practice, the tradeoff is: + +- standby stays fast because compression is not on the hot path +- restore from a compressed standby snapshot pays decompression cost +- restore from an uncompressed standby snapshot avoids that cost entirely + +### Supported algorithms + +- `zstd` + - default when compression is enabled + - supports configurable levels +- `lz4` + - optimized for lower decompression overhead + - does not currently accept a level setting + ### Restore (in-place) - Restore always applies to the original source VM. - Source VM must not be `Running`. diff --git a/lib/snapshot/types.go b/lib/snapshot/types.go index 0c19596d..53db7d88 100644 --- a/lib/snapshot/types.go +++ b/lib/snapshot/types.go @@ -19,15 +19,42 @@ const ( // Snapshot is a centrally stored immutable snapshot resource. type Snapshot struct { - Id string `json:"id"` - Name string `json:"name"` - Kind SnapshotKind `json:"kind"` - Tags tags.Tags `json:"tags,omitempty"` - SourceInstanceID string `json:"source_instance_id"` - SourceName string `json:"source_instance_name"` - SourceHypervisor hypervisor.Type - CreatedAt time.Time `json:"created_at"` - SizeBytes int64 `json:"size_bytes"` + Id string `json:"id"` + Name string `json:"name"` + Kind SnapshotKind `json:"kind"` + Tags tags.Tags `json:"tags,omitempty"` + SourceInstanceID string `json:"source_instance_id"` + SourceName string `json:"source_instance_name"` + SourceHypervisor hypervisor.Type + CreatedAt time.Time `json:"created_at"` + SizeBytes int64 `json:"size_bytes"` + CompressionState string `json:"compression_state,omitempty"` + CompressionError string `json:"compression_error,omitempty"` + Compression *SnapshotCompressionConfig `json:"compression,omitempty"` + CompressedSizeBytes *int64 `json:"compressed_size_bytes,omitempty"` + UncompressedSizeBytes *int64 `json:"uncompressed_size_bytes,omitempty"` +} + +// SnapshotCompressionAlgorithm defines supported compression algorithms. +type SnapshotCompressionAlgorithm string + +const ( + SnapshotCompressionAlgorithmZstd SnapshotCompressionAlgorithm = "zstd" + SnapshotCompressionAlgorithmLz4 SnapshotCompressionAlgorithm = "lz4" +) + +const ( + SnapshotCompressionStateNone = "none" + SnapshotCompressionStateCompressing = "compressing" + SnapshotCompressionStateCompressed = "compressed" + SnapshotCompressionStateError = "error" +) + +// SnapshotCompressionConfig defines requested or effective compression config. +type SnapshotCompressionConfig struct { + Enabled bool `json:"enabled"` + Algorithm SnapshotCompressionAlgorithm `json:"algorithm,omitempty"` + Level *int `json:"level,omitempty"` } // ListSnapshotsFilter contains optional filters for listing snapshots. diff --git a/openapi.yaml b/openapi.yaml index 1f0b5e1a..1e78106a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -305,6 +305,8 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor to use for this instance. Defaults to server configuration. example: cloud-hypervisor + snapshot_policy: + $ref: "#/components/schemas/SnapshotPolicy" skip_kernel_headers: type: boolean description: | @@ -418,6 +420,54 @@ components: format: int64 description: Total payload size in bytes example: 104857600 + compression_state: + type: string + enum: [none, compressing, compressed, error] + description: Compression status of the snapshot payload memory file + example: compressed + compression_error: + type: string + description: Compression error message when compression_state is error + nullable: true + example: "write compressed stream: no space left on device" + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" + compressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Compressed memory payload size in bytes + example: 73400320 + uncompressed_size_bytes: + type: integer + format: int64 + nullable: true + description: Uncompressed memory payload size in bytes + example: 4294967296 + + SnapshotCompressionConfig: + type: object + required: [enabled] + properties: + enabled: + type: boolean + description: Enable snapshot memory compression + example: true + algorithm: + type: string + enum: [zstd, lz4] + description: Compression algorithm (defaults to zstd when enabled) + example: zstd + level: + type: integer + description: Compression level for zstd only + example: 1 + + SnapshotPolicy: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" CreateSnapshotRequest: type: object @@ -433,6 +483,14 @@ components: example: pre-upgrade tags: $ref: "#/components/schemas/Tags" + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" + + StandbyInstanceRequest: + type: object + properties: + compression: + $ref: "#/components/schemas/SnapshotCompressionConfig" RestoreSnapshotRequest: type: object @@ -599,6 +657,8 @@ components: enum: [cloud-hypervisor, firecracker, qemu, vz] description: Hypervisor running this instance example: cloud-hypervisor + snapshot_policy: + $ref: "#/components/schemas/SnapshotPolicy" PathInfo: type: object @@ -1732,6 +1792,12 @@ paths: schema: type: string description: Instance ID or name + requestBody: + required: false + content: + application/json: + schema: + $ref: "#/components/schemas/StandbyInstanceRequest" responses: 200: description: Instance in standby @@ -1739,6 +1805,12 @@ paths: application/json: schema: $ref: "#/components/schemas/Instance" + 400: + description: Invalid request payload + content: + application/json: + schema: + $ref: "#/components/schemas/Error" 404: description: Instance not found content: diff --git a/skills/hypeman-remote-linux-tests/SKILL.md b/skills/hypeman-remote-linux-tests/SKILL.md new file mode 100644 index 00000000..cb2d7c5a --- /dev/null +++ b/skills/hypeman-remote-linux-tests/SKILL.md @@ -0,0 +1,107 @@ +--- +name: hypeman-remote-linux-tests +description: Run Hypeman tests on a remote Linux host that supports the Linux hypervisors. Use when validating cloud-hypervisor, Firecracker, QEMU, embedded guest artifacts, or Linux-only integration behavior on a server over SSH. +--- + +# Hypeman Remote Linux Tests + +Use this skill to run Hypeman tests on a remote Linux server in a way that is repeatable and low-friction for any developer. + +## Quick Start + +1. Pick a short remote working directory under the remote user's home directory. +2. Ensure the remote test shell includes `/usr/sbin` and `/sbin` on `PATH`. +3. Bootstrap embedded artifacts and bundled binaries before running `go test`. +4. Run the smallest relevant test target first. +5. Treat host-environment failures separately from product failures. + +## Remote Workspace + +Prefer a short path such as `~/hm`, `~/hypeman-test`, or another short directory name. + +Cloud Hypervisor uses UNIX sockets with strict path-length limits. Long paths from default test temp directories or deeply nested clones can make the VMM fail before the actual test starts. + +If a new test creates temporary directories for guest data, prefer a short root like: + +```bash +mktemp -d /tmp/hmcmp-XXXXXX +``` + +## Remote Bootstrap + +Before running Linux hypervisor tests, make sure the remote clone has the generated guest binaries and embedded VMM assets: + +```bash +export PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH +make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded +``` + +Why: + +- `go:embed` patterns in the repo require the bundled binaries to exist at build time. +- `mkfs.ext4` is commonly installed in `/usr/sbin` or `/sbin`, not always on the default non-login shell `PATH`. +- Running `go test` before this bootstrap often fails during package setup, not during test execution. + +QEMU is usually provided by the host OS, so there is no matching bundled `ensure-qemu-binaries` target. If QEMU tests fail before boot, verify the system `qemu-system-*` binary separately. + +## Test Execution Pattern + +Start with one focused test: + +```bash +go test ./lib/instances -run TestCloudHypervisorStandbyRestoreCompressionScenarios -count=1 -v +``` + +Then run the sibling hypervisor tests one at a time: + +```bash +go test ./lib/instances -run TestFirecrackerStandbyRestoreCompressionScenarios -count=1 -v +go test ./lib/instances -run TestQEMUStandbyRestoreCompressionScenarios -count=1 -v +``` + +For unit-only validation: + +```bash +go test ./lib/instances -run 'TestNormalizeCompressionConfig|TestResolveSnapshotCompressionPolicyPrecedence' -count=1 +``` + +Run one hypervisor at a time when the tests boot real VMs. This avoids noisy failures from competing KVM guests and makes logs much easier to interpret. + +## Syncing Local Changes + +If the main development is happening locally and the Linux host is only for execution, sync only the files you changed rather than recloning every time. + +Typical pattern: + +```bash +rsync -az lib/instances/... remote-host:~/hypeman-test/lib/instances/ +``` + +After syncing, re-run the bootstrap command if the changes affect embedded binaries, generated assets, or files referenced by `go:embed`. + +## Common Host Failures + +Classify failures quickly: + +- Missing embedded assets: + - symptom: package setup fails with `pattern ... no matching files found` + - fix: run `make ensure-ch-binaries ensure-firecracker-binaries ensure-caddy-binaries build-embedded` +- `mkfs.ext4` not found: + - symptom: overlay or disk creation fails with `exec: "mkfs.ext4": executable file not found` + - fix: add `/usr/sbin:/sbin` to `PATH`, then retry +- Cloud Hypervisor socket path too long: + - symptom: `path must be shorter than SUN_LEN` + - fix: shorten the remote repo path and any temp/data directories used by the test +- `/tmp` or shared test-lock issues: + - symptom: test-specific lock or temp-file permission failures + - fix: prefer no-network test setup when the test does not require networking; avoid unnecessary shared lock files + +## Reporting Guidance + +When reporting results, separate: + +1. Code failures +2. Test harness issues +3. Remote host environment issues + +This matters in Hypeman because Linux integration tests often fail before product code executes if the remote machine is missing bundled binaries, system tools, or short enough paths. diff --git a/skills/hypeman-remote-linux-tests/agents/openai.yaml b/skills/hypeman-remote-linux-tests/agents/openai.yaml new file mode 100644 index 00000000..244c1d28 --- /dev/null +++ b/skills/hypeman-remote-linux-tests/agents/openai.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Hypeman Remote Linux Tests" + short_description: "Run Hypeman tests on a remote Linux hypervisor host" + default_prompt: "Run Hypeman tests on a remote Linux host, prepare prerequisites, and report actionable failures."