diff --git a/cmd/api/api/api_test.go b/cmd/api/api/api_test.go index 38b7eadd..a6c2edcf 100644 --- a/cmd/api/api/api_test.go +++ b/cmd/api/api/api_test.go @@ -146,6 +146,7 @@ func ctxWithImage(svc *ApiService, name string) context.Context { // Returns the image name on success, or fails the test on error/timeout. func createAndWaitForImage(t *testing.T, svc *ApiService, imageName string, timeout time.Duration) string { t.Helper() + imageName = apiTestImageRef(t, imageName) t.Logf("Creating image %s...", imageName) imgResp, err := svc.CreateImage(ctx(), oapi.CreateImageRequestObject{ diff --git a/cmd/api/api/cp_test.go b/cmd/api/api/cp_test.go index 4a9d5646..0fee9eff 100644 --- a/cmd/api/api/cp_test.go +++ b/cmd/api/api/cp_test.go @@ -28,6 +28,7 @@ func TestCpToAndFromInstance(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/nginx:alpine" // Ensure system files (kernel and initrd) are available t.Log("Ensuring system files...") @@ -37,7 +38,7 @@ func TestCpToAndFromInstance(t *testing.T) { t.Log("System files ready") // Create and wait for nginx image (has a long-running process) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 30*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 30*time.Second) // Create instance t.Log("Creating instance...") @@ -45,11 +46,12 @@ func TestCpToAndFromInstance(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "cp-test", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -168,6 +170,7 @@ func TestCpDirectoryToInstance(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/nginx:alpine" // Ensure system files t.Log("Ensuring system files...") @@ -176,7 +179,7 @@ func TestCpDirectoryToInstance(t *testing.T) { require.NoError(t, err) // Create and wait for nginx image (has a long-running process) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 30*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 30*time.Second) // Create instance t.Log("Creating instance...") @@ -184,11 +187,12 @@ func TestCpDirectoryToInstance(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "cp-dir-test", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, diff --git a/cmd/api/api/exec_test.go b/cmd/api/api/exec_test.go index 2f30b1e5..b460e2fa 100644 --- a/cmd/api/api/exec_test.go +++ b/cmd/api/api/exec_test.go @@ -29,6 +29,7 @@ func TestExecInstanceNonTTY(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/nginx:alpine" // Ensure system files (kernel and initrd) are available t.Log("Ensuring system files...") @@ -38,7 +39,7 @@ func TestExecInstanceNonTTY(t *testing.T) { t.Log("System files ready") // Create and wait for nginx image (has a proper long-running process) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 30*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 30*time.Second) // Create instance t.Log("Creating instance...") @@ -46,11 +47,12 @@ func TestExecInstanceNonTTY(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "exec-test", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -170,6 +172,7 @@ func TestExecWithDebianMinimal(t *testing.T) { } svc := newTestService(t) + imageName := "docker.io/library/debian:12-slim" // Ensure system files (kernel and initrd) are available t.Log("Ensuring system files...") @@ -179,7 +182,7 @@ func TestExecWithDebianMinimal(t *testing.T) { t.Log("System files ready") // Create Debian 12 slim image (minimal, no iproute2) - createAndWaitForImage(t, svc, "docker.io/library/debian:12-slim", 60*time.Second) + imageName = createAndWaitForImage(t, svc, imageName, 60*time.Second) // Create instance with a long-running command so the VM stays alive for exec. // Debian's default CMD is "bash" which exits immediately (no stdin), @@ -190,12 +193,13 @@ func TestExecWithDebianMinimal(t *testing.T) { instResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "debian-exec-test", - Image: "docker.io/library/debian:12-slim", + Image: imageName, Cmd: &cmdOverride, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, diff --git a/cmd/api/api/images_test.go b/cmd/api/api/images_test.go index 8a060da1..7b8b1062 100644 --- a/cmd/api/api/images_test.go +++ b/cmd/api/api/images_test.go @@ -2,7 +2,6 @@ package api import ( "fmt" - "strings" "testing" "time" @@ -42,8 +41,8 @@ func TestCreateImage_Async(t *testing.T) { // Create images before alpine to populate the queue t.Log("Creating image queue...") queueImages := []string{ - "docker.io/library/busybox:latest", - "docker.io/library/nginx:alpine", + apiTestImageRef(t, "docker.io/library/busybox:latest"), + apiTestImageRef(t, "docker.io/library/nginx:alpine"), } for _, name := range queueImages { _, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ @@ -54,9 +53,10 @@ func TestCreateImage_Async(t *testing.T) { // Create alpine (should be last in queue) t.Log("Creating alpine image (should be queued)...") + alpineName := apiTestImageRef(t, "docker.io/library/alpine:latest") createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ Body: &oapi.CreateImageRequest{ - Name: "docker.io/library/alpine:latest", + Name: alpineName, }, }) require.NoError(t, err) @@ -65,14 +65,16 @@ func TestCreateImage_Async(t *testing.T) { require.True(t, ok, "expected 202 accepted response") img := oapi.Image(acceptedResp) - require.Equal(t, "docker.io/library/alpine:latest", img.Name) + require.Equal(t, alpineName, img.Name) require.NotEmpty(t, img.Digest, "digest should be populated immediately") t.Logf("Image created: name=%s, digest=%s, initial_status=%s, queue_position=%v", img.Name, img.Digest, img.Status, img.QueuePosition) // Construct digest reference for polling: repository@digest // GetImage expects format like "docker.io/library/alpine@sha256:..." - digestRef := "docker.io/library/alpine@" + img.Digest + alpineRef, err := images.ParseNormalizedRef(alpineName) + require.NoError(t, err) + digestRef := alpineRef.Repository() + "@" + img.Digest t.Logf("Polling with digest reference: %s", digestRef) // Poll until ready using digest (tag symlink doesn't exist until status=ready) @@ -135,7 +137,7 @@ func TestCreateImage_InvalidTag(t *testing.T) { t.Log("Creating image with invalid tag...") createResp, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ Body: &oapi.CreateImageRequest{ - Name: "docker.io/library/busybox:foobar", + Name: apiTestImageRef(t, "docker.io/library/busybox:foobar"), }, }) require.NoError(t, err) @@ -181,13 +183,13 @@ func TestCreateImage_Idempotent(t *testing.T) { ctx := ctx() // Create first image to occupy queue position 0 - t.Log("Creating first image (busybox) to occupy queue...") + t.Log("Creating first image (nginx) to occupy queue...") _, err := svc.CreateImage(ctx, oapi.CreateImageRequestObject{ - Body: &oapi.CreateImageRequest{Name: "docker.io/library/busybox:latest"}, + Body: &oapi.CreateImageRequest{Name: apiTestImageRef(t, "docker.io/library/nginx:alpine")}, }) require.NoError(t, err) - imageName := "docker.io/library/alpine:3.18" + imageName := apiTestImageRef(t, "docker.io/library/alpine:latest") // First call - should create and queue at position 1 t.Log("First CreateImage call (alpine)...") @@ -245,9 +247,9 @@ func TestCreateImage_Idempotent(t *testing.T) { } // Construct digest reference: repository@digest - // Extract repository from imageName (strip tag part) - repository := strings.Split(imageName, ":")[0] - digestRef := repository + "@" + img1.Digest + imageRef, err := images.ParseNormalizedRef(imageName) + require.NoError(t, err) + digestRef := imageRef.Repository() + "@" + img1.Digest t.Logf("Polling with digest reference: %s", digestRef) // Wait for build to complete - poll by digest (tag symlink doesn't exist until status=ready) diff --git a/cmd/api/api/instances.go b/cmd/api/api/instances.go index b771addf..ba5b448a 100644 --- a/cmd/api/api/instances.go +++ b/cmd/api/api/instances.go @@ -135,6 +135,41 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst if request.Body.Network != nil && request.Body.Network.Enabled != nil { networkEnabled = *request.Body.Network.Enabled } + var networkEgress *instances.NetworkEgressPolicy + if request.Body.Network != nil && request.Body.Network.Egress != nil { + enabled := request.Body.Network.Egress.Enabled != nil && *request.Body.Network.Egress.Enabled + networkEgress = &instances.NetworkEgressPolicy{Enabled: enabled} + if request.Body.Network.Egress.Enforcement != nil && request.Body.Network.Egress.Enforcement.Mode != nil { + networkEgress.EnforcementMode = instances.EgressEnforcementMode(*request.Body.Network.Egress.Enforcement.Mode) + } else if enabled { + networkEgress.EnforcementMode = instances.EgressEnforcementModeAll + } + } + var credentials map[string]instances.CredentialPolicy + if request.Body.Credentials != nil { + credentials = make(map[string]instances.CredentialPolicy, len(*request.Body.Credentials)) + for credentialName, credential := range *request.Body.Credentials { + policy := instances.CredentialPolicy{ + Source: instances.CredentialSource{ + Env: credential.Source.Env, + }, + Inject: make([]instances.CredentialInjectRule, 0, len(credential.Inject)), + } + for _, inject := range credential.Inject { + rule := instances.CredentialInjectRule{ + As: instances.CredentialInjectAs{ + Header: inject.As.Header, + Format: inject.As.Format, + }, + } + if inject.Hosts != nil { + rule.Hosts = append([]string(nil), (*inject.Hosts)...) + } + policy.Inject = append(policy.Inject, rule) + } + credentials[credentialName] = policy + } + } // Parse network bandwidth limits (0 = auto) // Supports both bit-based (e.g., "1Gbps") and byte-based (e.g., "125MB/s") formats @@ -255,6 +290,8 @@ func (s *ApiService) CreateInstance(ctx context.Context, request oapi.CreateInst Env: env, Tags: resourceTags, NetworkEnabled: networkEnabled, + NetworkEgress: networkEgress, + Credentials: credentials, Devices: deviceRefs, Volumes: volumes, Hypervisor: hvType, diff --git a/cmd/api/api/instances_test.go b/cmd/api/api/instances_test.go index 3c2be35e..ea883a08 100644 --- a/cmd/api/api/instances_test.go +++ b/cmd/api/api/instances_test.go @@ -66,9 +66,10 @@ func TestCreateInstance_AutoPullImage(t *testing.T) { Name: "test-auto-pull", Image: "docker.io/library/alpine:latest", Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -100,7 +101,7 @@ func TestCreateInstance_ParsesHumanReadableSizes(t *testing.T) { svc := newTestService(t) // Create and wait for alpine image - createAndWaitForImage(t, svc, "docker.io/library/alpine:latest", 30*time.Second) + imageName := createAndWaitForImage(t, svc, "docker.io/library/alpine:latest", 30*time.Second) // Ensure system files (kernel and initramfs) are available t.Log("Ensuring system files (kernel and initramfs)...") @@ -119,14 +120,15 @@ func TestCreateInstance_ParsesHumanReadableSizes(t *testing.T) { resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "test-sizes", - Image: "docker.io/library/alpine:latest", + Image: imageName, Size: &size, HotplugSize: &hotplugSize, OverlaySize: &overlaySize, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -171,9 +173,10 @@ func TestCreateInstance_InvalidSizeFormat(t *testing.T) { Image: "docker.io/library/alpine:latest", Size: &invalidSize, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -265,6 +268,146 @@ func TestCreateInstance_OmittedHotplugSizeDefaultsToZero(t *testing.T) { assert.Equal(t, int64(0), int64(hotplugBytes), "response should report zero hotplug_size when omitted") } +func TestCreateInstance_MapsNetworkEgressCredentials(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + networkEnabled := true + egressEnabled := true + credentials := map[string]oapi.CreateInstanceRequestCredential{ + "OUTBOUND_OPENAI_KEY": { + Source: oapi.CreateInstanceRequestCredentialSource{ + Env: "OUTBOUND_OPENAI_KEY", + }, + Inject: []oapi.CreateInstanceRequestCredentialInject{ + { + Hosts: &[]string{"api.openai.com", "*.openai.com"}, + As: oapi.CreateInstanceRequestCredentialInjectAs{ + Header: "Authorization", + Format: "Bearer ${value}", + }, + }, + }, + }, + "GITHUB_TOKEN": { + Source: oapi.CreateInstanceRequestCredentialSource{ + Env: "GITHUB_TOKEN", + }, + Inject: []oapi.CreateInstanceRequestCredentialInject{ + { + As: oapi.CreateInstanceRequestCredentialInjectAs{ + Header: "X-GitHub-Token", + Format: "${value}", + }, + }, + }, + }, + } + env := map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-openai-key-123", + "GITHUB_TOKEN": "real-gh-token-456", + } + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-egress-proxy-mock-env-vars", + Image: "docker.io/library/alpine:latest", + Env: &env, + Network: &struct { + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + }{ + Enabled: &networkEnabled, + Egress: &oapi.CreateInstanceRequestNetworkEgress{ + Enabled: &egressEnabled, + }, + }, + Credentials: &credentials, + }, + }) + require.NoError(t, err) + _, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.NetworkEgress) + assert.True(t, mockMgr.lastReq.NetworkEgress.Enabled) + assert.Equal(t, instances.EgressEnforcementModeAll, mockMgr.lastReq.NetworkEgress.EnforcementMode) + assert.Equal(t, "OUTBOUND_OPENAI_KEY", mockMgr.lastReq.Credentials["OUTBOUND_OPENAI_KEY"].Source.Env) + assert.Equal(t, []string{"api.openai.com", "*.openai.com"}, mockMgr.lastReq.Credentials["OUTBOUND_OPENAI_KEY"].Inject[0].Hosts) + assert.Equal(t, "Authorization", mockMgr.lastReq.Credentials["OUTBOUND_OPENAI_KEY"].Inject[0].As.Header) + assert.Equal(t, "Bearer ${value}", mockMgr.lastReq.Credentials["OUTBOUND_OPENAI_KEY"].Inject[0].As.Format) + assert.Equal(t, "real-openai-key-123", mockMgr.lastReq.Env["OUTBOUND_OPENAI_KEY"]) + assert.Equal(t, "real-gh-token-456", mockMgr.lastReq.Env["GITHUB_TOKEN"]) +} + +func TestCreateInstance_MapsNetworkEgressEnforcementMode(t *testing.T) { + t.Parallel() + svc := newTestService(t) + + origMgr := svc.InstanceManager + mockMgr := &captureCreateManager{Manager: origMgr} + svc.InstanceManager = mockMgr + + networkEnabled := true + egressEnabled := true + mode := oapi.HttpHttpsOnly + env := map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-openai-key-123", + } + credentials := map[string]oapi.CreateInstanceRequestCredential{ + "OUTBOUND_OPENAI_KEY": { + Source: oapi.CreateInstanceRequestCredentialSource{ + Env: "OUTBOUND_OPENAI_KEY", + }, + Inject: []oapi.CreateInstanceRequestCredentialInject{ + { + As: oapi.CreateInstanceRequestCredentialInjectAs{ + Header: "Authorization", + Format: "Bearer ${value}", + }, + }, + }, + }, + } + + resp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ + Body: &oapi.CreateInstanceRequest{ + Name: "test-egress-proxy-enforcement-mode", + Image: "docker.io/library/alpine:latest", + Env: &env, + Network: &struct { + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` + }{ + Enabled: &networkEnabled, + Egress: &oapi.CreateInstanceRequestNetworkEgress{ + Enabled: &egressEnabled, + Enforcement: &oapi.CreateInstanceRequestNetworkEgressEnforcement{ + Mode: &mode, + }, + }, + }, + Credentials: &credentials, + }, + }) + require.NoError(t, err) + _, ok := resp.(oapi.CreateInstance201JSONResponse) + require.True(t, ok, "expected 201 response") + + require.NotNil(t, mockMgr.lastReq) + require.NotNil(t, mockMgr.lastReq.NetworkEgress) + assert.Equal(t, instances.EgressEnforcementModeHTTPHTTPSOnly, mockMgr.lastReq.NetworkEgress.EnforcementMode) +} + func TestForkInstance_Success(t *testing.T) { t.Parallel() svc := newTestService(t) @@ -458,7 +601,7 @@ func TestInstanceLifecycle_StopStart(t *testing.T) { svc := newTestService(t) // Use nginx:alpine so the VM runs a real workload (not just exits immediately) - createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 60*time.Second) + imageName := createAndWaitForImage(t, svc, "docker.io/library/nginx:alpine", 60*time.Second) // Ensure system files (kernel and initramfs) are available t.Log("Ensuring system files (kernel and initramfs)...") @@ -473,11 +616,12 @@ func TestInstanceLifecycle_StopStart(t *testing.T) { createResp, err := svc.CreateInstance(ctx(), oapi.CreateInstanceRequestObject{ Body: &oapi.CreateInstanceRequest{ Name: "test-lifecycle", - Image: "docker.io/library/nginx:alpine", + Image: imageName, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, diff --git a/cmd/api/api/registry_test.go b/cmd/api/api/registry_test.go index 4f664ddf..b92f6846 100644 --- a/cmd/api/api/registry_test.go +++ b/cmd/api/api/registry_test.go @@ -52,7 +52,7 @@ func TestRegistryPushAndConvert(t *testing.T) { // Pull a small image from Docker Hub to push to our registry t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -110,7 +110,7 @@ func TestRegistryPushAndCreateInstance(t *testing.T) { // Pull and push alpine t.Log("Pulling alpine:latest...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -142,9 +142,10 @@ func TestRegistryPushAndCreateInstance(t *testing.T) { Image: imageName, Cmd: &cmd, Network: &struct { - BandwidthDownload *string `json:"bandwidth_download,omitempty"` - BandwidthUpload *string `json:"bandwidth_upload,omitempty"` - Enabled *bool `json:"enabled,omitempty"` + BandwidthDownload *string `json:"bandwidth_download,omitempty"` + BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + Egress *oapi.CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + Enabled *bool `json:"enabled,omitempty"` }{ Enabled: &networkEnabled, }, @@ -184,7 +185,7 @@ func TestRegistryLayerCaching(t *testing.T) { // Pull alpine image from Docker Hub t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -268,7 +269,7 @@ func TestRegistrySharedLayerCaching(t *testing.T) { // Pull alpine image (this will be our base) t.Log("Pulling alpine:latest...") - alpineRef, err := name.ParseReference("docker.io/library/alpine:latest") + alpineRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) alpineImg, err := remote.Image(alpineRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) require.NoError(t, err) @@ -300,7 +301,7 @@ func TestRegistrySharedLayerCaching(t *testing.T) { // Now pull a different alpine-based image (e.g., alpine:3.18) // which should share the base layer with alpine:latest t.Log("Pulling alpine:3.18 (shares base layer)...") - alpine318Ref, err := name.ParseReference("docker.io/library/alpine:3.18") + alpine318Ref, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:3.18")) require.NoError(t, err) alpine318Img, err := remote.Image(alpine318Ref, remote.WithAuthFromKeychain(authn.DefaultKeychain)) require.NoError(t, err) @@ -350,7 +351,7 @@ func TestRegistryTagPush(t *testing.T) { // Pull alpine image from Docker Hub t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) @@ -404,7 +405,7 @@ func TestRegistryDockerV2ManifestConversion(t *testing.T) { // Pull alpine image from Docker Hub (OCI format) t.Log("Pulling alpine:latest from Docker Hub...") - srcRef, err := name.ParseReference("docker.io/library/alpine:latest") + srcRef, err := name.ParseReference(apiTestImageRef(t, "docker.io/library/alpine:latest")) require.NoError(t, err) img, err := remote.Image(srcRef, remote.WithAuthFromKeychain(authn.DefaultKeychain)) diff --git a/cmd/api/api/test_prewarm_test.go b/cmd/api/api/test_prewarm_test.go new file mode 100644 index 00000000..e347f7aa --- /dev/null +++ b/cmd/api/api/test_prewarm_test.go @@ -0,0 +1,70 @@ +package api + +import ( + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const ( + testPrewarmStrictEnv = "HYPEMAN_TEST_PREWARM_STRICT" + testRegistryEnv = "HYPEMAN_TEST_REGISTRY" +) + +var apiRegistryLogOnce sync.Once +var apiRegistryLogMessage string + +func apiTestImageRef(t *testing.T, source string) string { + t.Helper() + + registry := strings.TrimSpace(os.Getenv(testRegistryEnv)) + if registry == "" { + if isTestPrewarmStrict() { + t.Fatalf("%s is required when %s is enabled", testRegistryEnv, testPrewarmStrictEnv) + } + return source + } + + registry = strings.TrimPrefix(strings.TrimPrefix(registry, "http://"), "https://") + if registry == "" { + t.Fatalf("%s must not be empty", testRegistryEnv) + } + + ref, err := images.ParseNormalizedRef(source) + if err != nil { + t.Fatalf("parse source image ref %q: %v", source, err) + } + + repo := ref.Repository() + if !strings.HasPrefix(repo, "docker.io/") { + return source + } + repo = strings.TrimPrefix(repo, "docker.io/") + + var mapped string + switch { + case ref.Tag() != "": + mapped = registry + "/" + repo + ":" + ref.Tag() + case ref.Digest() != "": + mapped = registry + "/" + repo + "@" + ref.Digest() + default: + mapped = registry + "/" + repo + ":latest" + } + + apiRegistryLogOnce.Do(func() { + apiRegistryLogMessage = fmt.Sprintf("using test registry mirror source=%s mapped=%s", source, mapped) + }) + if apiRegistryLogMessage != "" { + t.Log(apiRegistryLogMessage) + } + return mapped +} + +func isTestPrewarmStrict() bool { + v := strings.TrimSpace(os.Getenv(testPrewarmStrictEnv)) + return v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes") +} diff --git a/cmd/test-prewarm/main.go b/cmd/test-prewarm/main.go index e710f5db..48853006 100644 --- a/cmd/test-prewarm/main.go +++ b/cmd/test-prewarm/main.go @@ -29,8 +29,11 @@ const ( var defaultImages = []string{ "docker.io/library/alpine:latest", + "docker.io/library/alpine:3.18", + "docker.io/library/debian:12-slim", "docker.io/library/nginx:alpine", "docker.io/bitnami/redis:latest", + "docker.io/jrei/systemd-ubuntu:22.04", } type manifestImage struct { diff --git a/integration/systemd_test.go b/integration/systemd_test.go index 986ebb11..ad60510d 100644 --- a/integration/systemd_test.go +++ b/integration/systemd_test.go @@ -75,7 +75,7 @@ func TestSystemdMode(t *testing.T) { instanceManager.DeleteInstance(ctx, "systemd-test") }) - imageName := "docker.io/jrei/systemd-ubuntu:22.04" + imageName := integrationTestImageRef(t, "docker.io/jrei/systemd-ubuntu:22.04") // Pull the systemd image t.Log("Pulling systemd image:", imageName) diff --git a/integration/test_prewarm_test.go b/integration/test_prewarm_test.go new file mode 100644 index 00000000..aa5c82e9 --- /dev/null +++ b/integration/test_prewarm_test.go @@ -0,0 +1,70 @@ +package integration + +import ( + "fmt" + "os" + "strings" + "sync" + "testing" + + "github.com/kernel/hypeman/lib/images" +) + +const ( + testPrewarmStrictEnv = "HYPEMAN_TEST_PREWARM_STRICT" + testRegistryEnv = "HYPEMAN_TEST_REGISTRY" +) + +var integrationRegistryLogOnce sync.Once +var integrationRegistryLogMessage string + +func integrationTestImageRef(t *testing.T, source string) string { + t.Helper() + + registry := strings.TrimSpace(os.Getenv(testRegistryEnv)) + if registry == "" { + if isTestPrewarmStrict() { + t.Fatalf("%s is required when %s is enabled", testRegistryEnv, testPrewarmStrictEnv) + } + return source + } + + registry = strings.TrimPrefix(strings.TrimPrefix(registry, "http://"), "https://") + if registry == "" { + t.Fatalf("%s must not be empty", testRegistryEnv) + } + + ref, err := images.ParseNormalizedRef(source) + if err != nil { + t.Fatalf("parse source image ref %q: %v", source, err) + } + + repo := ref.Repository() + if !strings.HasPrefix(repo, "docker.io/") { + return source + } + repo = strings.TrimPrefix(repo, "docker.io/") + + var mapped string + switch { + case ref.Tag() != "": + mapped = registry + "/" + repo + ":" + ref.Tag() + case ref.Digest() != "": + mapped = registry + "/" + repo + "@" + ref.Digest() + default: + mapped = registry + "/" + repo + ":latest" + } + + integrationRegistryLogOnce.Do(func() { + integrationRegistryLogMessage = fmt.Sprintf("using test registry mirror source=%s mapped=%s", source, mapped) + }) + if integrationRegistryLogMessage != "" { + t.Log(integrationRegistryLogMessage) + } + return mapped +} + +func isTestPrewarmStrict() bool { + v := strings.TrimSpace(os.Getenv(testPrewarmStrictEnv)) + return v == "1" || strings.EqualFold(v, "true") || strings.EqualFold(v, "yes") +} diff --git a/integration/vgpu_test.go b/integration/vgpu_test.go index 17a92ad1..cbf59d09 100644 --- a/integration/vgpu_test.go +++ b/integration/vgpu_test.go @@ -97,7 +97,7 @@ func TestVGPU(t *testing.T) { t.Log("System files ready") // Step 2: Pull alpine image (lightweight for testing) - imageName := "docker.io/library/alpine:latest" + imageName := integrationTestImageRef(t, "docker.io/library/alpine:latest") t.Log("Step 2: Pulling alpine image...") _, err = imageManager.CreateImage(ctx, images.CreateImageRequest{ Name: imageName, diff --git a/lib/egressproxy/README.md b/lib/egressproxy/README.md new file mode 100644 index 00000000..42d77d42 --- /dev/null +++ b/lib/egressproxy/README.md @@ -0,0 +1,44 @@ +# Egress Proxy (Mock Secret Substitution) + +This module provides an optional, default-off networking mode for VM egress. + +When enabled for an instance, hypeman does three things: + +1. It starts (or reuses) a host-side HTTP/HTTPS MITM proxy bound to the VM bridge gateway. +2. It injects proxy environment variables into the guest (`HTTP_PROXY` / `HTTPS_PROXY`) and installs the proxy CA certificate in the guest trust store. +3. It enforces policy on the host to prevent direct outbound TCP egress from the VM unless traffic is going to the bridge gateway (the proxy), depending on `network.egress.enforcement.mode`. + +## Secret substitution flow + +- API callers provide real secret values in instance `env`. +- Per instance, `credentials` defines host-managed credential brokering policies keyed by guest-visible credential name. +- Each credential policy uses: + - `source.env` for the real value source in host env. + - `inject[*].hosts` to optionally restrict destination hosts: + - Exact host: `api.openai.com` + - Single-level wildcard: `*.openai.com` + - If omitted, injection is allowed for all destinations. + - `inject[*].as` to define the header/format template shape. +- Per instance, `network.egress.enforcement.mode` controls host-side direct egress blocking: + - `all` (default when proxy is enabled): reject direct non-proxy TCP egress from the VM TAP interface. + - `http_https_only`: reject direct TCP egress only on destination ports `80` and `443`. +- Inside the VM, each credential key is rewritten to `mock-` (for example `mock-OUTBOUND_OPENAI_KEY`). +- Header injection is applied to HTTPS requests only after MITM decryption. +- For HTTPS egress, the proxy validates upstream TLS certificates with the host trust store before forwarding. +- The proxy materializes the configured `inject[*].as.header` using `inject[*].as.format` with the real value only when the verified destination host matches the credential allowlist (if configured). +- The modified request is then forwarded upstream. + +This keeps real secrets out of the VM while still allowing authenticated egress requests. + +## Security behavior + +- Real secret values are persisted in the normal instance `env` metadata, which is already host-side state. +- TLS interception requires guest trust of the proxy CA; hypeman installs this CA in the guest when proxy mode is enabled. +- Egress enforcement is applied per instance TAP device and removed when the instance stops/standbys/deletes. +- Enforcement intentionally targets TCP egress only. DNS/other non-TCP traffic is not rewritten and is not blocked by `all` mode. + +## Limits of enforcement + +- Header injection is applied to HTTP headers only (not request/response bodies). +- Non-HTTP protocols or custom ports are not rewritten by the MITM layer. +- Plain HTTP requests are not eligible for secret substitution. diff --git a/lib/egressproxy/cert.go b/lib/egressproxy/cert.go new file mode 100644 index 00000000..4bfd36f1 --- /dev/null +++ b/lib/egressproxy/cert.go @@ -0,0 +1,153 @@ +package egressproxy + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "path/filepath" + "time" +) + +func loadOrCreateCA(dataDir string) (*x509.Certificate, *rsa.PrivateKey, string, error) { + dir := filepath.Join(dataDir, "egressproxy") + certPath := filepath.Join(dir, "ca.crt") + keyPath := filepath.Join(dir, "ca.key") + + certPEM, certErr := os.ReadFile(certPath) + keyPEM, keyErr := os.ReadFile(keyPath) + if certErr == nil && keyErr == nil { + cert, key, err := parseCA(certPEM, keyPEM) + if err == nil { + return cert, key, string(certPEM), nil + } + } + + if err := os.MkdirAll(dir, 0755); err != nil { + return nil, nil, "", fmt.Errorf("create egress proxy cert dir: %w", err) + } + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, nil, "", fmt.Errorf("generate egress proxy CA key: %w", err) + } + + serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, nil, "", fmt.Errorf("generate CA serial: %w", err) + } + + now := time.Now() + tmpl := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: "hypeman-egress-proxy-ca", + Organization: []string{"hypeman"}, + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(10 * 365 * 24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 1, + } + + certDER, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &caKey.PublicKey, caKey) + if err != nil { + return nil, nil, "", fmt.Errorf("create egress proxy CA cert: %w", err) + } + + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER}) + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(caKey)}) + + if err := os.WriteFile(certPath, certPEM, 0644); err != nil { + return nil, nil, "", fmt.Errorf("write egress proxy CA cert: %w", err) + } + if err := os.WriteFile(keyPath, keyPEM, 0600); err != nil { + return nil, nil, "", fmt.Errorf("write egress proxy CA key: %w", err) + } + + cert, key, err := parseCA(certPEM, keyPEM) + if err != nil { + return nil, nil, "", err + } + + return cert, key, string(certPEM), nil +} + +func parseCA(certPEM, keyPEM []byte) (*x509.Certificate, *rsa.PrivateKey, error) { + certBlock, _ := pem.Decode(certPEM) + if certBlock == nil { + return nil, nil, fmt.Errorf("parse egress proxy CA cert PEM") + } + cert, err := x509.ParseCertificate(certBlock.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parse egress proxy CA cert: %w", err) + } + keyBlock, _ := pem.Decode(keyPEM) + if keyBlock == nil { + return nil, nil, fmt.Errorf("parse egress proxy CA key PEM") + } + key, err := x509.ParsePKCS1PrivateKey(keyBlock.Bytes) + if err != nil { + return nil, nil, fmt.Errorf("parse egress proxy CA key: %w", err) + } + return cert, key, nil +} + +func signHostCertificate(caCert *x509.Certificate, caKey *rsa.PrivateKey, host string) (*tls.Certificate, error) { + leafKey, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + return nil, fmt.Errorf("generate leaf key: %w", err) + } + + serialNumber, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return nil, fmt.Errorf("generate leaf serial: %w", err) + } + + now := time.Now() + tmpl := &x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + CommonName: host, + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + if ip := net.ParseIP(host); ip != nil { + tmpl.IPAddresses = []net.IP{ip} + } else { + tmpl.DNSNames = []string{host} + } + + leafDER, err := x509.CreateCertificate(rand.Reader, tmpl, caCert, &leafKey.PublicKey, caKey) + if err != nil { + return nil, fmt.Errorf("sign leaf cert: %w", err) + } + + leafPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}) + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caCert.Raw}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(leafKey)}) + + tlsCert, err := tls.X509KeyPair(append(leafPEM, caPEM...), keyPEM) + if err != nil { + return nil, fmt.Errorf("build leaf keypair: %w", err) + } + if len(tlsCert.Certificate) > 0 { + if leaf, parseErr := x509.ParseCertificate(tlsCert.Certificate[0]); parseErr == nil { + tlsCert.Leaf = leaf + } + } + return &tlsCert, nil +} diff --git a/lib/egressproxy/domains.go b/lib/egressproxy/domains.go new file mode 100644 index 00000000..4d530203 --- /dev/null +++ b/lib/egressproxy/domains.go @@ -0,0 +1,152 @@ +package egressproxy + +import ( + "fmt" + "net" + "strings" +) + +type domainMatcher struct { + exact string + wildcardSuffix string +} + +func normalizeDestinationHost(raw string) string { + host := strings.TrimSpace(raw) + if host == "" { + return "" + } + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + host = strings.TrimPrefix(host, "[") + host = strings.TrimSuffix(host, "]") + host = strings.TrimSuffix(host, ".") + host = strings.ToLower(host) + if ip := net.ParseIP(host); ip != nil { + return ip.String() + } + return host +} + +// NormalizeAllowedDomainPattern validates and normalizes one allowlisted destination +// host pattern (exact host/IP or single-level wildcard *.example.com). +func NormalizeAllowedDomainPattern(pattern string) (string, error) { + p := strings.TrimSpace(pattern) + if p == "" { + return "", fmt.Errorf("allowed domain pattern must be non-empty") + } + if strings.Contains(p, "://") || strings.ContainsAny(p, "/?#") { + return "", fmt.Errorf("allowed domain pattern %q must be a host pattern without scheme/path", p) + } + if _, _, err := net.SplitHostPort(p); err == nil { + return "", fmt.Errorf("allowed domain pattern %q must not include a port", p) + } + + if strings.HasPrefix(p, "*.") { + suffixRaw := strings.TrimPrefix(p, "*.") + if strings.Contains(suffixRaw, ":") { + return "", fmt.Errorf("allowed wildcard domain pattern %q must not include a port", p) + } + + base := normalizeDestinationHost(suffixRaw) + if base == "" { + return "", fmt.Errorf("allowed wildcard domain pattern %q has empty suffix", p) + } + if net.ParseIP(base) != nil { + return "", fmt.Errorf("allowed wildcard domain pattern %q cannot target an IP", p) + } + if !isValidDNSName(base) { + return "", fmt.Errorf("allowed wildcard domain pattern %q has invalid hostname suffix", p) + } + if strings.Count(base, ".") < 1 { + return "", fmt.Errorf("allowed wildcard domain pattern %q must target at least a two-label domain", p) + } + return "*." + base, nil + } + + if strings.Contains(p, "*") { + return "", fmt.Errorf("allowed domain pattern %q has unsupported wildcard syntax", p) + } + + host := normalizeDestinationHost(p) + if host == "" { + return "", fmt.Errorf("allowed domain pattern %q has empty host", p) + } + if net.ParseIP(host) != nil { + return host, nil + } + if !isValidDNSName(host) { + return "", fmt.Errorf("allowed domain pattern %q has invalid hostname", p) + } + return host, nil +} + +func compileDomainMatchers(patterns []string) ([]domainMatcher, error) { + if len(patterns) == 0 { + return nil, nil + } + out := make([]domainMatcher, 0, len(patterns)) + for _, raw := range patterns { + normalized, err := NormalizeAllowedDomainPattern(raw) + if err != nil { + return nil, err + } + if strings.HasPrefix(normalized, "*.") { + out = append(out, domainMatcher{wildcardSuffix: "." + strings.TrimPrefix(normalized, "*.")}) + continue + } + out = append(out, domainMatcher{exact: normalized}) + } + return out, nil +} + +func matchesAnyDomain(host string, matchers []domainMatcher) bool { + if len(matchers) == 0 { + return true + } + for _, m := range matchers { + if m.matches(host) { + return true + } + } + return false +} + +func (m domainMatcher) matches(host string) bool { + if host == "" { + return false + } + if m.exact != "" { + return host == m.exact + } + if m.wildcardSuffix == "" || !strings.HasSuffix(host, m.wildcardSuffix) { + return false + } + prefix := strings.TrimSuffix(host, m.wildcardSuffix) + return prefix != "" && !strings.Contains(prefix, ".") +} + +func isValidDNSName(host string) bool { + if host == "" || len(host) > 253 { + return false + } + labels := strings.Split(host, ".") + for _, label := range labels { + if label == "" || len(label) > 63 { + return false + } + for i, ch := range label { + isLetter := ch >= 'a' && ch <= 'z' + isDigit := ch >= '0' && ch <= '9' + isHyphen := ch == '-' + if !(isLetter || isDigit || isHyphen) { + return false + } + if (i == 0 || i == len(label)-1) && isHyphen { + return false + } + } + } + return true +} diff --git a/lib/egressproxy/enforce_linux.go b/lib/egressproxy/enforce_linux.go new file mode 100644 index 00000000..898f8475 --- /dev/null +++ b/lib/egressproxy/enforce_linux.go @@ -0,0 +1,141 @@ +//go:build linux + +package egressproxy + +import ( + "fmt" + "os/exec" + "strings" + "unicode" +) + +const ( + enforcementSuffixPort80 = "80" + enforcementSuffixPort443 = "443" + enforcementSuffixAllTCP = "all-tcp" +) + +func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { + if instanceID == "" || tapDevice == "" || gatewayIP == "" || proxyPort <= 0 { + return fmt.Errorf("invalid egress enforcement inputs") + } + + comment80 := enforcementComment(instanceID, enforcementSuffixPort80) + comment443 := enforcementComment(instanceID, enforcementSuffixPort443) + commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP) + + // Clean old rules first so updates are idempotent across restarts and mode changes. + _ = removeRuleByComment(comment80) + _ = removeRuleByComment(comment443) + _ = removeRuleByComment(commentAllTCP) + + if blockAllTCPEgress { + if err := insertRejectAllTCPRule(tapDevice, gatewayIP, commentAllTCP); err != nil { + return fmt.Errorf("insert all-tcp egress enforcement: %w", err) + } + return nil + } + + if err := insertRejectRule(tapDevice, gatewayIP, 80, comment80); err != nil { + return fmt.Errorf("insert port 80 egress enforcement: %w", err) + } + if err := insertRejectRule(tapDevice, gatewayIP, 443, comment443); err != nil { + _ = removeRuleByComment(comment80) + return fmt.Errorf("insert port 443 egress enforcement: %w", err) + } + + return nil +} + +func removeEgressEnforcement(instanceID string) error { + if instanceID == "" { + return nil + } + comment80 := enforcementComment(instanceID, enforcementSuffixPort80) + comment443 := enforcementComment(instanceID, enforcementSuffixPort443) + commentAllTCP := enforcementComment(instanceID, enforcementSuffixAllTCP) + _ = removeRuleByComment(comment80) + _ = removeRuleByComment(comment443) + _ = removeRuleByComment(commentAllTCP) + return nil +} + +func insertRejectRule(tapDevice, gatewayIP string, port int, comment string) error { + cmd := exec.Command( + "iptables", "-I", "FORWARD", "1", + "-i", tapDevice, + "-p", "tcp", + "--dport", fmt.Sprintf("%d", port), + "!", "-d", gatewayIP, + "-m", "comment", "--comment", comment, + "-j", "REJECT", + ) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +func insertRejectAllTCPRule(tapDevice, gatewayIP, comment string) error { + cmd := exec.Command( + "iptables", "-I", "FORWARD", "1", + "-i", tapDevice, + "-p", "tcp", + "!", "-d", gatewayIP, + "-m", "comment", "--comment", comment, + "-j", "REJECT", + ) + if out, err := cmd.CombinedOutput(); err != nil { + return fmt.Errorf("iptables insert failed: %w: %s", err, strings.TrimSpace(string(out))) + } + return nil +} + +func removeRuleByComment(comment string) error { + listCmd := exec.Command("iptables", "-L", "FORWARD", "--line-numbers", "-n") + output, err := listCmd.Output() + if err != nil { + return err + } + + commentMarker := fmt.Sprintf("/* %s */", comment) + var ruleNums []string + for _, line := range strings.Split(string(output), "\n") { + if !strings.Contains(line, commentMarker) { + continue + } + fields := strings.Fields(line) + if len(fields) == 0 { + continue + } + ruleNums = append(ruleNums, fields[0]) + } + + for i := len(ruleNums) - 1; i >= 0; i-- { + delCmd := exec.Command("iptables", "-D", "FORWARD", ruleNums[i]) + _ = delCmd.Run() + } + return nil +} + +func enforcementComment(instanceID, suffix string) string { + safeID := sanitizeInstanceIDForComment(instanceID) + return fmt.Sprintf("hypeman-egress-%s-%s", safeID, suffix) +} + +func sanitizeInstanceIDForComment(instanceID string) string { + cleaned := strings.Map(func(r rune) rune { + switch { + case unicode.IsLetter(r), unicode.IsDigit(r): + return r + case r == '-', r == '_', r == '.': + return r + default: + return '_' + } + }, strings.TrimSpace(instanceID)) + if cleaned == "" { + return "unknown" + } + return cleaned +} diff --git a/lib/egressproxy/enforce_other.go b/lib/egressproxy/enforce_other.go new file mode 100644 index 00000000..3eb72c04 --- /dev/null +++ b/lib/egressproxy/enforce_other.go @@ -0,0 +1,12 @@ +//go:build !linux + +package egressproxy + +func applyEgressEnforcement(instanceID, tapDevice, gatewayIP string, proxyPort int, blockAllTCPEgress bool) error { + _ = blockAllTCPEgress + return nil +} + +func removeEgressEnforcement(instanceID string) error { + return nil +} diff --git a/lib/egressproxy/service.go b/lib/egressproxy/service.go new file mode 100644 index 00000000..75cd2e7f --- /dev/null +++ b/lib/egressproxy/service.go @@ -0,0 +1,476 @@ +package egressproxy + +import ( + "bufio" + "context" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "fmt" + "io" + "log/slog" + "net" + "net/http" + "net/textproto" + "net/url" + "strconv" + "strings" + "sync" + "time" +) + +type sourcePolicy struct { + headerInjectRules []headerInjectRule +} + +type headerInjectRule struct { + headerName string + headerValue string + domainMatchers []domainMatcher +} + +const defaultLeafCertCacheLimit = 512 + +// Service is a host-side per-process HTTP/HTTPS MITM egress proxy. +type Service struct { + mu sync.RWMutex + + dataDir string + gatewayIP string + listenPort int + + server *http.Server + listener net.Listener + + transport *http.Transport + + caCert *x509.Certificate + caKey *rsa.PrivateKey + caPEM string + + certCache map[string]*tls.Certificate + certCacheOrder []string + certCacheLimit int + + policiesBySourceIP map[string]sourcePolicy + sourceIPByInstance map[string]string +} + +func NewService(dataDir string, listenPort int) (*Service, error) { + return NewServiceWithOptions(dataDir, listenPort, ServiceOptions{}) +} + +func NewServiceWithOptions(dataDir string, listenPort int, opts ServiceOptions) (*Service, error) { + if listenPort <= 0 { + listenPort = DefaultListenPort + } + + caCert, caKey, caPEM, err := loadOrCreateCA(dataDir) + if err != nil { + return nil, err + } + + rootCAs, err := buildRootCAPool(opts) + if err != nil { + return nil, err + } + + transport := &http.Transport{ + Proxy: nil, + DialContext: (&net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second}).DialContext, + ForceAttemptHTTP2: false, + MaxIdleConns: 100, + IdleConnTimeout: 90 * time.Second, + TLSHandshakeTimeout: 15 * time.Second, + TLSClientConfig: &tls.Config{ + RootCAs: rootCAs, + }, + } + + return &Service{ + dataDir: dataDir, + listenPort: listenPort, + transport: transport, + caCert: caCert, + caKey: caKey, + caPEM: caPEM, + certCache: make(map[string]*tls.Certificate), + certCacheLimit: defaultLeafCertCacheLimit, + policiesBySourceIP: make(map[string]sourcePolicy), + sourceIPByInstance: make(map[string]string), + }, nil +} + +func buildRootCAPool(opts ServiceOptions) (*x509.CertPool, error) { + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + + for _, pemData := range opts.AdditionalRootCAPEM { + trimmed := strings.TrimSpace(pemData) + if trimmed == "" { + continue + } + if ok := pool.AppendCertsFromPEM([]byte(trimmed)); !ok { + return nil, fmt.Errorf("append additional root CA PEM failed") + } + } + return pool, nil +} + +func (s *Service) EnsureStarted(ctx context.Context, gatewayIP string) error { + s.mu.Lock() + defer s.mu.Unlock() + + if s.listener != nil { + if gatewayIP != "" && gatewayIP != s.gatewayIP { + return fmt.Errorf("%w: current=%s requested=%s", ErrGatewayMismatch, s.gatewayIP, gatewayIP) + } + return nil + } + + s.gatewayIP = gatewayIP + listenAddr := net.JoinHostPort(gatewayIP, strconv.Itoa(s.listenPort)) + ln, err := net.Listen("tcp", listenAddr) + if err != nil { + return fmt.Errorf("listen egress proxy on %s: %w", listenAddr, err) + } + + s.listener = ln + s.server = &http.Server{ + Handler: s, + ReadHeaderTimeout: 15 * time.Second, + } + + go func() { + if serveErr := s.server.Serve(ln); serveErr != nil && serveErr != http.ErrServerClosed { + slog.Error("egress proxy server exited", "error", serveErr) + } + }() + + return nil +} + +func (s *Service) Shutdown(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.server == nil { + return nil + } + err := s.server.Shutdown(ctx) + s.server = nil + s.listener = nil + return err +} + +func (s *Service) RegisterInstance(ctx context.Context, gatewayIP string, cfg InstanceConfig) (GuestConfig, error) { + if err := s.EnsureStarted(ctx, gatewayIP); err != nil { + return GuestConfig{}, err + } + + if err := applyEgressEnforcement(cfg.InstanceID, cfg.TAPDevice, gatewayIP, s.listenPort, cfg.BlockAllTCPEgress); err != nil { + return GuestConfig{}, err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if prevIP, ok := s.sourceIPByInstance[cfg.InstanceID]; ok { + delete(s.policiesBySourceIP, prevIP) + } + + injectRules, err := compileHeaderInjectRules(cfg.HeaderInjectRules) + if err != nil { + return GuestConfig{}, err + } + + s.sourceIPByInstance[cfg.InstanceID] = cfg.SourceIP + s.policiesBySourceIP[cfg.SourceIP] = sourcePolicy{headerInjectRules: injectRules} + + return GuestConfig{ + Enabled: true, + ProxyURL: s.proxyURLLocked(), + CACertPEM: s.caPEM, + }, nil +} + +func compileHeaderInjectRules(cfgRules []HeaderInjectRuleConfig) ([]headerInjectRule, error) { + if len(cfgRules) == 0 { + return nil, nil + } + out := make([]headerInjectRule, 0, len(cfgRules)) + for _, cfg := range cfgRules { + if strings.TrimSpace(cfg.HeaderName) == "" || cfg.HeaderValue == "" { + continue + } + matchers, err := compileDomainMatchers(cfg.AllowedDomains) + if err != nil { + return nil, err + } + out = append(out, headerInjectRule{ + headerName: textproto.CanonicalMIMEHeaderKey(strings.TrimSpace(cfg.HeaderName)), + headerValue: cfg.HeaderValue, + domainMatchers: matchers, + }) + } + return out, nil +} + +func (s *Service) UnregisterInstance(_ context.Context, instanceID string) { + s.mu.Lock() + sourceIP, ok := s.sourceIPByInstance[instanceID] + if ok { + delete(s.sourceIPByInstance, instanceID) + delete(s.policiesBySourceIP, sourceIP) + } + s.mu.Unlock() + + _ = removeEgressEnforcement(instanceID) +} + +func (s *Service) ProxyURL() string { + s.mu.RLock() + defer s.mu.RUnlock() + return s.proxyURLLocked() +} + +func (s *Service) proxyURLLocked() string { + if s.gatewayIP == "" { + return "" + } + return fmt.Sprintf("http://%s:%d", s.gatewayIP, s.listenPort) +} + +func (s *Service) ServeHTTP(w http.ResponseWriter, r *http.Request) { + sourceIP := sourceIPFromRemoteAddr(r.RemoteAddr) + if r.Method == http.MethodConnect { + s.handleConnect(w, r, sourceIP) + return + } + s.handleHTTPProxyRequest(w, r, sourceIP, false) +} + +func (s *Service) handleHTTPProxyRequest(w http.ResponseWriter, r *http.Request, sourceIP string, insideTunnel bool) { + outReq := r.Clone(r.Context()) + outReq.Header = cloneHeader(r.Header) + removeHopByHopHeaders(outReq.Header) + + if !insideTunnel { + if outReq.URL == nil || !outReq.URL.IsAbs() { + outReq.URL = &url.URL{ + Scheme: "http", + Host: r.Host, + Path: r.URL.Path, + RawPath: r.URL.RawPath, + RawQuery: r.URL.RawQuery, + } + } + } + + outReq.RequestURI = "" + destinationHost := normalizeDestinationHost(outReq.URL.Host) + if destinationHost == "" { + destinationHost = normalizeDestinationHost(outReq.Host) + } + s.applyHeaderInjections(sourceIP, destinationHost, outReq.Header, false) + + resp, err := s.transport.RoundTrip(outReq) + if err != nil { + slog.Warn("egress proxy upstream request failed", "destination_host", destinationHost, "error", err) + http.Error(w, "proxy upstream error", http.StatusBadGateway) + return + } + defer resp.Body.Close() + + removeHopByHopHeaders(resp.Header) + for k, vv := range resp.Header { + for _, v := range vv { + w.Header().Add(k, v) + } + } + w.WriteHeader(resp.StatusCode) + _, _ = io.Copy(w, resp.Body) +} + +func (s *Service) handleConnect(w http.ResponseWriter, r *http.Request, sourceIP string) { + hj, ok := w.(http.Hijacker) + if !ok { + http.Error(w, "hijacking not supported", http.StatusInternalServerError) + return + } + + clientConn, _, err := hj.Hijack() + if err != nil { + http.Error(w, fmt.Sprintf("hijack failed: %v", err), http.StatusServiceUnavailable) + return + } + defer clientConn.Close() + + targetAuthority := strings.TrimSpace(r.Host) + targetHost := normalizeDestinationHost(targetAuthority) + if targetHost == "" { + return + } + + _, _ = io.WriteString(clientConn, "HTTP/1.1 200 Connection Established\r\n\r\n") + + cert, err := s.getOrCreateLeafCert(targetHost) + if err != nil { + return + } + + tlsConn := tls.Server(clientConn, &tls.Config{ + Certificates: []tls.Certificate{*cert}, + }) + if err := tlsConn.Handshake(); err != nil { + return + } + defer tlsConn.Close() + + reader := bufio.NewReader(tlsConn) + for { + req, err := http.ReadRequest(reader) + if err != nil { + if err == io.EOF { + return + } + return + } + + if req.URL == nil { + req.URL = &url.URL{} + } + if req.Host == "" { + req.Host = targetAuthority + } + req.URL.Scheme = "https" + req.URL.Host = targetAuthority + req.RequestURI = "" + req.Header = cloneHeader(req.Header) + removeHopByHopHeaders(req.Header) + s.applyHeaderInjections(sourceIP, targetHost, req.Header, true) + + resp, err := s.transport.RoundTrip(req) + if err != nil { + _, _ = io.WriteString(tlsConn, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } + + removeHopByHopHeaders(resp.Header) + if err := resp.Write(tlsConn); err != nil { + resp.Body.Close() + return + } + resp.Body.Close() + + if req.Close || resp.Close { + return + } + } +} + +func (s *Service) getOrCreateLeafCert(host string) (*tls.Certificate, error) { + s.mu.RLock() + cached := s.certCache[host] + s.mu.RUnlock() + if cached != nil { + return cached, nil + } + + cert, err := signHostCertificate(s.caCert, s.caKey, host) + if err != nil { + return nil, err + } + + s.mu.Lock() + defer s.mu.Unlock() + if existing := s.certCache[host]; existing != nil { + return existing, nil + } + if s.certCacheLimit > 0 && len(s.certCache) >= s.certCacheLimit && len(s.certCacheOrder) > 0 { + evictHost := s.certCacheOrder[0] + s.certCacheOrder = s.certCacheOrder[1:] + delete(s.certCache, evictHost) + } + s.certCache[host] = cert + s.certCacheOrder = append(s.certCacheOrder, host) + return cert, nil +} + +func (s *Service) applyHeaderInjections(sourceIP, destinationHost string, headers http.Header, isHTTPS bool) { + rules := s.resolveHeaderInjectRules(sourceIP, destinationHost, isHTTPS) + if len(rules) == 0 { + return + } + + for _, rule := range rules { + headers.Set(rule.headerName, rule.headerValue) + } +} + +func (s *Service) resolveHeaderInjectRules(sourceIP, destinationHost string, isHTTPS bool) []headerInjectRule { + if !isHTTPS { + return nil + } + + host := normalizeDestinationHost(destinationHost) + if host == "" { + return nil + } + + s.mu.RLock() + policy, ok := s.policiesBySourceIP[sourceIP] + s.mu.RUnlock() + if !ok { + return nil + } + + resolved := make([]headerInjectRule, 0, len(policy.headerInjectRules)) + for _, rule := range policy.headerInjectRules { + if rule.headerName == "" || rule.headerValue == "" { + continue + } + if !matchesAnyDomain(host, rule.domainMatchers) { + continue + } + resolved = append(resolved, rule) + } + return resolved +} + +var hopByHopHeaders = []string{ + "Connection", + "Proxy-Connection", + "Keep-Alive", + "Proxy-Authenticate", + "Proxy-Authorization", + "Te", + "Trailer", + "Transfer-Encoding", + "Upgrade", +} + +func removeHopByHopHeaders(h http.Header) { + for _, k := range hopByHopHeaders { + h.Del(k) + } +} + +func cloneHeader(src http.Header) http.Header { + dst := make(http.Header, len(src)) + for k, vv := range src { + copied := make([]string, len(vv)) + copy(copied, vv) + dst[k] = copied + } + return dst +} + +func sourceIPFromRemoteAddr(remoteAddr string) string { + host, _, err := net.SplitHostPort(remoteAddr) + if err != nil { + return remoteAddr + } + return host +} diff --git a/lib/egressproxy/service_test.go b/lib/egressproxy/service_test.go new file mode 100644 index 00000000..84aa487f --- /dev/null +++ b/lib/egressproxy/service_test.go @@ -0,0 +1,122 @@ +package egressproxy + +import ( + "context" + "errors" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeAllowedDomainPattern(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + in string + want string + shouldErr bool + }{ + {name: "exact host", in: "API.OpenAI.com", want: "api.openai.com"}, + {name: "exact ip", in: "127.0.0.1", want: "127.0.0.1"}, + {name: "single wildcard", in: "*.OpenAI.com", want: "*.openai.com"}, + {name: "reject empty", in: "", shouldErr: true}, + {name: "reject scheme", in: "https://api.openai.com", shouldErr: true}, + {name: "reject port", in: "api.openai.com:443", shouldErr: true}, + {name: "reject global wildcard", in: "*", shouldErr: true}, + {name: "reject multi wildcard", in: "*.*.openai.com", shouldErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NormalizeAllowedDomainPattern(tt.in) + if tt.shouldErr { + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, tt.want, got) + }) + } +} + +func TestDomainMatcherSingleLevelWildcard(t *testing.T) { + t.Parallel() + + matchers, err := compileDomainMatchers([]string{"*.openai.com"}) + require.NoError(t, err) + require.Len(t, matchers, 1) + + require.True(t, matchesAnyDomain("api.openai.com", matchers)) + require.False(t, matchesAnyDomain("openai.com", matchers)) + require.False(t, matchesAnyDomain("a.b.openai.com", matchers)) +} + +func TestApplyHeaderInjectionsHTTPSOnlyAndDomainGated(t *testing.T) { + t.Parallel() + + matchers, err := compileDomainMatchers([]string{"api.openai.com"}) + require.NoError(t, err) + + svc := &Service{ + policiesBySourceIP: map[string]sourcePolicy{ + "10.0.0.2": { + headerInjectRules: []headerInjectRule{ + { + headerName: "Authorization", + headerValue: "Bearer real-openai-key-123", + domainMatchers: matchers, + }, + }, + }, + }, + } + + httpsAllowed := http.Header{} + svc.applyHeaderInjections("10.0.0.2", "api.openai.com", httpsAllowed, true) + require.Equal(t, "Bearer real-openai-key-123", httpsAllowed.Get("Authorization")) + + httpsBlocked := http.Header{ + "Authorization": []string{"Bearer mock-OUTBOUND_OPENAI_KEY"}, + } + svc.applyHeaderInjections("10.0.0.2", "api.github.com", httpsBlocked, true) + require.Equal(t, "Bearer mock-OUTBOUND_OPENAI_KEY", httpsBlocked.Get("Authorization")) + + httpAllowedDomain := http.Header{ + "Authorization": []string{"Bearer mock-OUTBOUND_OPENAI_KEY"}, + } + svc.applyHeaderInjections("10.0.0.2", "api.openai.com", httpAllowedDomain, false) + require.Equal(t, "Bearer mock-OUTBOUND_OPENAI_KEY", httpAllowedDomain.Get("Authorization")) +} + +func TestHandleHTTPProxyRequest_DoesNotLeakUpstreamErrorDetails(t *testing.T) { + t.Parallel() + + sentinelErr := errors.New("dial failed: test internal network detail") + svc := &Service{ + transport: &http.Transport{ + DialContext: func(context.Context, string, string) (net.Conn, error) { + return nil, sentinelErr + }, + }, + policiesBySourceIP: map[string]sourcePolicy{}, + sourceIPByInstance: map[string]string{}, + } + + req := httptest.NewRequest(http.MethodGet, "http://api.example.com/v1/chat/completions", nil) + rec := httptest.NewRecorder() + + svc.ServeHTTP(rec, req) + + resp := rec.Result() + require.Equal(t, http.StatusBadGateway, resp.StatusCode) + body := rec.Body.String() + require.Contains(t, body, "proxy upstream error") + require.NotContains(t, body, sentinelErr.Error()) + require.NotContains(t, body, "dial failed") + require.False(t, strings.Contains(body, "internal network detail")) +} diff --git a/lib/egressproxy/types.go b/lib/egressproxy/types.go new file mode 100644 index 00000000..29c2e838 --- /dev/null +++ b/lib/egressproxy/types.go @@ -0,0 +1,39 @@ +package egressproxy + +import "errors" + +const ( + DefaultListenPort = 18080 +) + +var ( + ErrGatewayMismatch = errors.New("egress proxy already initialized with different gateway") +) + +// InstanceConfig defines per-instance proxy behavior. +type InstanceConfig struct { + InstanceID string + SourceIP string + TAPDevice string + BlockAllTCPEgress bool + HeaderInjectRules []HeaderInjectRuleConfig +} + +// HeaderInjectRuleConfig defines one host-managed outbound header injection policy. +type HeaderInjectRuleConfig struct { + HeaderName string + HeaderValue string + AllowedDomains []string // optional exact or *.example.com patterns; empty means allow all +} + +// ServiceOptions customizes service construction (primarily for tests). +type ServiceOptions struct { + AdditionalRootCAPEM []string +} + +// GuestConfig is injected into guest config.json when proxy mode is enabled. +type GuestConfig struct { + Enabled bool `json:"enabled"` + ProxyURL string `json:"proxy_url"` + CACertPEM string `json:"ca_cert_pem"` +} diff --git a/lib/instances/configdisk.go b/lib/instances/configdisk.go index 4e17051c..f85fa7af 100644 --- a/lib/instances/configdisk.go +++ b/lib/instances/configdisk.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/images" "github.com/kernel/hypeman/lib/network" "github.com/kernel/hypeman/lib/vmconfig" @@ -16,7 +17,7 @@ import ( // createConfigDisk generates an ext4 disk with instance configuration. // The disk contains /config.json read by the guest init binary. -func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig) error { +func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig, proxyCfg *egressproxy.GuestConfig) error { // Create temporary directory for config files tmpDir, err := os.MkdirTemp("", "hypeman-config-*") if err != nil { @@ -25,7 +26,7 @@ func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInf defer os.RemoveAll(tmpDir) // Generate config.json - cfg := m.buildGuestConfig(ctx, inst, imageInfo, netConfig) + cfg := m.buildGuestConfig(ctx, inst, imageInfo, netConfig, proxyCfg) configData, err := json.MarshalIndent(cfg, "", " ") if err != nil { return fmt.Errorf("marshal config: %w", err) @@ -46,7 +47,7 @@ func (m *manager) createConfigDisk(ctx context.Context, inst *Instance, imageInf } // buildGuestConfig creates the vmconfig.Config struct for the guest init binary. -func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig) *vmconfig.Config { +func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInfo *images.Image, netConfig *network.NetworkConfig, proxyCfg *egressproxy.GuestConfig) *vmconfig.Config { // Use instance overrides if set, otherwise fall back to image defaults // (like docker run overriding CMD) entrypoint := imageInfo.Entrypoint @@ -79,6 +80,23 @@ func (m *manager) buildGuestConfig(ctx context.Context, inst *Instance, imageInf cfg.GuestDNS = netConfig.DNS } + if proxyCfg != nil && proxyCfg.Enabled { + cfg.EgressProxy = &vmconfig.EgressProxyConfig{ + Enabled: true, + ProxyURL: proxyCfg.ProxyURL, + CACertPEM: proxyCfg.CACertPEM, + } + if len(inst.Credentials) > 0 { + for credentialName := range inst.Credentials { + cfg.Env[credentialName] = mockValueForCredential(credentialName) + } + } + cfg.Env["HTTP_PROXY"] = proxyCfg.ProxyURL + cfg.Env["HTTPS_PROXY"] = proxyCfg.ProxyURL + cfg.Env["http_proxy"] = proxyCfg.ProxyURL + cfg.Env["https_proxy"] = proxyCfg.ProxyURL + } + // Volume mounts // Volumes are attached as /dev/vdd, /dev/vde, etc. (after vda=rootfs, vdb=overlay, vdc=config) deviceIdx := 0 diff --git a/lib/instances/create.go b/lib/instances/create.go index a5a2aa21..d423dc0e 100644 --- a/lib/instances/create.go +++ b/lib/instances/create.go @@ -8,6 +8,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/guestmemory" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" @@ -84,7 +85,7 @@ func (m *manager) createInstance( } // 1. Validate request - if err := validateCreateRequest(req); err != nil { + if err := validateCreateRequest(&req); err != nil { log.ErrorContext(ctx, "invalid create request", "error", err) return nil, err } @@ -319,6 +320,8 @@ func (m *manager) createInstance( Env: req.Env, Tags: tags.Clone(req.Tags), NetworkEnabled: req.NetworkEnabled, + NetworkEgress: cloneNetworkEgressPolicy(req.NetworkEgress), + Credentials: cloneCredentialPolicies(req.Credentials), CreatedAt: time.Now(), StartedAt: nil, StoppedAt: nil, @@ -425,8 +428,19 @@ func (m *manager) createInstance( // 16. Create config disk (needs Instance for buildVMConfig) inst := &Instance{StoredMetadata: *stored} + var proxyGuestConfig *egressproxy.GuestConfig + proxyGuestConfig, err = m.maybeRegisterEgressProxy(ctx, stored, netConfig) + if err != nil { + log.ErrorContext(ctx, "failed to configure egress proxy", "instance_id", id, "error", err) + return nil, fmt.Errorf("configure egress proxy: %w", err) + } + if proxyGuestConfig != nil { + cu.Add(func() { + m.unregisterEgressProxyInstance(ctx, id) + }) + } log.DebugContext(ctx, "creating config disk", "instance_id", id) - if err := m.createConfigDisk(ctx, inst, imageInfo, netConfig); err != nil { + if err := m.createConfigDisk(ctx, inst, imageInfo, netConfig, proxyGuestConfig); err != nil { log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) return nil, fmt.Errorf("create config disk: %w", err) } @@ -476,8 +490,12 @@ func (m *manager) createInstance( return &finalInst, nil } -// validateCreateRequest validates the create instance request -func validateCreateRequest(req CreateInstanceRequest) error { +// validateCreateRequest validates the create instance request. +// The request is mutated in-place to persist normalized egress/credential policy fields. +func validateCreateRequest(req *CreateInstanceRequest) error { + if req == nil { + return fmt.Errorf("%w: request is required", ErrInvalidRequest) + } if err := validateInstanceName(req.Name); err != nil { return err } @@ -496,6 +514,29 @@ func validateCreateRequest(req CreateInstanceRequest) error { if req.Vcpus < 0 { return fmt.Errorf("vcpus cannot be negative") } + if req.NetworkEgress != nil && req.NetworkEgress.Enabled { + if !req.NetworkEnabled { + return fmt.Errorf("%w: network.egress requires network.enabled=true", ErrInvalidRequest) + } + mode, err := normalizeEgressEnforcementMode(req.NetworkEgress.EnforcementMode) + if err != nil { + return err + } + req.NetworkEgress.EnforcementMode = mode + } + normalizedCredentials, err := normalizeCredentialPolicies(req.Credentials) + if err != nil { + return err + } + req.Credentials = normalizedCredentials + if len(normalizedCredentials) > 0 { + if req.NetworkEgress == nil || !req.NetworkEgress.Enabled { + return fmt.Errorf("%w: credentials require network.egress.enabled=true", ErrInvalidRequest) + } + if err := validateCredentialEnvBindings(normalizedCredentials, req.Env); err != nil { + return err + } + } if err := tags.Validate(req.Tags); err != nil { return fmt.Errorf("%w: %v", ErrInvalidRequest, err) } diff --git a/lib/instances/create_egress_proxy_test.go b/lib/instances/create_egress_proxy_test.go new file mode 100644 index 00000000..f7ef69bd --- /dev/null +++ b/lib/instances/create_egress_proxy_test.go @@ -0,0 +1,242 @@ +package instances + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestValidateCreateRequest_NetworkEgressRequiresNetwork(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: false, + NetworkEgress: &NetworkEgressPolicy{Enabled: true}, + } + + err := validateCreateRequest(&req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "network.egress requires network.enabled=true") +} + +func TestValidateCreateRequest_CredentialsRequireNetworkEgress(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-credentials", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + Credentials: map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{ + {As: CredentialInjectAs{Header: "Authorization", Format: "Bearer ${value}"}}, + }, + }, + }, + } + + err := validateCreateRequest(&req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "credentials require network.egress.enabled=true") +} + +func TestValidateCreateRequest_CredentialSourceEnvMustExist(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-credentials", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + NetworkEgress: &NetworkEgressPolicy{Enabled: true}, + Env: map[string]string{}, + Credentials: map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{ + {As: CredentialInjectAs{Header: "Authorization", Format: "Bearer ${value}"}}, + }, + }, + }, + } + + err := validateCreateRequest(&req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "must be present in env") +} + +func TestValidateCreateRequest_CredentialSourceEnvMustBeNonEmpty(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-credentials", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + NetworkEgress: &NetworkEgressPolicy{Enabled: true}, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": " ", + }, + Credentials: map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{ + {As: CredentialInjectAs{Header: "Authorization", Format: "Bearer ${value}"}}, + }, + }, + }, + } + + err := validateCreateRequest(&req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "must be non-empty") +} + +func TestValidateCreateRequest_RejectsInvalidEgressEnforcementMode(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + NetworkEgress: &NetworkEgressPolicy{ + Enabled: true, + EnforcementMode: EgressEnforcementMode("bogus"), + }, + } + + err := validateCreateRequest(&req) + require.Error(t, err) + assert.ErrorIs(t, err, ErrInvalidRequest) + assert.Contains(t, err.Error(), "invalid network.egress.enforcement.mode") +} + +func TestValidateCreateRequest_AllowsHTTPHTTPSOnlyEgressMode(t *testing.T) { + t.Parallel() + + cfg := &NetworkEgressPolicy{ + Enabled: true, + EnforcementMode: EgressEnforcementModeHTTPHTTPSOnly, + } + req := CreateInstanceRequest{ + Name: "test-egress", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + NetworkEgress: cfg, + } + + err := validateCreateRequest(&req) + require.NoError(t, err) + assert.Equal(t, EgressEnforcementModeHTTPHTTPSOnly, cfg.EnforcementMode) +} + +func TestValidateCreateRequest_NormalizesCredentialsInPlace(t *testing.T) { + t.Parallel() + + req := CreateInstanceRequest{ + Name: "test-normalize-creds", + Image: "docker.io/library/alpine:latest", + NetworkEnabled: true, + NetworkEgress: &NetworkEgressPolicy{ + Enabled: true, + }, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-key", + }, + Credentials: map[string]CredentialPolicy{ + " OUTBOUND_OPENAI_KEY ": { + Source: CredentialSource{Env: " OUTBOUND_OPENAI_KEY "}, + Inject: []CredentialInjectRule{ + { + Hosts: []string{" API.OpenAI.com ", "api.openai.com"}, + As: CredentialInjectAs{ + Header: " Authorization ", + Format: " Bearer ${value} ", + }, + }, + }, + }, + }, + } + + err := validateCreateRequest(&req) + require.NoError(t, err) + + policy, ok := req.Credentials["OUTBOUND_OPENAI_KEY"] + require.True(t, ok) + _, hasRawKey := req.Credentials[" OUTBOUND_OPENAI_KEY "] + require.False(t, hasRawKey) + require.Equal(t, "OUTBOUND_OPENAI_KEY", policy.Source.Env) + require.Len(t, policy.Inject, 1) + require.Equal(t, []string{"api.openai.com"}, policy.Inject[0].Hosts) + require.Equal(t, "Authorization", policy.Inject[0].As.Header) + require.Equal(t, "Bearer ${value}", policy.Inject[0].As.Format) +} + +func TestNormalizeCredentialPolicies_RejectsInvalidHostPattern(t *testing.T) { + t.Parallel() + + _, err := normalizeCredentialPolicies(map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{ + { + Hosts: []string{"https://api.openai.com"}, + As: CredentialInjectAs{Header: "Authorization", Format: "Bearer ${value}"}, + }, + }, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "invalid host pattern") +} + +func TestNormalizeCredentialPolicies_DedupesAndNormalizesHosts(t *testing.T) { + t.Parallel() + + normalized, err := normalizeCredentialPolicies(map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{ + { + Hosts: []string{ + " API.OpenAI.com ", + "*.OpenAI.com", + "api.openai.com", + }, + As: CredentialInjectAs{Header: "Authorization", Format: "Bearer ${value}"}, + }, + }, + }, + }) + require.NoError(t, err) + policy := normalized["OUTBOUND_OPENAI_KEY"] + require.Len(t, policy.Inject, 1) + assert.Equal(t, []string{"api.openai.com", "*.openai.com"}, policy.Inject[0].Hosts) +} + +func TestNormalizeCredentialPolicies_RejectsMissingValueTemplate(t *testing.T) { + t.Parallel() + + _, err := normalizeCredentialPolicies(map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{ + { + As: CredentialInjectAs{Header: "Authorization", Format: "Bearer token"}, + }, + }, + }, + }) + require.Error(t, err) + assert.Contains(t, err.Error(), "must include ${value}") +} diff --git a/lib/instances/delete.go b/lib/instances/delete.go index b54d5b3c..21fbfaab 100644 --- a/lib/instances/delete.go +++ b/lib/instances/delete.go @@ -76,6 +76,7 @@ func (m *manager) deleteInstance( // 6. Release network allocation if inst.NetworkEnabled { + m.unregisterEgressProxyInstance(ctx, id) log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil { // Log error but continue with cleanup diff --git a/lib/instances/egress_proxy.go b/lib/instances/egress_proxy.go new file mode 100644 index 00000000..8800d72e --- /dev/null +++ b/lib/instances/egress_proxy.go @@ -0,0 +1,249 @@ +package instances + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/kernel/hypeman/lib/egressproxy" + "github.com/kernel/hypeman/lib/network" +) + +const mockSecretPrefix = "mock-" + +func cloneNetworkEgressPolicy(cfg *NetworkEgressPolicy) *NetworkEgressPolicy { + if cfg == nil { + return nil + } + return &NetworkEgressPolicy{ + Enabled: cfg.Enabled, + EnforcementMode: cfg.EnforcementMode, + } +} + +func cloneCredentialPolicies(in map[string]CredentialPolicy) map[string]CredentialPolicy { + if len(in) == 0 { + return nil + } + out := make(map[string]CredentialPolicy, len(in)) + for name, policy := range in { + cloned := CredentialPolicy{ + Source: CredentialSource{Env: policy.Source.Env}, + } + if len(policy.Inject) > 0 { + cloned.Inject = make([]CredentialInjectRule, 0, len(policy.Inject)) + for _, inject := range policy.Inject { + next := CredentialInjectRule{ + As: CredentialInjectAs{ + Header: inject.As.Header, + Format: inject.As.Format, + }, + } + if len(inject.Hosts) > 0 { + next.Hosts = append([]string(nil), inject.Hosts...) + } + cloned.Inject = append(cloned.Inject, next) + } + } + out[name] = cloned + } + return out +} + +func mockValueForCredential(name string) string { + return mockSecretPrefix + name +} + +func normalizeEgressEnforcementMode(mode EgressEnforcementMode) (EgressEnforcementMode, error) { + trimmed := strings.TrimSpace(string(mode)) + switch EgressEnforcementMode(trimmed) { + case "", EgressEnforcementModeAll: + return EgressEnforcementModeAll, nil + case EgressEnforcementModeHTTPHTTPSOnly: + return EgressEnforcementModeHTTPHTTPSOnly, nil + default: + return "", fmt.Errorf("%w: invalid network.egress.enforcement.mode %q", ErrInvalidRequest, trimmed) + } +} + +func normalizeCredentialPolicies(in map[string]CredentialPolicy) (map[string]CredentialPolicy, error) { + if len(in) == 0 { + return nil, nil + } + + out := make(map[string]CredentialPolicy, len(in)) + for rawCredentialName, rawPolicy := range in { + credentialName := strings.TrimSpace(rawCredentialName) + if credentialName == "" { + return nil, fmt.Errorf("%w: credentials key must be non-empty", ErrInvalidRequest) + } + if _, exists := out[credentialName]; exists { + return nil, fmt.Errorf("%w: duplicate credentials key %q after normalization", ErrInvalidRequest, credentialName) + } + + sourceEnv := strings.TrimSpace(rawPolicy.Source.Env) + if sourceEnv == "" { + return nil, fmt.Errorf("%w: credentials[%q].source.env is required", ErrInvalidRequest, credentialName) + } + if len(rawPolicy.Inject) == 0 { + return nil, fmt.Errorf("%w: credentials[%q].inject must have at least one rule", ErrInvalidRequest, credentialName) + } + + policy := CredentialPolicy{ + Source: CredentialSource{Env: sourceEnv}, + Inject: make([]CredentialInjectRule, 0, len(rawPolicy.Inject)), + } + for idx, rawInject := range rawPolicy.Inject { + header := strings.TrimSpace(rawInject.As.Header) + if header == "" { + return nil, fmt.Errorf("%w: credentials[%q].inject[%d].as.header is required", ErrInvalidRequest, credentialName, idx) + } + format := strings.TrimSpace(rawInject.As.Format) + if format == "" { + return nil, fmt.Errorf("%w: credentials[%q].inject[%d].as.format is required", ErrInvalidRequest, credentialName, idx) + } + if !strings.Contains(format, "${value}") { + return nil, fmt.Errorf("%w: credentials[%q].inject[%d].as.format must include ${value}", ErrInvalidRequest, credentialName, idx) + } + + hosts, err := normalizeCredentialHosts(rawInject.Hosts) + if err != nil { + return nil, fmt.Errorf("%w: credentials[%q].inject[%d].hosts: %v", ErrInvalidRequest, credentialName, idx, err) + } + policy.Inject = append(policy.Inject, CredentialInjectRule{ + Hosts: hosts, + As: CredentialInjectAs{ + Header: header, + Format: format, + }, + }) + } + + out[credentialName] = policy + } + + return out, nil +} + +func normalizeCredentialHosts(in []string) ([]string, error) { + if len(in) == 0 { + return nil, nil + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, rawPattern := range in { + normalized, err := egressproxy.NormalizeAllowedDomainPattern(rawPattern) + if err != nil { + return nil, fmt.Errorf("invalid host pattern %q: %w", rawPattern, err) + } + if _, ok := seen[normalized]; ok { + continue + } + seen[normalized] = struct{}{} + out = append(out, normalized) + } + return out, nil +} + +func validateCredentialEnvBindings(credentials map[string]CredentialPolicy, env map[string]string) error { + for credentialName, policy := range credentials { + real, ok := env[policy.Source.Env] + if !ok { + return fmt.Errorf("%w: credentials[%q].source.env %q must be present in env", ErrInvalidRequest, credentialName, policy.Source.Env) + } + if strings.TrimSpace(real) == "" { + return fmt.Errorf("%w: env var %q must be non-empty when referenced by credentials[%q]", ErrInvalidRequest, policy.Source.Env, credentialName) + } + } + return nil +} + +func buildEgressProxyInjectRules(egressPolicy *NetworkEgressPolicy, credentials map[string]CredentialPolicy, env map[string]string) []egressproxy.HeaderInjectRuleConfig { + if egressPolicy == nil || !egressPolicy.Enabled || len(credentials) == 0 { + return nil + } + names := make([]string, 0, len(credentials)) + for name := range credentials { + names = append(names, name) + } + sort.Strings(names) + + out := make([]egressproxy.HeaderInjectRuleConfig, 0) + for _, credentialName := range names { + policy := credentials[credentialName] + real := strings.TrimSpace(env[policy.Source.Env]) + if real == "" { + continue + } + + for _, inject := range policy.Inject { + rule := egressproxy.HeaderInjectRuleConfig{ + HeaderName: inject.As.Header, + HeaderValue: strings.ReplaceAll(inject.As.Format, "${value}", real), + } + if len(inject.Hosts) > 0 { + rule.AllowedDomains = append([]string(nil), inject.Hosts...) + } + out = append(out, rule) + } + } + if len(out) == 0 { + return nil + } + return out +} + +func (m *manager) getOrCreateEgressProxyService() (*egressproxy.Service, error) { + m.egressProxyMu.Lock() + defer m.egressProxyMu.Unlock() + + if m.egressProxy != nil { + return m.egressProxy, nil + } + + svc, err := egressproxy.NewServiceWithOptions(m.paths.DataDir(), egressproxy.DefaultListenPort, m.egressProxyServiceOptions) + if err != nil { + return nil, err + } + m.egressProxy = svc + return svc, nil +} + +func (m *manager) maybeRegisterEgressProxy(ctx context.Context, stored *StoredMetadata, netConfig *network.NetworkConfig) (*egressproxy.GuestConfig, error) { + if stored == nil || stored.NetworkEgress == nil || !stored.NetworkEgress.Enabled { + return nil, nil + } + if !stored.NetworkEnabled || netConfig == nil { + return nil, fmt.Errorf("network.egress requires network.enabled=true") + } + + svc, err := m.getOrCreateEgressProxyService() + if err != nil { + return nil, fmt.Errorf("create egress proxy service: %w", err) + } + + guestCfg, err := svc.RegisterInstance(ctx, netConfig.Gateway, egressproxy.InstanceConfig{ + InstanceID: stored.Id, + SourceIP: netConfig.IP, + TAPDevice: netConfig.TAPDevice, + BlockAllTCPEgress: stored.NetworkEgress.EnforcementMode != EgressEnforcementModeHTTPHTTPSOnly, + HeaderInjectRules: buildEgressProxyInjectRules(stored.NetworkEgress, stored.Credentials, stored.Env), + }) + if err != nil { + return nil, fmt.Errorf("register instance with egress proxy: %w", err) + } + + return &guestCfg, nil +} + +func (m *manager) unregisterEgressProxyInstance(ctx context.Context, instanceID string) { + _ = ctx + m.egressProxyMu.Lock() + svc := m.egressProxy + m.egressProxyMu.Unlock() + if svc == nil { + return + } + svc.UnregisterInstance(context.Background(), instanceID) +} diff --git a/lib/instances/egress_proxy_integration_test.go b/lib/instances/egress_proxy_integration_test.go new file mode 100644 index 00000000..96fdd86c --- /dev/null +++ b/lib/instances/egress_proxy_integration_test.go @@ -0,0 +1,192 @@ +package instances + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "github.com/kernel/hypeman/lib/egressproxy" + "github.com/kernel/hypeman/lib/images" + "github.com/stretchr/testify/require" +) + +func TestEgressProxyRewritesHTTPSHeaders(t *testing.T) { + requireKVMAccess(t) + + manager, _ := setupTestManager(t) + ctx := context.Background() + + caPEM, cert := mustGenerateTLSChain(t, []string{"localhost"}) + manager.egressProxyServiceOptions = egressproxy.ServiceOptions{ + AdditionalRootCAPEM: []string{caPEM}, + } + + target := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, r.Header.Get("Authorization")) + })) + target.TLS = &tls.Config{Certificates: []tls.Certificate{cert}} + target.StartTLS() + defer target.Close() + targetHostPort := strings.TrimPrefix(target.URL, "https://") + targetHost, targetPort, err := net.SplitHostPort(targetHostPort) + require.NoError(t, err) + + imageRef := integrationTestImageRef(t, "docker.io/library/nginx:alpine") + t.Logf("Pulling %s image...", imageRef) + created, err := manager.imageManager.CreateImage(ctx, images.CreateImageRequest{Name: imageRef}) + require.NoError(t, err) + + for i := 0; i < 120; i++ { + img, err := manager.imageManager.GetImage(ctx, created.Name) + if err == nil && img.Status == images.StatusReady { + break + } + time.Sleep(1 * time.Second) + } + img, err := manager.imageManager.GetImage(ctx, created.Name) + require.NoError(t, err) + require.Equal(t, images.StatusReady, img.Status) + + require.NoError(t, manager.systemManager.EnsureSystemFiles(ctx)) + require.NoError(t, manager.networkManager.Initialize(ctx, nil)) + + inst, err := manager.CreateInstance(ctx, CreateInstanceRequest{ + Name: "test-egress-proxy", + Image: imageRef, + Size: 2 * 1024 * 1024 * 1024, + HotplugSize: 512 * 1024 * 1024, + OverlaySize: 5 * 1024 * 1024 * 1024, + Vcpus: 1, + NetworkEnabled: true, + NetworkEgress: &NetworkEgressPolicy{ + Enabled: true, + }, + Credentials: map[string]CredentialPolicy{ + "OUTBOUND_OPENAI_KEY": { + Source: CredentialSource{Env: "OUTBOUND_OPENAI_KEY"}, + Inject: []CredentialInjectRule{ + { + Hosts: []string{"127.0.0.1"}, + As: CredentialInjectAs{ + Header: "Authorization", + Format: "Bearer ${value}", + }, + }, + }, + }, + }, + Env: map[string]string{ + "OUTBOUND_OPENAI_KEY": "real-openai-key-123", + }, + Entrypoint: []string{"/bin/sh", "-lc"}, + Cmd: []string{"sleep 3600"}, + }) + require.NoError(t, err) + + deleted := false + t.Cleanup(func() { + if !deleted { + _ = manager.DeleteInstance(context.Background(), inst.Id) + } + }) + + require.NoError(t, waitForVMReady(ctx, inst.SocketPath, 10*time.Second)) + require.NoError(t, waitForLogMessage(ctx, manager, inst.Id, "[guest-agent] listening", 45*time.Second)) + + envOutput, envExitCode, err := execCommand(ctx, inst, "sh", "-lc", "printf '%s' \"$OUTBOUND_OPENAI_KEY\"") + require.NoError(t, err) + require.Equal(t, 0, envExitCode) + require.Equal(t, "mock-OUTBOUND_OPENAI_KEY", envOutput) + + allowedCmd := fmt.Sprintf( + "NO_PROXY= no_proxy= curl -k -sS https://%s:%s", + targetHost, targetPort, + ) + output, exitCode, err := execCommand(ctx, inst, "sh", "-lc", allowedCmd) + require.NoError(t, err) + require.Equal(t, 0, exitCode, "curl output: %s", output) + require.Contains(t, output, "Bearer real-openai-key-123") + + blockedCmd := fmt.Sprintf( + "NO_PROXY= no_proxy= curl -k -sS https://localhost:%s", + targetPort, + ) + blockedOutput, blockedExitCode, err := execCommand(ctx, inst, "sh", "-lc", blockedCmd) + require.NoError(t, err) + require.Equal(t, 0, blockedExitCode, "curl output: %s", blockedOutput) + require.Equal(t, "", blockedOutput) + + require.NoError(t, manager.DeleteInstance(ctx, inst.Id)) + deleted = true +} + +func mustGenerateTLSChain(t *testing.T, dnsNames []string) (string, tls.Certificate) { + t.Helper() + + caKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + serialLimit := new(big.Int).Lsh(big.NewInt(1), 128) + caSerial, err := rand.Int(rand.Reader, serialLimit) + require.NoError(t, err) + + now := time.Now() + caTemplate := &x509.Certificate{ + SerialNumber: caSerial, + Subject: pkix.Name{ + CommonName: "egress-proxy-test-ca", + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, + BasicConstraintsValid: true, + IsCA: true, + MaxPathLen: 1, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTemplate, caTemplate, &caKey.PublicKey, caKey) + require.NoError(t, err) + + caCert, err := x509.ParseCertificate(caDER) + require.NoError(t, err) + + serverKey, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + serverSerial, err := rand.Int(rand.Reader, serialLimit) + require.NoError(t, err) + + serverTemplate := &x509.Certificate{ + SerialNumber: serverSerial, + Subject: pkix.Name{ + CommonName: dnsNames[0], + }, + NotBefore: now.Add(-1 * time.Hour), + NotAfter: now.Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + DNSNames: dnsNames, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + SubjectKeyId: []byte{1, 2, 3, 4}, + } + serverDER, err := x509.CreateCertificate(rand.Reader, serverTemplate, caCert, &serverKey.PublicKey, caKey) + require.NoError(t, err) + + serverCertPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: serverDER}) + serverKeyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(serverKey)}) + cert, err := tls.X509KeyPair(serverCertPEM, serverKeyPEM) + require.NoError(t, err) + + caPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}) + return string(caPEM), cert +} diff --git a/lib/instances/fork.go b/lib/instances/fork.go index 108381f8..d68b8c22 100644 --- a/lib/instances/fork.go +++ b/lib/instances/fork.go @@ -467,6 +467,12 @@ func cloneStoredMetadataForFork(src StoredMetadata) StoredMetadata { dst.Env[k] = v } } + if src.NetworkEgress != nil { + dst.NetworkEgress = cloneNetworkEgressPolicy(src.NetworkEgress) + } + if src.Credentials != nil { + dst.Credentials = cloneCredentialPolicies(src.Credentials) + } if src.Tags != nil { dst.Tags = make(map[string]string, len(src.Tags)) for k, v := range src.Tags { diff --git a/lib/instances/manager.go b/lib/instances/manager.go index 611a60ed..8305a0c5 100644 --- a/lib/instances/manager.go +++ b/lib/instances/manager.go @@ -7,6 +7,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/guestmemory" "github.com/kernel/hypeman/lib/hypervisor" "github.com/kernel/hypeman/lib/images" @@ -71,19 +72,22 @@ type ResourceValidator interface { } type manager struct { - paths *paths.Paths - imageManager images.Manager - systemManager system.Manager - networkManager network.Manager - deviceManager devices.Manager - volumeManager volumes.Manager - limits ResourceLimits - resourceValidator ResourceValidator // Optional validator for aggregate resource limits - instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks - bootMarkerScans sync.Map // map[string]time.Time next allowed boot-marker rescan - hostTopology *HostTopology // Cached host CPU topology - metrics *Metrics - now func() time.Time + paths *paths.Paths + imageManager images.Manager + systemManager system.Manager + networkManager network.Manager + deviceManager devices.Manager + volumeManager volumes.Manager + limits ResourceLimits + resourceValidator ResourceValidator // Optional validator for aggregate resource limits + instanceLocks sync.Map // map[string]*sync.RWMutex - per-instance locks + bootMarkerScans sync.Map // map[string]time.Time next allowed boot-marker rescan + hostTopology *HostTopology // Cached host CPU topology + metrics *Metrics + now func() time.Time + egressProxy *egressproxy.Service + egressProxyServiceOptions egressproxy.ServiceOptions + egressProxyMu sync.Mutex // Hypervisor support vmStarters map[hypervisor.Type]hypervisor.VMStarter diff --git a/lib/instances/restore.go b/lib/instances/restore.go index 83842b74..369525fe 100644 --- a/lib/instances/restore.go +++ b/lib/instances/restore.go @@ -75,10 +75,15 @@ func (m *manager) restoreInstance( } var allocatedNet *network.Allocation + proxyRegistered := false releaseNetwork := func() { if !stored.NetworkEnabled { return } + if proxyRegistered { + m.unregisterEgressProxyInstance(ctx, id) + proxyRegistered = false + } if allocatedNet != nil { if err := m.networkManager.ReleaseAllocation(ctx, allocatedNet); err != nil { log.WarnContext(ctx, "failed to release allocated network", "instance_id", id, "error", err) @@ -126,6 +131,7 @@ func (m *manager) restoreInstance( TAPDevice: netConfig.TAPDevice, Gateway: netConfig.Gateway, Netmask: netConfig.Netmask, + DNS: netConfig.DNS, } stored.IP = netConfig.IP stored.MAC = netConfig.MAC @@ -169,6 +175,43 @@ func (m *manager) restoreInstance( } } + // 4b. Register proxy/enforcement once network identity is active. + // Restore config disk refresh is only required for instances using network egress mediation. + if requiresRestoreConfigDiskRefresh(stored) { + alloc, allocErr := m.networkManager.GetAllocation(ctx, id) + if allocErr != nil { + log.ErrorContext(ctx, "failed to fetch allocation for proxy setup", "instance_id", id, "error", allocErr) + releaseNetwork() + return nil, fmt.Errorf("get network allocation for proxy setup: %w", allocErr) + } + if alloc == nil { + log.ErrorContext(ctx, "missing allocation for proxy setup", "instance_id", id) + releaseNetwork() + return nil, fmt.Errorf("get network allocation for proxy setup: allocation not found") + } + + proxyCfg := networkConfigFromAllocation(alloc) + proxyGuestConfig, err := m.maybeRegisterEgressProxy(ctx, stored, proxyCfg) + if err != nil { + log.ErrorContext(ctx, "failed to configure egress proxy", "instance_id", id, "error", err) + releaseNetwork() + return nil, fmt.Errorf("configure egress proxy: %w", err) + } + imageInfo, err := m.imageManager.GetImage(ctx, stored.Image) + if err != nil { + log.ErrorContext(ctx, "failed to load image for config disk refresh", "instance_id", id, "image", stored.Image, "error", err) + releaseNetwork() + return nil, fmt.Errorf("get image for restore config disk: %w", err) + } + instForConfig := &Instance{StoredMetadata: *stored} + if err := m.createConfigDisk(ctx, instForConfig, imageInfo, proxyCfg, proxyGuestConfig); err != nil { + log.ErrorContext(ctx, "failed to refresh config disk for restore", "instance_id", id, "error", err) + releaseNetwork() + return nil, fmt.Errorf("refresh restore config disk: %w", err) + } + proxyRegistered = true + } + // 5. Transition: Standby → Paused (start hypervisor + restore) var restoreSpan trace.Span if m.metrics != nil && m.metrics.tracer != nil { @@ -311,6 +354,24 @@ func reconfigureGuestNetwork(ctx context.Context, stored *StoredMetadata, alloc return nil } +func networkConfigFromAllocation(alloc *network.Allocation) *network.NetworkConfig { + if alloc == nil { + return nil + } + return &network.NetworkConfig{ + IP: alloc.IP, + MAC: alloc.MAC, + Gateway: alloc.Gateway, + Netmask: alloc.Netmask, + DNS: alloc.DNS, + TAPDevice: alloc.TAPDevice, + } +} + +func requiresRestoreConfigDiskRefresh(stored *StoredMetadata) bool { + return stored != nil && stored.NetworkEnabled && stored.NetworkEgress != nil && stored.NetworkEgress.Enabled +} + func netmaskToPrefix(mask string) (int, error) { ip := net.ParseIP(mask).To4() if ip == nil { diff --git a/lib/instances/restore_egress_test.go b/lib/instances/restore_egress_test.go new file mode 100644 index 00000000..3d4423f9 --- /dev/null +++ b/lib/instances/restore_egress_test.go @@ -0,0 +1,52 @@ +package instances + +import ( + "testing" + + "github.com/kernel/hypeman/lib/network" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestNetworkConfigFromAllocation_PreservesDNS(t *testing.T) { + t.Parallel() + + alloc := &network.Allocation{ + IP: "192.168.1.10", + MAC: "02:00:00:00:00:01", + Gateway: "192.168.1.1", + Netmask: "255.255.255.0", + DNS: "1.1.1.1", + TAPDevice: "hype-abcd1234", + } + + cfg := networkConfigFromAllocation(alloc) + require.NotNil(t, cfg) + assert.Equal(t, alloc.IP, cfg.IP) + assert.Equal(t, alloc.MAC, cfg.MAC) + assert.Equal(t, alloc.Gateway, cfg.Gateway) + assert.Equal(t, alloc.Netmask, cfg.Netmask) + assert.Equal(t, alloc.DNS, cfg.DNS) + assert.Equal(t, alloc.TAPDevice, cfg.TAPDevice) +} + +func TestRequiresRestoreConfigDiskRefresh(t *testing.T) { + t.Parallel() + + assert.False(t, requiresRestoreConfigDiskRefresh(nil)) + assert.False(t, requiresRestoreConfigDiskRefresh(&StoredMetadata{ + NetworkEnabled: false, + NetworkEgress: &NetworkEgressPolicy{Enabled: true}, + })) + assert.False(t, requiresRestoreConfigDiskRefresh(&StoredMetadata{ + NetworkEnabled: true, + })) + assert.False(t, requiresRestoreConfigDiskRefresh(&StoredMetadata{ + NetworkEnabled: true, + NetworkEgress: &NetworkEgressPolicy{Enabled: false}, + })) + assert.True(t, requiresRestoreConfigDiskRefresh(&StoredMetadata{ + NetworkEnabled: true, + NetworkEgress: &NetworkEgressPolicy{Enabled: true}, + })) +} diff --git a/lib/instances/standby.go b/lib/instances/standby.go index 66949db9..51ef7b85 100644 --- a/lib/instances/standby.go +++ b/lib/instances/standby.go @@ -136,6 +136,7 @@ func (m *manager) standbyInstance( // TAP devices with explicit Owner/Group fields do NOT auto-delete when VMM exits // They must be explicitly deleted if inst.NetworkEnabled { + m.unregisterEgressProxyInstance(ctx, id) log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil { // Log error but continue - snapshot was created successfully diff --git a/lib/instances/start.go b/lib/instances/start.go index 7d5ab265..0345e395 100644 --- a/lib/instances/start.go +++ b/lib/instances/start.go @@ -6,6 +6,7 @@ import ( "time" "github.com/kernel/hypeman/lib/devices" + "github.com/kernel/hypeman/lib/egressproxy" "github.com/kernel/hypeman/lib/logger" "github.com/kernel/hypeman/lib/network" "go.opentelemetry.io/otel/trace" @@ -105,6 +106,20 @@ func (m *manager) startInstance( }) } + var proxyGuestConfig *egressproxy.GuestConfig + if stored.NetworkEnabled { + proxyGuestConfig, err = m.maybeRegisterEgressProxy(ctx, stored, netConfig) + if err != nil { + log.ErrorContext(ctx, "failed to configure egress proxy", "instance_id", id, "error", err) + return nil, fmt.Errorf("configure egress proxy: %w", err) + } + if proxyGuestConfig != nil { + cu.Add(func() { + m.unregisterEgressProxyInstance(ctx, id) + }) + } + } + // 4b. Recreate vGPU mdev if this instance had a GPU profile // Note: GPU availability was already validated in step 2b if stored.GPUProfile != "" { @@ -128,7 +143,7 @@ func (m *manager) startInstance( // 5. Regenerate config disk with new network configuration instForConfig := &Instance{StoredMetadata: *stored} log.DebugContext(ctx, "regenerating config disk", "instance_id", id) - if err := m.createConfigDisk(ctx, instForConfig, imageInfo, netConfig); err != nil { + if err := m.createConfigDisk(ctx, instForConfig, imageInfo, netConfig, proxyGuestConfig); err != nil { log.ErrorContext(ctx, "failed to create config disk", "instance_id", id, "error", err) return nil, fmt.Errorf("create config disk: %w", err) } diff --git a/lib/instances/stop.go b/lib/instances/stop.go index cff5d008..812c6db6 100644 --- a/lib/instances/stop.go +++ b/lib/instances/stop.go @@ -199,6 +199,9 @@ func (m *manager) stopInstance( } // 6. Release network allocation (delete TAP device) + if inst.NetworkEnabled { + m.unregisterEgressProxyInstance(ctx, id) + } if inst.NetworkEnabled && networkAlloc != nil { log.DebugContext(ctx, "releasing network", "instance_id", id, "network", "default") if err := m.networkManager.ReleaseAllocation(ctx, networkAlloc); err != nil { diff --git a/lib/instances/types.go b/lib/instances/types.go index 8bf2043f..8dc48837 100644 --- a/lib/instances/types.go +++ b/lib/instances/types.go @@ -22,6 +22,13 @@ const ( StateUnknown State = "Unknown" // Failed to determine state (VMM query failed) ) +type EgressEnforcementMode string + +const ( + EgressEnforcementModeAll EgressEnforcementMode = "all" + EgressEnforcementModeHTTPHTTPSOnly EgressEnforcementMode = "http_https_only" +) + // VolumeAttachment represents a volume attached to an instance type VolumeAttachment struct { VolumeID string // Volume ID @@ -31,6 +38,36 @@ type VolumeAttachment struct { OverlaySize int64 // Size of overlay disk in bytes (max diff from base) } +// NetworkEgressPolicy configures host-mediated outbound networking behavior. +type NetworkEgressPolicy struct { + Enabled bool // Whether host-mediated egress policy is enabled + EnforcementMode EgressEnforcementMode // all (default) blocks direct non-proxy TCP egress, http_https_only blocks only 80/443 +} + +// CredentialSource references where real credential material is loaded from. +type CredentialSource struct { + Env string // Host env variable name +} + +// CredentialInjectAs describes how the credential is materialized for outbound requests. +// Header templating is currently supported; future types (e.g., request signing) can extend this. +type CredentialInjectAs struct { + Header string // Header name to set/mutate + Format string // Format template containing ${value} +} + +// CredentialInjectRule scopes a credential injection policy to destination hosts. +type CredentialInjectRule struct { + Hosts []string // Optional host patterns (api.example.com, *.example.com); empty means all + As CredentialInjectAs // Current v1 injection shape +} + +// CredentialPolicy configures one host-managed credential brokering policy. +type CredentialPolicy struct { + Source CredentialSource + Inject []CredentialInjectRule +} + // StoredMetadata represents instance metadata that is persisted to disk type StoredMetadata struct { // Identification @@ -51,8 +88,10 @@ type StoredMetadata struct { Env map[string]string Tags tags.Tags // User-defined key-value tags NetworkEnabled bool // Whether instance has networking enabled (uses default network) - IP string // Assigned IP address (empty if NetworkEnabled=false) - MAC string // Assigned MAC address (empty if NetworkEnabled=false) + NetworkEgress *NetworkEgressPolicy + Credentials map[string]CredentialPolicy + IP string // Assigned IP address (empty if NetworkEnabled=false) + MAC string // Assigned MAC address (empty if NetworkEnabled=false) // Attached volumes Volumes []VolumeAttachment // Volumes attached to this instance @@ -155,26 +194,28 @@ type GPUConfig struct { // CreateInstanceRequest is the domain request for creating an instance type CreateInstanceRequest struct { - Name string // Required - Image string // Required: OCI reference - Size int64 // Base memory in bytes (default: 1GB) - HotplugSize int64 // Hotplug memory in bytes (default: 0, set explicitly to enable) - OverlaySize int64 // Overlay disk size in bytes (default: 10GB) - Vcpus int // Default 2 - NetworkBandwidthDownload int64 // Download rate limit bytes/sec (0 = auto, proportional to CPU) - NetworkBandwidthUpload int64 // Upload rate limit bytes/sec (0 = auto, proportional to CPU) - DiskIOBps int64 // Disk I/O rate limit bytes/sec (0 = auto, proportional to CPU) - Env map[string]string // Optional environment variables - Tags tags.Tags // Optional user-defined key-value tags - NetworkEnabled bool // Whether to enable networking (uses default network) - Devices []string // Device IDs or names to attach (GPU passthrough) - Volumes []VolumeAttachment // Volumes to attach at creation time - Hypervisor hypervisor.Type // Optional: hypervisor type (defaults to config) - GPU *GPUConfig // Optional: vGPU configuration - Entrypoint []string // Override image entrypoint (nil = use image default) - Cmd []string // Override image cmd (nil = use image default) - SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) - SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) + Name string // Required + Image string // Required: OCI reference + Size int64 // Base memory in bytes (default: 1GB) + HotplugSize int64 // Hotplug memory in bytes (default: 0, set explicitly to enable) + OverlaySize int64 // Overlay disk size in bytes (default: 10GB) + Vcpus int // Default 2 + NetworkBandwidthDownload int64 // Download rate limit bytes/sec (0 = auto, proportional to CPU) + NetworkBandwidthUpload int64 // Upload rate limit bytes/sec (0 = auto, proportional to CPU) + DiskIOBps int64 // Disk I/O rate limit bytes/sec (0 = auto, proportional to CPU) + Env map[string]string // Optional environment variables + Tags tags.Tags // Optional user-defined key-value tags + NetworkEnabled bool // Whether to enable networking (uses default network) + NetworkEgress *NetworkEgressPolicy // Optional host-mediated egress policy + Credentials map[string]CredentialPolicy // Optional host-managed credential brokering policies + Devices []string // Device IDs or names to attach (GPU passthrough) + Volumes []VolumeAttachment // Volumes to attach at creation time + Hypervisor hypervisor.Type // Optional: hypervisor type (defaults to config) + GPU *GPUConfig // Optional: vGPU configuration + Entrypoint []string // Override image entrypoint (nil = use image default) + Cmd []string // Override image cmd (nil = use image default) + SkipKernelHeaders bool // Skip kernel headers installation (disables DKMS) + SkipGuestAgent bool // Skip guest-agent installation (disables exec/stat API) } // StartInstanceRequest is the domain request for starting a stopped instance diff --git a/lib/network/derive.go b/lib/network/derive.go index 87528a11..17f3d83d 100644 --- a/lib/network/derive.go +++ b/lib/network/derive.go @@ -81,6 +81,7 @@ func (m *manager) deriveAllocation(ctx context.Context, instanceID string) (*All TAPDevice: tap, Gateway: gateway, Netmask: netmask, + DNS: m.config.Network.DNSServer, State: state, }, nil } diff --git a/lib/network/types.go b/lib/network/types.go index 5b55aceb..7cd28905 100644 --- a/lib/network/types.go +++ b/lib/network/types.go @@ -34,6 +34,7 @@ type Allocation struct { TAPDevice string Gateway string // Gateway IP for this network Netmask string // Netmask in dotted decimal notation + DNS string // DNS server for guest configuration State string // "running", "standby" (derived from CH or snapshot) } diff --git a/lib/oapi/oapi.go b/lib/oapi/oapi.go index e4290fb0..161067ad 100644 --- a/lib/oapi/oapi.go +++ b/lib/oapi/oapi.go @@ -54,6 +54,12 @@ const ( CreateInstanceRequestHypervisorVz CreateInstanceRequestHypervisor = "vz" ) +// Defines values for CreateInstanceRequestNetworkEgressEnforcementMode. +const ( + All CreateInstanceRequestNetworkEgressEnforcementMode = "all" + HttpHttpsOnly CreateInstanceRequestNetworkEgressEnforcementMode = "http_https_only" +) + // Defines values for DeviceType. const ( Gpu DeviceType = "gpu" @@ -301,6 +307,12 @@ type CreateInstanceRequest struct { // Cmd Override image CMD (like docker run ). Omit to use image default. Cmd *[]string `json:"cmd,omitempty"` + // Credentials Host-managed credential brokering policies keyed by guest-visible env var name. + // Those guest env vars receive mock placeholder values, while the real values remain + // host-scoped in the request `env` map and are only materialized on the mediated + // egress path according to each credential's `source` and `inject` rules. + Credentials *map[string]CreateInstanceRequestCredential `json:"credentials,omitempty"` + // Devices Device IDs or names to attach for GPU/PCI passthrough Devices *[]string `json:"devices,omitempty"` @@ -336,6 +348,11 @@ type CreateInstanceRequest struct { // BandwidthUpload Upload bandwidth limit (VM→external, e.g., "1Gbps", "125MB/s"). Defaults to proportional share based on CPU allocation. BandwidthUpload *string `json:"bandwidth_upload,omitempty"` + // Egress Host-mediated outbound network policy. + // Omit this object, or set `enabled: false`, to preserve normal direct outbound networking + // when `network.enabled` is true. + Egress *CreateInstanceRequestNetworkEgress `json:"egress,omitempty"` + // Enabled Whether to attach instance to the default network Enabled *bool `json:"enabled,omitempty"` } `json:"network,omitempty"` @@ -370,6 +387,65 @@ type CreateInstanceRequest struct { // CreateInstanceRequestHypervisor Hypervisor to use for this instance. Defaults to server configuration. type CreateInstanceRequestHypervisor string +// CreateInstanceRequestCredential defines model for CreateInstanceRequestCredential. +type CreateInstanceRequestCredential struct { + Inject []CreateInstanceRequestCredentialInject `json:"inject"` + Source CreateInstanceRequestCredentialSource `json:"source"` +} + +// CreateInstanceRequestCredentialInject defines model for CreateInstanceRequestCredentialInject. +type CreateInstanceRequestCredentialInject struct { + // As Current v1 transform shape. Header templating is supported now; other transform + // types (for example request signing) can be added in future revisions. + As CreateInstanceRequestCredentialInjectAs `json:"as"` + + // Hosts Optional destination host patterns (`api.example.com`, `*.example.com`). + // Omit to allow injection on all destinations. + Hosts *[]string `json:"hosts,omitempty"` +} + +// CreateInstanceRequestCredentialInjectAs Current v1 transform shape. Header templating is supported now; other transform +// types (for example request signing) can be added in future revisions. +type CreateInstanceRequestCredentialInjectAs struct { + // Format Template that must include `${value}`. + Format string `json:"format"` + + // Header Header name to set/mutate for matching outbound requests. + Header string `json:"header"` +} + +// CreateInstanceRequestCredentialSource defines model for CreateInstanceRequestCredentialSource. +type CreateInstanceRequestCredentialSource struct { + // Env Name of the real credential in the request `env` map. + // The guest-visible env var key can receive a mock placeholder, while the mediated + // egress path resolves that placeholder back to this real value only on the host. + Env string `json:"env"` +} + +// CreateInstanceRequestNetworkEgress Host-mediated outbound network policy. +// Omit this object, or set `enabled: false`, to preserve normal direct outbound networking +// when `network.enabled` is true. +type CreateInstanceRequestNetworkEgress struct { + // Enabled Whether to enable the mediated egress path. + // When false or omitted, the instance keeps normal direct outbound networking and + // host-managed credential rewriting is disabled. + Enabled *bool `json:"enabled,omitempty"` + + // Enforcement Egress enforcement policy applied when mediation is enabled. + Enforcement *CreateInstanceRequestNetworkEgressEnforcement `json:"enforcement,omitempty"` +} + +// CreateInstanceRequestNetworkEgressEnforcement Egress enforcement policy applied when mediation is enabled. +type CreateInstanceRequestNetworkEgressEnforcement struct { + // Mode `all` (default) rejects direct non-mediated TCP egress from the VM, + // while `http_https_only` rejects direct egress only on TCP ports 80 and 443. + Mode *CreateInstanceRequestNetworkEgressEnforcementMode `json:"mode,omitempty"` +} + +// CreateInstanceRequestNetworkEgressEnforcementMode `all` (default) rejects direct non-mediated TCP egress from the VM, +// while `http_https_only` rejects direct egress only on TCP ports 80 and 443. +type CreateInstanceRequestNetworkEgressEnforcementMode string + // CreateSnapshotRequest defines model for CreateSnapshotRequest. type CreateSnapshotRequest struct { // Kind Snapshot capture kind @@ -13298,201 +13374,217 @@ func (sh *strictHandler) GetVolume(w http.ResponseWriter, r *http.Request, id st // Base64 encoded, gzipped, json marshaled Swagger object var swaggerSpec = []string{ - "H4sIAAAAAAAC/+x97XLbOJboq6B0d2vkHUmWP+I42ura68RJ2ttx4hvH3rvTylUgEpLQJgE2AMpRUvk7", - "DzCPOE9yCwcAvwRKlGM78SZTUx2ZBPFxcHBwvs/nVsDjhDPClGwNPrdkMCMxhp9HSuFgdsmjNCZvyZ8p", - "kUo/TgRPiFCUQKOYp0yNEqxm+q+QyEDQRFHOWoPWGVYzdD0jgqA59ILkjKdRiMYEwXckbHVa5COOk4i0", - "Bq3tmKntECvc6rTUItGPpBKUTVtfOi1BcMhZtDDDTHAaqdZggiNJOpVhT3XXCEukP+nCN1l/Y84jglnr", - "C/T4Z0oFCVuD34vLeJ815uM/SKD04EdzTCM8jsgxmdOALIMhSIUgTI1CQedELIPimXkfLdCYpyxEph1q", - "szSKEJ0gxhnZKgGDzWlINSR0Ez10a6BESjyQCWFOIxp6duDZCTKv0ckxas/Ix/Igu4/Hh636LhmOyXKn", - "v6YxZl0NXD0t1z+0Lfb9at/XM+VxnI6mgqfJcs8nb05PLxC8RCyNx0QUezzczfqjTJEpEbrDJKAjHIaC", - "SOlfv3tZnFu/3+8P8O6g3+/1fbOcExZyUQtS89oP0p1+SFZ02Qiktv8lkL6+PDk+OULPuEi4wPDt0kgV", - "xC6Cp7iuItqUd8WH/09TGoXLWD/Wj4kYUSYVZjU4eGJfanDxCVIzgux36PIUtSdcoJCM0+mUsulWE3zX", - "BCsiioQjrJaHg6ki24ZyhhSNiVQ4Tlqd1oSLWH/UCrEiXf2m0YCC4DXD6RaNBls+aqnZyVEs63p3TRBl", - "KKZRRCUJOAtlcQzK1MF+/WIKB4YIwT0U6rl+jGIiJZ4S1NZkU9NuhqTCKpWISjTBNCJhoz3yIYJZzB98", - "jGhImKITWj7fBp26eBzs7O55aUeMp2QU0qm9icrdH8NzjWK6H4WgtX8h+qAtmq0DhhRksjzeCyDdMIgg", - "EyKIxvGvHC4RfE6YPi16vH+BcVv/azu/orft/bwNwDzLm3/ptP5MSUpGCZfUzHCJctk3Go0A1Ai+8M8Z", - "Xq3a6wJGSYXF6vMBLW7hJJr5NYLNuWn6pdNSeLr2k3e6TZV2Amm0Q5aoQC2JfD4nzMMkBZwp+6IMnVd8", - "iiLKCLIt7F5omqgH+CXiQBJvCQ4Z+JcPv573DYiXeVDTm37XaRGWxhqYEZ8WoTkjWKgxKQGz5gqzHeWz", - "qwX/Wen4VO4qLMloNQU5o4yREOmW9mCbliiVwKkuLR9O0RVVozkR0nvmYFq/UYVsi9quIh5cTWhERjMs", - "Z2bGOAzhvOLorLQSD7dWYn9xoomg6xC4CIkUR+e/Hu0+OkB2AA8MJU9FYGawvJLC17p70xYpLMY4iry4", - "UY9um9/Ryxjix4Dz7GDU3T0ZBjrENJSuZXdTd99pJamcmV9Au/Ws4O7TZECjV6R/v/cs+hkQCSMl1MpM", - "fh7wTWI2G00jrmG6QCmjf6YlBruHTrSsoJC+KGhIwg7C8EKTbJwq3p0SRoSmU2gieAzcVoEJRm3Sm/Y6", - "aKj5wq7mgrt4t9vvd/vDVpmNjfa70yTVoMBKEaEn+P9+x91PR92/9btP3uc/R73u+7/+iw8BmnLmjiu0", - "62y7s99BbrJFdr060XWs/I2pf3H6PopjtvpE04lNd/rZyTLjYNYa8uCKiB7l2xEdCywW22xK2cdBhBWR", - "qrzy1W1vFRawjhVAYFMNpg3BUBF6AI3bEb8mItAUOCIa8WRHE2GqZAdhLTcD8UL6lvx3FGCmz4JhLrhA", - "hIXomqoZwtCuDK140cUJ7VIz1VanFeOPrwibqllrcLC3hOcaydv2R/f9v7lHW//hRXWRRsSD5G95qiib", - "InhtbvUZlSifA1UkXrsjDrppBGxeTNmJ+WwnmwkWAi++fofdQlbttBHmarc6iD2c/5s5EYKG7lZ9dnqM", - "2hG9IhbdkUgZGqb9/l4ADeAnsU8CHseYhebZVg+9ianSt1maX9JGG9QrbvfvLRLMOPAZUcT1gjJQ1zAx", - "OQwNHfJs57HTpEhkpXO4VzHoyWB7X55dbGvKlmAp1UzwdDorz8qS1c3mQ+XViPLROPHNicordLL9Bmmi", - "jyKqoZMR+Z1+//Tpthy29B+P3B9bPXRsQAbT1/vHhb175AwLAhxQiDhDz84uEI4iHlj5c6IZ1QmdpoKE", - "vYraA3r3HQ7ClFgknPoY4Apm5E2XEaTbzd9ugAfbY8q2pd6GbrAZ3AmbfwUb9pzNqeAs1qzwHAuqaVxJ", - "CfW59frN8fPR89eXrYE+RGEaWI3O2Zu371qD1l6/32/5OB2NQWvO+Muzi2ewU7r9jKskSqcjST95yPBR", - "tj4Uk5gLI37Yb1B7VqbShjtDsDnD1t7Lpwa5dl4CXrlNCamE1q4X03EZY3ZfPvVhy2yREDGn0qej+DV7", - "53a+QFMNYSrjtiRiTkSGtIDFvQLvF0Q8DbuFITutCRUkEFijXavT+pPEmgmaf9Kok8/d851fddDo8l9z", - "q+MooYysuNa/k+v1mouriOOwu3PLtysjSve9vMTX5kV5fy1OkAwlWp0lUZCF1zRUs1HIr5mesoeu2jco", - "a5wR1496JTj659//cXma86g7L8eJpbQ7u4++ktJWaKvu2it/ZgtJE/8yLhL/Ii5P//n3f7iVfNtFEKbx", - "MyzZdYz6p7yU/5oRNSOicOO6DdaPjAABnyOHL4XhS/qkohFoibjyORERXhSIpZ1Ta6cPFKsyK0EVnC/7", - "nSZ9V0h/vIZ06t7cxfyyKtTs9v3E0TMpz5ye6vNtaXmTmWQT2dk9tT93l6dUM6Mrmoymmhcc4Wmm41pl", - "nju/ogmCL7rwhdnGKDKHN0x1z2jMueoN2X/NCEOwd7DB5CMJgE5pIR4dnZ1IdE2jCCRiIATL18GQvSuQ", - "AtNcKv1fkbIOGqcKCRJzRZBlNGGQFOYCjccEpQw7+19vyIpQsQus4pUFyxURjESjGcEhEbIhZMxHyH5U", - "CxxY6gRLRYSh0GlShtfxb6fnqH28YDimAfrN9HrKwzQi6DxN9BneKkOvM2SJIHPCQGbRTAW14/IJ4qnq", - "8klXCULcFGPoLNMpWOPU/OXZhTVvyq3ekL0lGrCEhSSEObtbQiI1wwqFnP1Fn1gSlrstjl8Buv8sbyL7", - "dFrzIEnLO7Jb3Y3XYIDUa59ToVIcafJW4uC89khj6fZw6saQXpQYLNnKkBOrsiGpqYBoegaz9zIf65fz", - "DHNSL+edM5zIGVe1ct4VZeG6eblOftNta/mUTO8lbfO7ZlUSQbppMhUYDLW3yajcWPoGaNbvxhofDJ+x", - "LYNqkErF44LJDbUrikJaVimWgTXnUTfECgNT15DzNNNdNl/HC9OVOSJ199toOvZon/U1Rhma0ikeL1RZ", - "ktrp+w7i16pC3Fx821LnBmIONglHiq82hNMJcm2b2L3AaWSk+Gg+oZ6eM9Yo16JSiYKKz4klN7qLbhJQ", - "S6Q76HpGNTMlkQMC0OnL06IWozdkXbhYBug4GyDrNutSH0zQmEMXbS4Kk6Bg/EDjxRbC6PK0h95ls/2L", - "RAwrOifOL2aGJRoTwlAKTDgJYXy4NIsTSKW+qaiqfm5vJONCswXKGm7f9ZAWImNsb3d9FGKsaAAK9zGt", - "rAeMomaj9EiadLMib9GIF1jlPvCWTKlUouI8gNpvXzzb29t7UuUKdx91+zvdnUfvdvqDvv7/35r7Gdy+", - "l5Cvr6MybbEmjCL1eXZxcrxrWdDyOOrTPn5y+PEjVk8O6LV88ikei+kfe/he/Ij8pOw4t72gdiqJ6Doy", - "qbHKZ3EpGDZqLCo3NpTckd0jN+Ouamsg8U63vAsHKZ/p3Rp+N3dhqhLMtcb7wuKW1qOfai4wPyUFBZK1", - "kQXUaw08pvLqqSD4KuTXzHNvayZMjsx95tfsplqyHi8Q+agZdhIiwbmaSKNBKjOjO/uP9w/3DvYP+32P", - "X9AywvOAjgJ9AzWawJtnJyjCCyIQfIPaIPqHaBzxcRnRH+0dHD7uP9nZbToPIzg3g0PGK7uvUNtC5K/O", - "x9S9KU1qd/fxwd7eXv/gYHe/0awsG99oUo7lL7Ekj/ce7+8c7u43goJPEfHc+WlVfUlCnxI3SSJq1C5d", - "mZCATmiAwNML6Q9QO4YrjGQ6gPKZHONwJCx76b07FKaRXKk7NoPZlsatL04jRZOImHewIY3kGVj5MfTk", - "08tTxogYZW5sG/RkvdvW6krdWrImqOSlWALdKZXAheTMEyVRODAndC2dg93MJ/a+Dg/sGhpiwystOnUj", - "MidREQnM1aUnG3NBUIYnZtNKq6JsjiMajihLUi9K1ILyRSqAFzWdIjzmqTLKG9iw4iBgOwfZY6LJdTM3", - "jxdcXK21QuqbeCRSxnQ3a/UuR1HEr/UWX2nYwC2Okf3aOboUmL5MyWJUUfa9RG/NF0ZVlT9OUoUoU1xL", - "pywcLzowEgmhHUOCSMWBkuLgSnOYtpum3KWfb3mtGRanCDfj5bTznqwA3YlRwt6uhC2mRI2kwmotx6Ix", - "5R20P4fmjZ0a9IdrFSAN4M7I9X0AHbw+uhptu5Lh5G4gvsosl+ka8kZwCwsakh6C0wX2AedlWjlp54on", - "CQkz/U9vyM7NUckeSRSnEnSeVwYOakaoQFzQKS0PbI/NPdj3NkFFh003Rsfih8scKrwEpXj9occTRYSB", - "oHOgL3rB2U1odVoW9q1Oy1KiMmjcQw9EcqPz0hRfnl1saqVLBJ/QyLNc0DDbt1Yyc/arV/v98+7O/zG2", - "aI1vwKJRZrTSMQ9JrxKjAu2b3Twvzy7O6uaUBQih4uyW1pTZETyUI1M3O4hYDXmAGRoTZCUYh/76YskG", - "yXnvJz5ediJwTMbpZELEKPYo117o98g0MAYjytDp0zI/q/nmplLzWWlzQGye4MDGdzSDvkchV1lGpwDN", - "9/7tekvMNVznFaq3Stg21jG0h15nIVno5dmFRLntx6OpK29vrefQ2WwhaYAj06Nx8qasqGAD5GzMIZ/l", - "H1pVpIdPjr28oTsIqD2fJikcw/O33ZM3l9txSOad0pzAXjPjEdHz3ipQi7nzDc3dnEpEYl6n6TCIIZse", - "oAKsshPcGEiF8+qBjuIKRyMZceWZzTv9EsFL1L58YXz39Aw6KCltpX5egEIJvw+8J0ZTpLphz2HAqsq0", - "dMC9smM5ktGoVwrLKw3qOyq/EhyZAM4yPudhBm7j+VV5o/nV2tNrO/GNe+JcZBr4ED47PTYMQ8CZwpQR", - "gWKisA0XLTh8ATvU6rS6+o4KMYnBgDn599XOXzUq+AxdVilxny1Ff92JArcmakETuWhOQhRjRidEKhu1", - "UBpZzvDuo4OBia0KyWT/0UGv19vUW+957p7XaCu2jTNTwXGvJ2dftw934JTXZC2fW2dH735tDVrbqRTb", - "EQ9wtC3HlA0Kf2d/5i/gh/lzTJnXma9ROB6dLIXhlU2a+s4yzwd6JYwEGUJyEODXmphq5BmNmhH9RELk", - "9VpXeKrlE4NxX+uefuMAtjyKWhUC14p2+gZBbPTTak2oY4ygjR0zZYpGeXzfsg70RhGacmUQy1IAS0JY", - "FrYSReZXwNlcnwpfDEuJgLt3X2U/uDYC3SikHkz+LyvthVrwUuBfuv68tbZxkqxHWz+jmNG/prF71sPe", - "cxN9c6p/ExtbefQ30//88//Ks8d/7Pz56vLyv+cv//P4Nf3vy+jszVf5kq4OrvimERK3FhQBhqVSZERT", - "VDrFKvAwVDMuVQ2E7RukOIr1xz30DAS/wZB10SuqiMDRAA1bOKE9C/hewONhC7XJRxwo8xXiDOmurP/Y", - "lv74zKh/9MefnWz5pdpHaB3FhN2QzKdTpuOQx5iyrSEbMtsXcguRYNPXv0IU4ESlgujd0zxstEBjgYPc", - "QSwfvIM+4yT5sjVkIOGSj0roFSRYqCwazI0ASGFnZXwGbHMSojmOUiKthDxk2b0EIr/uxOhoepkSBHTz", - "Fc1qDVC84gsXZQfHw37Hs49It9MbGVGpCEOZtoNKQHTUdp6qh/0SqTjsH/bXMvgZDq1APzgJy7laHFI2", - "OEsGgWFoQ7hHM6WSBrp0TZvMGUG/vnt3psGg/z1HrqMcFtkWGyEPJ0lEiTQ6QhUBr2Odg7dafoc/vbsN", - "F2SUZPBZ1MAX8zkMjN69OkeKiJgyQ+vbgQbnhAZ6fWD+p1KmGhUpRkfPTp9v9RokmwHYZvNfsY/vshVW", - "LMdOaVanC8wwXsO3g06OO5pNsyc0Z+DAreYFFygyBCY/1wN0IUnZPRG2ylj1zU5Gi1zzZm6AYWvL9ZhU", - "KcUAvc34RpxNJYtSzZHBdZmfS+jWGl6Mz89S753yXMGbycpFlrSBhw9WyNo54dquJwWrj78H4nDmOavq", - "NDc720VlqB7Mjxr53t85t7K3qYy6aaha2TO9EImQRas1DzO7i3CtZXntI1WjWiM80q+tyd1JJZenaIYl", - "+4uClxXZZGfvcaOkLXrUpubrouGaT8yUslPl3Nwzs6tx+L+iUWS8GSSdMhyhJ6h9fvLyt5NXr7ZQF715", - "c1rdilVf+PanQdSaQ+2XZxcQCoblyFmA6p0ece44TD5SqeRyFEAjQ+rqKLlfS5Fs3rCKrVsMb3PW56Vl", - "3Efg2rd06/v+guZWhrl9bayaZXbvKFStlrj6wrzKdNY8vt2gszuZTil8zEcfijyB87m+ccRYp0U9/qZH", - "UpNAEqKTszzTRq6Uct1X1vRkt7dzcNjb6fd7O/0mKroYByvGPj161nzw/q5RRAzweBCEAzL5ChWhRWzD", - "vOHoGi8kGjr2etgy/HyBkS8cW8uCNzK/Lgfm3SwOr8pQrIu02ySyrlnI3Ip0WeflRFmNebRHf/uqnFqk", - "6c1sXRfsV6NNlNcEBTyNQs0HjfXJM2IVCa30J4nKc5DBYb1gV4xfs/LSjQ5Tn98/UyIW6PL0tKTxFmRi", - "Uyw1WDi4PNTsA0822obdNazy2tncMHrtPiLWqlSzcFvdenxaUeXmXCgNhjZQveXco9fsTZnZGo0nK9ZU", - "UZqEZD5KUx9TpF+5wImLi5PjEnJgfLBz2D980j0c7xx098P+Thfv7B10dx/h/mQveLxXk+SwudvLzT1Z", - "yqe5PlAJAA8KSBOHFg70ectcUcapQpmbmj7IzzR3iQpsrAnLAZ3ACaOK4oh+omyquwER3XK51zO9JogJ", - "RpRRpd+ZIF3K9JJBF6I7sc5HA/QS2sIrHEO4kJuElm3KagAcLowaVBMGN3QCf62e8vksVZrtgm/kLFVI", - "/wXL1mCw0sbqLgyNGaDXHL4RzkeU8arYYpqD79Vy86qI07ZeQc57FAazBHOAXmREMiOzlqy2JbE/De22", - "js3gtL1Vcp2zO97S2JLvXMErrNMyEG11Wg5Q4D227Edm5+UNkSiios8+QHAEJDT300kVjegnc4r1SqhU", - "NDBCH4bNrTvJNjMBCUfmBq+z9hnnD3vLZx85QnF5itoQjfhXZGVC/ddWZhksnsr93Sf7Tw4e7z45aBRz", - "kE9wPYF/Bq5Jy5NbS+2DJB25/LE1S392dgF3n75XZRobId+uveDimQgeaGaTMpQnpM0Hf9J7Ugy1CHk6", - "jgpKIxuXBf78TbIH15i3/qTRnE4m7M9PwdXuH4LGOx8P5O7YK5tlA/kZ2ZOionNJ6iPjrskm4/eGB4QS", - "sjZg5C2RsAJ0ThQC/OkiHMAlnXkUWZRzYSUW4l7E2t/b2zt8/Gi3EV7Z2RUOzgjEz+VZntoZFI4YtETt", - "t+fnaLuAcKZP52aZCCL14ky8pPecIZvVq1/ywNSiz54PS2r4pRxrbN/zuBbkl5YJsouyQAfHqIxBWjrl", - "Xmjv7fUf7z86fNTsGFuBayQ+rqYwtp11KBAkIHRe2vk2KMffHZ0h3buY4KAsYOzs7u0/Onh8uNGs1Eaz", - "UgIzGVOlNprY4eODR/t7uzvNIp98CnAb01c6sGXa5Tl0HqTw7IYHFMukt1N3W/gYz2VvzJUOoLlHadV9", - "cBN/4Tzmm0rolRZcVVFb82VFHrcQt7zVRM3hJ5F6nLqs9JoDberKu9pz9wyr2Qmb8GULxybypvWHcprv", - "RPNBEnLwhoRREjralQmelrUCD6tIEhSmxELOsEoCW4BjY+VJsJoB/wsfUjYt+5YvDdhECjRzWB3hD+Pa", - "hk0UVtLvl/NOpAAro2KWCOceOo305VSO/ILKcseCTNMIC1R1V18xZbmII8qumvQuF/GYRzRA+oOqNmHC", - "o4hfj/Qr+QusZavR6vQHo9zAXNEOmMlZ9wKzIZVx8yX8ole5VXFugpt/23y/DWVHmuj/vFanF1p2Mh7d", - "F4x+LCB6OQR2f7df5/dW02nJ4205GmBT2m5R1nfinaP+UZY8zGPdNPajilBc5oNL6/WtFgyUq7z8ljkB", - "1HYqRRdiXIZrIdS30UXczEZaVZ672WxLEpRH3z989PigYaz1V7HaKwozfAVjPY9XMNQ1O3XahGs7fHT4", - "5Mne/qMnuxvxR87OUrM/dbaW4v5UcgRWeLZHffjfRpMylhb/lGqsLeUJlfL93XhCX1Yc3TzGpkbqXlUU", - "Kd9JJ+aXGfBmLO4KbumoxHIVUtq2yWRCAkXnZGTg1s0nU/HNajSHACc4oGrhkQDxNbiroKxJJVakQe+V", - "yXpAavu24X6acsl0nLsDtN3g6N+MZFfBhcPGKRtkOq6TIt9URzUypPHvCisaigYKAoMRPpv8dQZMdI1l", - "yaigfweKhJ1CyuKq9cm0aF7YwuF6Vtsit6v74p38dSyK21/ZzoLUUWKSqxBfdYXWH0HNEYDzWBOdvedG", - "9njWBut9Oir0wV6AN/tqNC4mU1mZraaUeSW/dTcft1my5eXvzA22+XgFB4JNPqzmlQB8tHOwIM/77pRQ", - "ogabFBfr0wDeQXS4UWnfKD7casPvJUTcPr6TsPCl7TgveEE19/lzX/nLk5XsmAfd/l63f/BuZ2/w6GCw", - "s3MXAQqZDaNOlfv4087142gXT/ajw8XjP3dmj6e78Z7X6+MO0k9WcuRWslHaNSREVDOCVDPpSBJRRroy", - "M3+sN0SvCD0ySrkEL4DJWyGRbSIGuEI/K07teXmRxcOLVQ6car7S+/BPs7NfKctUp39yvHraN7InVCfi", - "R7DqVACfmk0GAuZ2bjXTKOhV4fR4AVmzKB/KlOzxJSR+v4KC/WYPbh2lsm7hdoZ5dgp3mJx9sYQ1+esl", - "QPlI7OoEGpVLyNhIi/lKMpfU282e8c5uaZ0DcyGpy+6jg3JWl6Pu30wWFzTqDbZ/+ev/7r7/t3/xZ/Iq", - "iY6SiG5IJsAxX5FFFwJqkEauXjn6FZyrW1Jhm/9KERwDtQuuiKGuMf5YnO+jfqbLXrzG8dISQNSIKcv+", - "Xrsgf5GpJUQzziZ1qVljV+e4kjCLmsqRNvQeFRqjNokTtXDhpU6nvrWZ88tR1qGXjb5lx/3+k9sIM7xY", - "GVf4AyYGLvomuQmt9Upa2v/aYB6/Uu646iNsNN822WHZp7WSwk2qFcVUVxXuNhW0Qa1tA+mmaTWjwAbF", - "uusMGfkpc446rlr3Ov38SqthYWWFmdTvjXFM+8rK5lS6kuY3BJlVMq+PTDOOOZoT7VazYZqELYKC1toC", - "yABWgyAzRCxbO1b71p7ij9kIwIFiucQtwzoKladePoUkTG9dVkQ6cV3ANKq1QZ5+Xcl3h1XLm7GqBrxz", - "ffQePEurVlC/urNVQc58jM7qMvOazJEgFVQtzjUZshEABAsijlKDhkCfYBHwOB8cojO/fAFl/MSjk3up", - "pTEaoKOzE8CSGDO4y9HlKYrohASLICI2uG7JoQ3k6TfPTromKjgr4QBVWRUAxGXVPjo7gSS9th5qq9/b", - "7UEVK54QhhPaGrT2ejuQsliDAZa4Dckc4Kc1t+lzCLfeSWhv56emif5K4JgoKKjxu8dspYgwySEkOCzg", - "aYGxSTAVlrNJIjCmGYmM6m/Bn9gR+IG5JToG4LipY65UC6taJMkbu63vNTrIhDNpNnS336/UBMZ58tbt", - "P6SxfeXjNuIyTIH2ZefaJZbPcToW5F86rf3+zkbzWZtv1TfsBcOpmnFBPxGY5qMNgXCjQU+YsXe4MmDE", - "NszPGaBQ8YT9/l7vl0zjGIuFA1cOq4TLOhaNSIQh4+PYlZrtISv6QTignPE0CqGcTGJy2msyipHCojf9", - "hLAIZnROhszeHiZ3LhYQLh0jfWsYvVX5aJihze4bskOkesrDRQW6WXfbujvgtsoA3rhYcqY3SWqqJvso", - "usk3LQPuTbRNGGYqT19sEk1fEXAvm9CP3g4b+UlqggfbQqCuQRZ+v7vlt9BCNJnfueE4e+dqdpcvOS0j", - "UBZEaZhzAuVayd6sS6bmr83HfUU8jNNLaGGBUgy8c1cu4yExQVTJQs04M7/TccpUan6PBb+WROiL2QZT", - "W1jbZLQWdaEwAo0hoNmkatFjbpspbn++IosvvSE7CmOXhsdWYcKR5DZRufFHpRJltcAAd2vK6/uFhWe2", - "cIlJElzMq2qmyVOVpKqHzEKIshHg0BzS7soZCYdMcfRZmCoLiy/bn/MRvwBHTXCo8aTQxCxp+zMNv9TN", - "Wo6wXv0ImnpkEgIAGLb07TJs6d9TgTVHncoZwgF4zeqHxS1tm4PNBXArW1UIB5ihhCdppHk/QCqTf73U", - "B2TTwFGEFBwl963mgWAna9Zjje2+1JDW0m5Mo5VjBEkiC4epv3/oP0+SBIL4xO7/PH/zGsFVBSXFoVke", - "4QkwokzfolnlKz16b8ie42CGDN8Ezv/DFg2Hrbx09BbMNZXWFNDtAuP1C9TUN8N0aPhLr6e7MjzdAP3+", - "2fQy0GcpiUeKXxE2bH3poMKLKVWzdJy9e+8HaJ3B8rxECFDb0P4tlwtJr7BwDZp7A7MQcUtrowXCKKdA", - "Rel+TBkWKxM5eUBvIagFTDyVRWB8HoKKZ9gaDJ2SZ9jqDFuEzeGZ1QQNW1/8ELCJx+o9zU0uK9ssR6KD", - "fn9rvSeRha+HhS411MfvyxL3tXtrjIdlupYZD1vf3obJ6B20ReaB3boHzucpDl2ei58s3hoWz8rTBeYN", - "vi/eAwZ9I2J0xxUOTAvgkePAVkonBi0gTgwkDuf3ZwQO6ji4HHmL4kdVyFwWK/brTlkAU4wc/u3fA/7B", - "uHlmfxj3yX2NiyNTg8rluX5Y6Aib5RCx45eIXxL1PWBc/75IqStA8g3x96Hgz0ti+b4caBVqtg01Povq", - "lmrssyA4lrYX01jLqucwp+45YQo9h6c9+6+TeCBU9EPEpx8GyIAw4lMUUUak9cnIbBj6UrSwhI9Mqsbs", - "O5vtNJhhNiUStc39+c+//wMmRdn0n3//h+amzS847tvG3x8iIT/MCBZqTLD6MEC/EZJ0cUTnxC0GYpXI", - "nIgF2uvbqsbwypM7VQ7ZkL0lKhVMZp7+el0AE9OhLeWh10NZSiSSAEIoTjexLuhG7ekR4d1ZNqC81xPd", - "WZK57AoKC9C3osMB8CmkJhzUyl8tv/bMrLmkP6tqcJd0+uvpiyIflcHerpnghgQGQOw7d/DCLhq1z8+f", - "b/UQyBgGKyDMADjmvBvLPPd+0qT1NMlQlDJBASgb2lRIm1+r/z22bZopgG2PP5IGuK4OQL0K2Kg8iCCh", - "g9dPWaGJOtgPN6ca9ulnj13ZwHoF7c3XWxzC+Wk2EoRvb58d7i3D3NbPzEH2LURg1LblzLIUlqUind8K", - "6e/l1ijUds2uDsRN4sx7E8uecTaJaKBQ180FsmTEJBPVygjyUMjBWztrhN26qgG9xfttuxSfUnvTZaEq", - "+ZV397dHZdBNrpE86DjHtZ83yTrUOaYy4PrbArZ0A5zYBJ6GfcnOaRGL1imkjuF5duWsZJeOs6rP9kDe", - "n2rKDp2y6t1wD0TxuEIQvyEhrCQlLITpPyRsvsh20ZVIXqG5+r5Qs39/XNB9a7F8aP6Q1FhhBWyaCs6y", - "UlV16GWLWd3hRtsRPAs/J8KdajNRk+AuX5b5FAUzElyZBdlK3qs4ghNX7LuJ6Gv6+5EkX1NFbAOOxYL8", - "J4vSQNjNYbVKwD2xmRrvTr6FETYSb2/PzmsRzANkcDYZO421SYKI5YIFWz+UqfdebrNqtfAHdJLO0ihy", - "Fo85ESqvpVa8A7Y/g1vSet7enbaV18HF21ddwgIOfmiZD5WfiXIljm6XwzcbZpbyE02ayIQAKocY9Qz0", - "V+y/cRdEWb78f919YTPm/+vuC5Mz/1/3jkzW/K07Q5b+fZHm++a4HzDyaYabloEGpMkUIlrHoWatGjKp", - "rv0PxafaonabcKoZXH8yq02Y1SK4VvKrWX3BO+RYbSm2b2OSyZDNB2145fwTfzBO9X61fBYjC1X7S2YP", - "m3KSi7z8ma35/fAcKGmGccVro6G6Oj+QK68Ph7onxx1b2c7Uo8sCRO5Jee3mce/MrR33/jXXR/GYTlOe", - "ymLsCRQyJNIGK0WkTIAfGtudX8+1jPd3jKX9+7w67p2v/on3d8TxVzfUEG9jgVrH87tWTXl+2x5KBppq", - "FCZ27a2rcmHTqGzVOBW6OjBN0bhUsmjZ2dE3L58sgi60oJKLCwgkiMGQ/YeWP35XBMfvf3FBMmm/v3sA", - "zwmbv//FxcmwU4cqhClBiURYEHT0+hjMflOIXodkaHlIXnUeJsWZqQ1ty5b+jxOQcstncwnJYeFPCamR", - "hFQA12oJKauicpcikhnkm8lIDt98ALepNX5KSfchJcl0MqEBJUzlGYCXnMRsAvEHGFvGrH2o4NxRumgb", - "S0l5aaPVDGie9u7eHXuywe9fOHIZ9h6mjzw3UTGhE0fyy7BeHvne8KF/v8T5/uWQh4xihuGvgm6ZEG1P", - "bAJiP4PwgourppjnycN56wh4+9xJcYXfIW+ip0cKVQ6/IYsCl7fxrddIU+Zc7uFALiVX/ZYunQ4SVrg1", - "QZGUTfM6l1TNeGqyqozsQ5OVTZ8KW00GWJ7A9vqtyYse/R4Y0NdcIRonEYkJZG3rGmyC4qJpknCR1R+j", - "spCKeDPyp49N0cHWJLexVYA7yCZsBmWd27A26O2Xt8tLNSM+XR9Umw3uIkg9UbVDdiFNkpcPhhX+gDIi", - "ixRHkkQkUOh6RoMZRNjqZ9C/CcDFSfIhS6mx5YqlFjOLwOBtSQTFEVR55JGpV/phHscfBssZ4C5PT+Ej", - "E1xrcr19GCCX9S27IKRuVYyY1auIsFTotY0DbmtMEjyKzI5+0LdQYX1bNpY2T3kyZL64WkaubYd0gj4U", - "Qmw/1MTYOoL6ik/lt+KXOvWJqsxaFEcCAGdwk7CwVafYoZE/unan3/flT2kY6WumcceBvkuTecWnWZKs", - "EirjJGmKvnaagMXzOF6Bw6hdSGYuVchT9VepQiIEfGyxuw65URsH5g+FrzSiMluKzKWDB/Tzqi9N1hov", - "qDRRLeSTNn/N47jVadn5eKrnfn3EdLXDZTWb3plCWPRPTnuTgOcysS9EPFduDlu3op7ltuU4fnh5z5W7", - "/sZo+A30Y/ksKHOsCuxtXkf8YUVOmkotVV7MJM/3nZGs1Ev9KSkrlc/zNP3/A0VUs9ZqfZ57FlIzEPsk", - "s1J5i28unWbVNn5KqJmEygUKUzNcpd7NDyt2ZgQFpawkeVr29KayZ5ZkLgMz1CFkKw0COc3b/ux+ntyA", - "XfhOKGGntupLXTqjfNHfA8mtqYnWiOZ+Iz7JXqsFBuEbkmBXne2+KXAGFS3uZVTuuyDD5sBl1LhIc6Dy", - "PnWFF38S45Ia0GhKb0qMHfO5pAsskGfKukmE6+iy5VNrCbCtAvXDy2u5rPKDS2wBF8K4joEz2kMKXSzY", - "DAuiZzvBqSSd7MB0nN368vR0q+7QCLXyyIjvw6B9M86hUpYzDv11kQUNXZL6Z6fHNqU9lUikrIfexBQy", - "x18RkkBKSspTicAHsFesN1ZXBS0rKEaYEouEU6bWziJvejeT+XKjJN33TKds8PYPr1ayhXYfGpEC2qFv", - "b7uA1UKVMmX2vGY6Z7aizGTW18wHHvNU975UDw1NaETkQioSG5vdJI3gEEF6D5v91X5nfNc6iCoJ1cM7", - "4OuTEBFTKSlncsjGZKK5koQIPTYUnKQRKZgffJatc4UzqnlmSN/3YdqCEmlgzcGqDmrl6mg4SVx1NJ/5", - "JCvoduMpvQBbFZKLeMwjGqCIsiuJ2hG9Mjw4mksU6R9bK41dI/jutnPb3vxkaUifsAn3pv8zOJsh849A", - "4U4qZM0Z8x8cWXtJiofF0R/YaD9Zk2vpmiA4giKgmZstShWN6CdD6nQnVCoamJpJOIMdlHsx4/WG7JQo", - "odtgQVDAo4gEyukathPBg+1h2u/vBQmFeIg9ApMDglf/OoYRn51dQDtTkqYzZPoP6Pjd0RmiGqYTbEXm", - "wkRtYXt0sv1mjfn/HMD0P1geMwtcdSz8G/7Tsru5D2XtGZI1R5QnqwQgnvzwCgPLwf3UFjxMbQE4sWer", - "aU8FDoAplrNUhfya+TUDpkKq3P5sfpysC4VQOJhdulLR3we3a6vFrhvGLfBBHEq7ppCY9KTfRF9vC/o+", - "0HROGnBuCcDEFIM6/LeAKRT+o2H37RvrinD8Di11FqIu9e93c7bu++azc3ARfkV4PJRjbjDNrQRKVha1", - "T1k441rZLEiFIExBKpictQxwggOqFh2EI1dN1ZZHynRIeSH4sSD4St+0vSF7mwVS2vJMWrrqONEKhVRe", - "mR6s9NRDb+ZEyHScTQ4BYTJyHgDfFlQNcBSYSqRkMiGBonNiSoTKGukrm8pdpuXNB/FstHtpQffQRA4/", - "TsDu5WhhpY6Sp1xt+obzrFWz9A1ZrwVvmIKnyEqf55FraKrgb6Ky8wx+RWvd4u2rzbzXftMfNRy77CXl", - "n4R99ZWr/FGy4p0XnFOaJn3IMfyh5V8ozLx0VEsOXusDwRt7dN2lh9W6QPBs8PsOBD/3Ovk8sHRUuOS2", - "VRcB/v0hQv9+vYvvOwL8YeOWZiXkEujqKVGDSPDvAgPvJgT8G3vX3yAE/Lvy94QQ3m/nd/9deXpaj8XM", - "0/NnkPddOniaSG8IaK1z8DRUz2qeVwpKl7ZNMzHJ9vgjcfBWWbkB/+7A/jNlWwORoQAsdwtXyA3QfmkR", - "nsSJWjhtFJ+A302eU1DST+C95wucy5TOdxevdgN97O2hh8PTWm3sz1Rv96bwzfNhnxw//PxuxTNXuli2", - "9a3TxSKY0XkpXmvVCbYgSgTpJjwBPWtoAGbh4e4yhUVv+gnZ7ntD9m5G3F+IumwZJEQhFSRQ0QJRpjhQ", - "BDPGXyQSXEsC8J6LhU99Wzy5LwSPj+xq1tyH9kxZZVju5hcvuiFWuDt31GaFCu0rTFan+CON0xgIHqIM", - "vXyK2uSjEiZ5A5poyQfRSQZS8jEgJJSAk1vFCe/0azSb9BMZTcdNZrkiDccbm+YEBalUPHZ7f3KM2jhV", - "vDslTO+FZvUnwMkmgs9paHLk5kCd88hAdacGoJvqXTVTYf3Bc+HCTO6b8DBNLqTpJ5qUyYJxe2wNWmPK", - "MExubcKL8pkyHrh6PEzBDy4/Ow5zWj+vsGqVbY2JWshxQFSco0hz9Fs/r7mHfM0VPRncnVa67ZplMW3m", - "3NDQ5+AuMphmji/3q7a+/H7s8YWqxA9QdT7PBNI6tfn3hYL9+7sf7ltdfvmA/bdeEid8F1Tl0IHu0Ycw", - "r3iAIxSSOYl4Emu20rRtdVqpiFqD1kypZLC9Hel2My7V4LB/2G99ef/l/wcAAP//du3TWocdAQA=", + "H4sIAAAAAAAC/+x97XLbOpLoq6B0Z2vkGUmWP+I42jq117GTHO+JE9849t6do1wZIiEJYxLgAUA5Sip/", + "5wHmEedJbqEB8EugRDu2E28yNXUikyA+Gt2N7kZ/fG4FPE44I0zJ1uBzSwYzEmP4eaAUDmYXPEpj8o78", + "kRKp9ONE8IQIRQk0innK1CjBaqb/CokMBE0U5aw1aJ1iNUPXMyIImkMvSM54GoVoTBB8R8JWp0U+4jiJ", + "SGvQ2oyZ2gyxwq1OSy0S/UgqQdm09aXTEgSHnEULM8wEp5FqDSY4kqRTGfZEd42wRPqTLnyT9TfmPCKY", + "tb5Aj3+kVJCwNfi9uIwPWWM+/jsJlB78YI5phMcROSJzGpBlMASpEISpUSjonIhlUBya99ECjXnKQmTa", + "oTZLowjRCWKckY0SMNichlRDQjfRQ7cGSqTEA5kQ5jSioWcHDo+ReY2Oj1B7Rj6WB9l+Ot5v1XfJcEyW", + "O/01jTHrauDqabn+oW2x79e7vp4pj+N0NBU8TZZ7Pn57cnKO4CViaTwmotjj/nbWH2WKTInQHSYBHeEw", + "FERK//rdy+Lc+v1+f4C3B/1+r++b5ZywkItakJrXfpBu9UOyostGILX9L4H0zcXx0fEBOuQi4QLDt0sj", + "VRC7CJ7iuopoU94VH/4/T2kULmP9WD8mYkSZVJjV4OCxfanBxSdIzQiy36GLE9SecIFCMk6nU8qmG03w", + "XTOsiCgSjrBaHg6mimwbyhlSNCZS4ThpdVoTLmL9USvEinT1m0YDCoLXDKdbNBpsmdRSs5OjWNb17pog", + "ylBMo4hKEnAWyuIYlKm93frFFAiGCME9HOqFfoxiIiWeEtTWbFPzboakwiqViEo0wTQiYaM98iGCWczf", + "+RjRkDBFJ7RM3wadungcbG3veHlHjKdkFNKpPYnK3R/Bc41iuh+FoLV/IZrQFs3WAUMKMlke7yWwbhhE", + "kAkRROP4Vw6XCD4nTFOLHu9PMG7rf23mR/SmPZ83AZinefMvndYfKUnJKOGSmhkucS77RqMRgBrBF/45", + "w6tVe13AKKmwWE0f0OIOKNHMrxFszkzTL52WwtO1n7zXbaq8E1ijHbLEBWpZ5Is5YR4hKeBM2Rdl6Lzm", + "UxRRRpBtYfdC80Q9wC8RB5Z4R3DIwL9M/Hret2Be5kFNb/pdp0VYGmtgRnxahOaMYKHGpATMmiPMdpTP", + "rhb8pyXyqZxVWJLRag5yShkjIdItLWGbliiVIKkuLR+o6Iqq0ZwI6aU5mNZvVCHborariAdXExqR0QzL", + "mZkxDkOgVxydllbikdZK4i9ONBN0HYIUIZHi6OzXg+0ne8gO4IGh5KkIzAyWV1L4Wndv2iKFxRhHkRc3", + "6tHt5mf0Mob4MeAsI4y6syfDQIeYhtO17G7q7jutJJUz8wt4t54VnH2aDWj0ivTvD55FHwKTMFpCrc7k", + "lwHfJmaz0TTiGqYLlDL6R1oSsHvoWOsKCumDgoYk7CAMLzTLxqni3SlhRGg+hSaCxyBtFYRg1Ca9aa+D", + "hlou7GopuIu3u/1+tz9slcXYaLc7TVINCqwUEXqC/+933P100P1bv/vsQ/5z1Ot++OuffAjQVDJ3UqFd", + "Z9vRfge5yRbF9epE14nyt+b+xen7OI7Z6mPNJ26604fHy4KDWWvIgysiepRvRnQssFhssillHwcRVkSq", + "8spXt71TWMA6VgCBTTWYbgiGitIDaNyO+DURgebAEdGIJzuaCVMlOwhrvRmYF9Kn5L+jADNNC0a44AIR", + "FqJrqmYIQ7sytOJFFye0S81UW51WjD++JmyqZq3B3s4Snmskb9sf3Q9/cY82/sOL6iKNiAfJ3/FUUTZF", + "8Nqc6jMqUT4Hqki8dkccdNMIxLyYsmPz2VY2EywEXnz9DruFrNppo8zVbnUQeyT/t3MiBA3dqXp4coTa", + "Eb0iFt2RSBkapv3+TgAN4CexTwIex5iF5tlGD72NqdKnWZof0sYa1Ctu9+8tEsw4yBlRxPWCMlDXCDE5", + "DANBQD/B0cpjeBWIvcA6zPpdPrR/5VJ1Y8zwlIA2aRuiseBXRE8UJTyiASUSXZGFFlIWaKo77c6ppJp8", + "CJujOTZGg96QvZ9xSUwT90orIgGhc4JiHlyhJMIBmXFQxOc4SonsoOuZlhg0MxYER/YxEiTGlA3ZTE9S", + "BjwhodYhTDNYGrokbH6JYpwAlWJBgERRjBURFEf0EwkRN5/EJKT6gBoyAniNEqxJNgi40Kev3luCg1kB", + "Cn+W6NLIG5fQ/SVlGisvDV31hqy4859bb8/fP397/uZo9Pb0xZuD49FvL/5bPzYftQa/f24Z+2YmaDwn", + "WBCB/vQZ1vvFSKchEa1B6yBVMy7oJ2Ns+dJpaRhIjV84oT2eEIZpL+Bxq9P6S/HPD18+OHlKD0XYXJOB", + "Z2JfvLKMOQo9HOXIGfMksgYiEO0wmGqBw7w6Pd/Uh2uCpVQzwdPprEwY9mS/EUmEVF6NKB+NE9+cqLxC", + "x5tvkZY7UEQ1gWZyxla/f/J8Uw5b+o8n7o+NHjoyVAvT1yyECyv+yJlGHy2EA8ocnp4jHEU8sCaQidaV", + "JnSaChL2KpY36N3HnwlTYpFw6tPBKswpb7rMo7rd/O0NWNHmmLJNqbehG9wM7oA3t9YEXrA5FZzFWhub", + "Y0H1MSvLtPLm7dGL0Ys3F62B5uNhGlij4unbd+9bg9ZOv99v+RBUY9AaHvjq9PwQdsqQjUqidDqS9JNH", + "EjjI1odiEnNhNGD7DWrPyoKCoVsEmzNs7bx6bpBr6xXglduUkEpo7XoxHZcxZvvVcx+2zBYJEXMqfWay", + "X7N3bucLx7ph92XclkTMiciQFrC4V1A/goinYbcwZKc1oYIEAmu0a3Vaf5BYy+HzTxp18rl7vvNbrxrJ", + "n2sESxwllJEVkuV3IuFdc3EVcRx2t+5YwGNE6b6Xl/jGvCjvr8UJkqFEq7NkjWDhNQ3VbBTya6an7OGr", + "9g3KGmfM9aNeCY7+9Y9/XpzkatLWq3FiOe3W9pOv5LQV3qq79ppAsoWkiX8Z54l/ERcn//rHP91Kvu0i", + "jCByK6HO7v8L0wOwbI3rYema0lgzy2D5rxlRMyIKp7dDFv3I6MPwOXK4V1hKyTxavNNcYtR8TkSEFwXG", + "a+fU2uoD96vMSlAFtGq/02z0CumP17Bh3Zs75F9VdfTtvp/ReiblmdNzzSvsudBkJtlEtrZP7M/t5SnV", + "zOiKJiOQmkd4mplsV902n13RxIri8IXZxigyjCBMQXgfc656Q/ZfM8IQ7B1sMPlIAuB5UmGFDk6PJbqm", + "UQQGHmAqy0eLFuxztmKaS6X/K1LWQeNUaWmdK4Ks3gSDpDAXaDwmKGXYXWdXZGe7wCpeWbBcEcFINDKy", + "sWwIGfMRsh/VAgeWOsFSEWG4fZqU4XX028kZah8tGI5pgH4zvZ7wMI0IOksTzQ82ytDrDFkiyFyrEGwK", + "xkZqx+UTxFPV5ZOuEoS4KcbQWWYis3et81en5/a2Xm70huwd0YAlLCQhzNmdOBKpGVYo5OzPmmJJWO62", + "OH4F6H5avokq32nNgyQt78h2dTfewH26XvucCpXiSLPKkjTovV43jhseqd/4hRS1D8u2MuTEqnwv2tTe", + "YXoGL45lmdhvtjCCTmOzRUETXzJgODXxc7PJrun/mLmJrDTb5JriV4x1Zjqpgsj23XEruwWUjjOYlGGF", + "7wY8B7KgWdeaxUMiFWUGnXRbZAU6idqXWhm3eKzV78sOuvxL6YEmXacZaPHgGhloADtg+lGx/6pNYa22", + "31ynq2wOlrffjwNZ62eE5ltICcykPhq1iJSQHvoVeDBSJE40I2JTRCWShneSEDF+/e+IG5nEfTpkemrS", + "eGlYcGQ2H0mnjLLphpbS9bmCw9AYhiapSoVuN6cyh2YZdZzxpbqA92Z2xLDTOJX6QA2iNCTo0hloLsti", + "3bL5Zlmjs/acJQXFgAQUE9DV1GacKj28XnCMVTDTcOKpMm5bdumyPIGykWjddaadS3bRdYv9P8vYRRmo", + "1lxQYfx6cfaKBax6BfNinRXPyhl+C+MVWcCWO2siXrInFg2JfnOfIJJHc2JPzaIpcoyDK3OUGM8Ja4U0", + "9kRrQtTkXyFRr3Ft3VZoeDUGf1nSX0YlsODaxeYYY4V3Y75dZFxIL86M19F6rSQAfNAcBgikqcuOUXUI", + "GBAQ08gSoZAKEqil7imbDhl4cFzaJz3b26Umci1i+IjQp6t4RbmCsmK+KW0tKuysk9qgG700HlOlSNgp", + "ywZXhCRy/aK0dGztzh7juCDXgjpGZu09YUPpirAJFwGJrYz/dXrfi0JnXi3sZl0sO1QY+BbmbPEJ4SSJ", + "KAmN947ZD7CSSrtPYCKteuyGFaXLXOCXh7zEUXSJ2rbRBhJEr0W6vWKc5cj+/vDUoUB26Xxx0tEYqbnA", + "5UypZKT/I0eaii+rndlvHYXr7vSZJNF+H9Sj3d0du6vWZmYmXOm2bB7zOiXUb80Zw4mccVV7r3VFWbgO", + "UVwnv+m2tUaxTKCRtvl928USQbppMhUYHFPv0ip269tGgGY9513jc+5zLsygGqRS8bjgYojaFccIWnah", + "KANrzqNuiBUGC2JDM6eZ7rK7brwwXRkdqs4AMpqOPd429JPmlmhKp3i8UGWz/Vbfp6l97dWvm4tvW+rc", + "3o3mR8KR4qsdf+kEubZN/PzgHBgpPppPqKfn7DjKvUaoREHFx97qo7qLbhJQq8WDbBLMjF+mAQIIexcn", + "xSuz3pB14dgcoKNsgKzbrEsMMiEOzYVFm4vCJCg4e6HxYgNhdHHSQ++z2f5ZIq1ozImLA5hhicaEMJSC", + "xRdOsa45Q4sTSCUcdqr6uTVZmJCBDbgZ5PZdD/26SEiMrflHk0KMFQ3AwWhMK+uBY8RslL2KxaxofGpk", + "LFrlLv2OTKlUouIsjdrvXh7u7Ow8q5oNt590+1vdrSfvt/qDvv7/35r7Vd99VISvr4Myb7EuW0Xuc3h+", + "fLRtbZTlcdSnXfxs/+NHrJ7t0Wv57FM8FtO/7+AHiZvws7Kj3NcMtVNJRNexSY1VPg+zgiNXjQfZrR3D", + "7snPK3dbXdXWQOK9bnkfASE+V2Pr6HrzkI0qw1zrrFxY3LIGvkhAX8yppCB5WZ/AgHq9H4+ovHouCL4K", + "+TXznNsxnhI5MueZ340glca3hXy0VgnBuZpIc11ZtlZu7T7d3d/Z293v9z1xEMsIzwM6CvQJ1GgCbw+P", + "UYQXRCD4BrXhnilE44iPy4j+ZGdv/2n/2dZ203mYm5VmcMgUJvcValuI/NXF1Lk3pUltbz/d29nZ6e/t", + "be82mpW18zaalLMJl0SSpztPd7f2t3cbQcEniL9wcSlV3/nQ5zGg9R5zx9eVCQnohAYIIluQ/gC1YzjC", + "SHZJVKbJMQ5H1ujhPzsUppFc6ahgBrMtjYEsTiNFk4iYd7AhjWzIsPIj6MnnBEIZI2KUhe3coCcbzbP2", + "Yt6tJWuCSlFZJdCdUAlSSC48URKFA0Oha/kc7GY+sQ91eGDX0BAbXmvVqRuROYmKSGCOLj3ZmAuCMjwx", + "m1ZaFWVzHNFwRFmSelGiFpQvUwGyqOkU4TFPlbndgw0rDgK+wqB7TDS7bqafvuTiaq3XpT6JRyJlTHez", + "1ppzAAbwiTWxwCmOkf3aOfYXhL7sFs7cVdr3Er0zXxjLTv44SRWiTHGtnbJwvOjASNYCxJAgUnHgpNbQ", + "Z7tpKl365RYwcjqvCzNezjsfyOWkOzG39HerYYspUSOpsForsWhMeQ/tz6B5Yydu/eFaA0gDuDNy/RBA", + "By/3rkbbrmQ4uR+Ir/IBy2wNeSM4hQUNSQ8BdYEziouqq1DameJJQsLM/tMbsjNDKtkjaW4+9IcGDmpG", + "qEBc0CktD1w2jN2nM9lNUNFh063RsfjhsoQKL8Frop7o8UQRYSDoAoaLUT92E1qdloV9q9OynKgMGvfQ", + "A5Hcw3Fpiq9Oz2/qEpYIPqGRZ7nggmDfWs3MOUu93u2fdbf+j3F81PgGIhplxm0h5iHpVWLyoX2zk+fV", + "6flp3ZyyhAioOLulNWWOJh7OkfkjOIjYyyB7m2g1GIf++mDJBsll72c+WXYicEzG6WRCxCj2GNde6vfI", + "NDAeRZShk+dleVbLzU215tPS5oDaPMGBjWdvBn2PQa6yjE4Bmh/82/WOmGO4LgpOb5WwbWwgXA+9yVJQ", + "oFen5xLlzkEeS115e2vd1E9nC0kDHJkeTVArZUUDGyBnYwn5NP/QmiI9cnLslQ0dIaD2fJqkQIZn77rH", + "by8245DMO6U5gUPPjEdEz3ujwC3mLhYu96kvMYl5naXDIIZsSkAFWGUU3BhIBXr1QEdxhaORjLjPyeK9", + "fongJWpfvDSxSnoGHZSUtlI/L0ChhN97XorRHKlu2DMYsGoyLRG4V3csZ24x5pXC8kqD+kjlV4Ijk7Cm", + "jM95WLXbeH5V3mh+tZZ6bSe+cY+dP3aDmKnDkyMjMAScKUwZESgmCtv0OAXXFBCHWp1WV59RISYxeLhN", + "/n21V0qNCb4YBFVrxD1cynZxLwbcmijtd8Z1IEQxZnRCpLJR2qWR5QxvP9kbmFwSIZnsPtnr9Xo3DQ15", + "kceCNNqKTeM5X4gS6cnZ1+3DPUSANFnL59bpwftfW4PWZirFZsQDHG3KMWWDwt/Zn/kL+GH+HFPmjRxp", + "lH6ETpbSjpSvNPWZZZ4P9EqYdeXSuMRBgV97xVSjz4BHAoSreaN0FZ5q/cRg3NeG4946YUeeNUoVEnUU", + "HTkbJO2gn1ZbQp1gBG3smClTNMrzmSzbQG+VkUauDNpfCthPCMvC9KPI/Ao4m2uq8MXslxi4e/dV9wfW", + "O2UUUg8m/5fV9oxzAwQzrae31iZOkvVo6xcUM/7XNFeJjSj2nETfnOvf5o6tPPrb6X/+8X/l6dO/b/3x", + "+uLiv+ev/vPoDf3vi+j07VcFLq0OJv+mEeF3FgQOF0ulSPCmqHSCVeARqGZcqhoI2zdIceNn2UOHoPgN", + "hqyLXlNFBI4GaNiquPYOW6hNPuJAma8QZ0h3ZQMMNvTHp8b8oz/+7HTLL9U+QhtJIOyGZAFEMh2HPMaU", + "bQzZkNm+kFuIhDt9/StEAU5UKojePS3DRgs0FjjIIwjywTvoM06SLxtDBhou+aiEXkGChcqyX7gRACns", + "rIzPgG1OQhePbTTkIcvOpSwc29hoepkRBGzzVU9JP1C86gsX5QiY/b4vcB28tfRGRlQqAg7VGWZrNMrc", + "yNB+v8Qq9vv7/bUCfoZDK9APKGE5N6VDyga0ZBAYhjaMGzzLGtjSNW8yNIJ+ff/+VINB/3uGXEc5LLIt", + "Nkqe8d2TxkaoIlnw2tto+SNC9O42XJAxksFnUYNgnRfGrfP96zOkiIido3070OCc0ECvD67/qZSpRkWK", + "0cHhyYuNXoPkmgDbbP4r9vF9tsJqUIY1mtXZAjOM1/DtoOMjcKu1FJoLcOBW85ILFBkGk9P1AJ1LUvZR", + "ha0yt/pmJ6NFbnkzJ8CwteF6TKqcYoDeZXIjzqaSOUjmyOC6zOkSurUXL8bnZ6n3ij8teDNZvciyNvDw", + "wSpz7tYnbj0rWE3+HogDzVt/7IJN82a0XTSG6sH8qJHv/b1LKzs31VFvmhehHLpYCHvNUiM0z2lwH7kB", + "lvW1j1SNai/hkX5tr9ydVnJxgmZYsj8reFnRTbZ2njZKUqlHbXp9Xby45hMzpYyqXBxkdu1qIkKvaBQZ", + "bwZJpwxH6Blqnx2/+u349esN1EVv355Ut2LVF779aZAiwaH2q9NziFLBcuRugOqdHnHuOEw+Uqnkcpho", + "o4vU1SkZfi2lTfDG3W7cYS4Fd/u8tIyHyJLwLd36vr8MDStzKnxtYgQr7N5TXoRa5urLKVDms+bx3WY4", + "uJfplGJ2fPyhKBM4n+tbpxTotKjH3/RAahZIQnR8mmcWzI1SrvvKmp5t97b29ntb/X5vq9/ERBfjYMXY", + "JweHzQfvbxtDxACPB0E4IJOvMBFaxDbCG46u8UKioROvhy0jzxcE+QLZWhG80fXrcuaG2yVqqAoU61Ix", + "3CT1QrOcCivSA5+VEwM3ltGe/O2rcgiTpiezdV2wX41uYrwmKOBpFGo5aKwpz6hVJLTanyQqz7kMxHrO", + "rhi/ZuWlGxumpt8/UiIW6OLkpGTxFmRiU8o2WDi4PNTsA09utA3ba0TltbO5ZXqDh0hpUOWahdPqzhMY", + "FE1uzoXSYGgD01suPXqvvSkzW6PxZMWaKkaTkMxHaeoTivQrFzhxfn58VEIOjPe29vv7z7r746297m7Y", + "3+rirZ297vYT3J/sBE93apK6N3d7ub0nS5ma6wOVAPBggDRxaOFA01vmijJOFcrc1DQhH2rpEhXEWBOW", + "AzaBY0YVZD6kbKq7ARXdSrkmLtIkZ6SMKgjEhywulOklgy1Ed2KdjwboFbSFVziGcCE3Ca3blM0AOFwY", + "M6hmDG7oBP5aPeWzWaq02AXfyFmqkP4Llq3BYLWN1V0YHjNAbzh8I5yPKONVtcU0B9+r5eZVFadtvYKc", + "9ygMZhnmAL3MmGTGZi1bbUtifxrebR2bwWl7o+Q6Z3e8pbEl37mCV1inZSDa6rQcoMB7bNmPzM7LGyJR", + "REXf/QDBEbDQ3E8nVTSyuQVgJVQqGhilD8Pm1lGyTYNFwpE5wetu+4zzhz3ls48co7g4QW2IRvwrsjqh", + "/msjuxksUuXu9rPdZ3tPt5/tNYo5yCe4nsEfgmvS8uTWcvsgSUeuXkbN0g9Pz+Hs0+eqTGOj5Nu1F1w8", + "E8EDLWxShvICHPngz3rPiqEWIU/HUcFoZOOywJ+/SbWUmuutP2g0p5MJ++NTcLX9d0HjrY97cnvs1c2y", + "gfyC7HHR0Lmk9ZFx16Qu9HvDA0IJWRsw8o5IWAE6IwoB/nQRDuCQzjyKLMq5sBILcS9i7e7s7Ow/fbLd", + "CK/s7AqEMwL1c3mWJ3YGBRKDlqj97uwMbRYQzvTp3CwhLQOzApyfzpDNYtwveWBq1WfHhyU18lKONbbv", + "eVwL8gsrBNlFWaCDY1QmIC1RuRfaOzv9p7tP9p80I2OrcI3Ex9UcxqXCMOCx2UOKO98G4/j7g1OkexcT", + "HJQVjK3tnd0ne0/3bzQrdaNZQeYbk7HiBhPbf7r3ZHdne6tZ5JPPAG5j+koEW+ZdHqLzIIVnNzygWGa9", + "nbrTwid4LntjrnQAzT1Kq+6DN/EXzmO+qYReacFVFbW1XFaUcQtxyxtNzBx+FqnHqavCpSXQpq68qz13", + "T7GaHbMJ9yT0uYG+af2hnOU70XKQhJojIWGUhI53ZYqnFa3AwyqSBIUpsZAzopLAFuDY3PJA4h7mZDLK", + "pmXf8qUBm2iBZg6rI/xhXNuwicFK+v1y3osUYGVMzBLh3EOnkb2cypFfUVnuWJBpGmGBqu7qK6YsF3FE", + "2VWT3uUiHvOIBkh/ULUmTHgU8euRfiV/gbVsNFqd/mCUXzBXrANmcta9wGxIZdx8Cb/oVW5UnJvg5N80", + "329CmcUm9j/vrdNLrTsZj+5zRj8WEL0cAru73a/ze6vptOTxthwNcFPeblHWR/HOUf8gy1Trud0090cV", + "pbgsB5fW61stXFCu8vJblgRQ25kUXYhxGa6FUN9GB3GzO9Kq8dzNZlOSoDz67v6Tp3sNY62/StReUYju", + "KwTrebxCoK7ZqZMmUtv+k/1nz3Z2nzzbvpF85O5Zavan7q6luD+VhNQVme1JH/53o0mZmxb/lGpuW8oT", + "KiWXvvWEvqwg3TzGpkbrXlUENt9Jp+aXBfBmIu4KaemgJHIV6ie0yWRCAkXnZGTg1s0nU/HNajSHACc4", + "oGrh0QDxtcnTmTWpxIo06L0yWQ9Ibd823E9zLpmOc3eAthsc/cVodhVc2G+cskGm4zot8m11VKND2txs", + "FQtFAwNBng22eid/nQETXWNZulTQvwNItJfXx6jePpkWzQv5OVzPavnl9+q+eCd/3b7i9le2s6B1lITk", + "KsRXHaH1JKglgsZ5fD0nsq800Hqfjgp/sAfg7b4ajYvJVFZmqyllXslP3ZuP26yyx/J35gS7+XgFB4Kb", + "fFjNKwH4aOdgQZ733SmhRA02KS7WpwG8h+hwY9K+VXy4tYY/SIi4fXwvYeFL23FW8IJq7vPnvvKXYy7d", + "Y+51+zvd/t77rZ3Bk73B1tZ9BChkdxh1ptynn7aun0bbeLIb7S+e/rE1ezrdjne8Xh/3kH6yUkShko3S", + "riEhopoRpJpJR5KIMtKV2fXH+ovoFaFHxiiX4AUIeSs0spuoAa6w6QqqPSsvski8WOXAqSa0fwj/NDv7", + "lbpMdfrHR6unfav7hOpE/AhWnQrgU7PJQMDc1p1mGgW7KlCPF5A1i/KhTOk+voTEH1ZwsN8s4dZxKusW", + "bmeYZ6dwxOTuF0tYk79eApSPxa5OoFE5hMwdaTFfSeaSerfZM97bLa1zYC4kddl+slfO6nLQ/ZvJ4oJG", + "vcHmL3/9390Pf/mTP5NXSXWURHRDMgGJ+Yosuia1uEauXjn61RTskwrb/FeK4Bi4XXBFDHeN8cfifJ/0", + "M1v24g2Ol5YAqkZMWfb32gX5i+ouIZpxNqlLzRprCvEkzKKmUr4NvUeFxqhN4kQtXHips6lv3Mz55SDr", + "sKbC5p067vef3UWY4fnKuMIfMDFw0TfJTWitV9LS/tcG8/iNckdVH2Fj+bbJDss+rZUUblJ16212MU+Z", + "GoHpedm+pt8Zs7YNpJum1YwCmzFTmzZsdznak+AQcpCvvMjIqcw56nTho/X2+ZW3hoWVFWZSvzfGMW05", + "Wm4FgE41aK5nRJDCRsAHeezhDUFmjczrI9OMY46WRLvVbJgmYYugYLW2ADKA1SDILiKWbztW+9ae4I/Z", + "CCCBYrkkLcM6CmVOXz2HJEzvXFZEOnFdwDSqheier8eiJvUYljejiFXL6zbtvYRnedUK7ldHWxXkzMco", + "oeYyPmo2R4JUULU402zIRgBANZeD1KAh8CdYBDzOB4foTCieS+3NaeXmVmtjNEAHp8e2lguDsxxdnKCI", + "TkiwCCJig+uWHNpAn357eNw1UcFZjS89PFUAEJdV++D0GJL0CmnG7fe2e1AyFcoDJbQ1aO30tiBlsQYD", + "LHETkjnAT3vdpukQTr3j0J7Oz00T/ZXAMVFQce13z7WVIsIkh5DgsICnBcEmwVRYySaJ4DLNaGRUfwv+", + "xI7BD8wp0TEAx00dc6VaWNMiSd7abf2g0UEmnEmzodv9vsncyZQ9DnCevHXz79LcfeXjNpIyADwe59ol", + "kc9JOhbkXzqt3f7WjeazNt+qb9hzhm1RIALTfHJDINxq0GNm7jtczVliG+Z0BihUpLDfTaHqNI6xWDhw", + "5bBKuKwT0YhEGDI+mswkf+fjHrKqH4QDyhlPoxDqDSYmp71moxgpLHrTTwiLYEbnZMjs6WFy52IB4dIx", + "0qeGsVuVScMMbXbfsB0i1XMeLirQzbrb1N2BtFUGcDXsSJIRuF+P6tIOZXaThDIG6Uvzqs8u/8YSRzf5", + "pqFkuq/UFcNM5emLTaLpKwLuZRP60dthIz9JzfBgWwjUNcjC77c3/De0EE3md244yt4hC97yIad1BFug", + "K5ME3KUBFmMcRd6sS9OIj3Fk83FfEY/g9ApaWKAUA+/ckct4SEwQVbJQM87M73ScMpWa32PBryUR+mC2", + "wdQW1q4ckUFdKIxAYwhoNqla9JibZoqbn6/I4ktvyA7C2KXhsWU6cSS5TVSeFazKCs8OWW24X43d5NAW", + "LjFJgot5Vc00eaqSVPWQWQhRNgIcmkPaXTkj4ZApjj4LU2Vh8WXzcz7iF5CoCQ41nhSamCVtfqbhl7pZ", + "yxHWqx9BU49OQgAAw5Y+XYYt/XsqsJaoUwmV/YmEOpHT4pa2DWFzAdLKRhXCAWYo4Ulqa8ERZPOvl/qA", + "bBo4ipACUnLfahkIdrJmPfay3Zca0t60m6vRChlBksgCMfV39/30JEkgiE/t/s+zt28QHFV6D0yzPMIT", + "YGTKEmalUfXovSF7gYOZrRcGzv/DFg2HrUzmDTdgrqm0VwHdLghev+ip/WKG6dDwl15Pd2VkugH6/bPp", + "ZaBpKYlHil8RNmx96aDCiylVs3ScvfvgB2jdheVZiRGgtuH9Gy4XEhT4yo9Bc25gFiJueW20QBjlHKio", + "3Y8pw2JlIicP6C0EtYKJp7IIjM9DMPEMW4OhM/IMW51hi7A5PLOWoGHrix8CNvFYvae5yWVlm+VItNfv", + "b6z3JLLw9YjQpYaa/L4sSV/bdyZ4WKFrWfAwi3NhMnoHTVYyI249gOTzHGe1GX+KeGtEPKtPF4Q3+L54", + "Dhj0jYixHVckMK2AR04CW6mdGLSAODHQOJzfn1E4qJPgcuQtqh9VJXNZrdito7IAphg5/Nt9APyDcfPM", + "/jDus4caF0emBpXLc/240BE2yyFix68RvyLqe8C4/kOxUleA5Bvi72PBn1fEyn050CrcbBOKwBfNLdXY", + "Z0FwLG0vprHWVc9gTt0zwhR6AU979l+n8UCo6GXEp5cDZEAY8SmKKCPS+mRkdxj6ULSwhI9MqsbsO5vt", + "NJhhNiUStc35+a9//BMmRdn0X//4p5amzS8g903j7w+RkJczgoUaE6wuB+g3QpIujuicuMVArBKZE7FA", + "O31p68rqV57cqXLIhuwdUalgMvP01+sCmJgObSkPvR7KUiKRBBBCcbqJdUH/JS8666dlA8oHpejOks5l", + "V1BYgD4VHQ6ATyE14aBW/2r5rWdmzSX7WdWCu2TTX89fFPmoDPZ2zQRvyGAAxD66gxd20ah9dvZio4dA", + "xzBYAWEGIDHn3VjhufeTJ63nSYajlBkKQNnwpkLa/Fr775Ft08wAbHv8kSzAdXUA6k3AxuRBBAkdvH7q", + "Ck3MwX64OdOwzz575MoG1htob7/e4hDOT7ORInx3++xwbxnmtn5mDrJvoQKjti1nlqWwLBXp/FZI/yCn", + "RqG2a3Z0IG4SZz6YWnbI2SSigUJdNxfIkhGTTFUrI8hjYQfv7KwRduuqBvQWz7fNUnxK7UmXharkR979", + "nx6VQW9yjORBxzmu/TxJ1qHOEZUB198WsKUb4MQm8DTiS0anRSxaZ5A6gufZkbNSXDrKqj5bgnw405Qd", + "OmXVs+EBmOJRhSF+Q0ZYSUpYCNN/TNh8nu2iK5G8wnL1faFm/+GkoIe2YvnQ/DGZscIK2DQXnGWlqurQ", + "yxazuseNtiN4Fn5GhKNqM1GT4C5flvkUBTMSXJkF2UreqySCY1fsu4nqa/r7kTRfU0XsBhKLBflPEaWB", + "spvDapWCe2wzNd6ffgsj3Ei9vbt7XotgHiCDs8nYWaxNEkQsFyzY+KGueh/kNKtWC39ElHSaRpG78ZgT", + "ofJaasUzYPMzuCWtl+0dta08Ds7fve4SFnDwQ8t8qPxClCtxdLcSvtkws5SfaNJEJwRQOcSoF6C/Yv+N", + "uyDK8uX/2/ZLmzH/37Zfmpz5/7ZzYLLmb9wbsvQfijU/tMT9iJFPC9y0DDRgTaYQ0ToJNWvVUEh17X8o", + "OdUWtbuJpJrB9aew2kRYLYJrpbya1Re8R4nVlmL7NlcyGbL5oA2vnH/iDyapPqyVz2JkoWp/6drDppzk", + "Ii9/Zmt+Pz4HSpphXPHYaGiuzgly5fHhUPf4qGMr25l6dFmAyAMZr908Hly4teM+vOX6IB7TacpTWYw9", + "gUKGRNpgpYiUGfBjE7vz47lW8P6OsbT/kEfHg8vVP/H+niT+6oYa5m1uoNbJ/K5VU5nftoeSgaYahYld", + "e+eqXNg0Khs1ToWuDkxTNC6VLFp2dvTNy6eLoHOtqOTqAgINYjBk/6H1j98VwfGHX1yQTNrvb+/Bc8Lm", + "H35xcTLsxKEKYUpQIhEWBB28OYJrvylEr0MytDwkrzoPk+LM1Ia2ZUv/xylI+c1ncw3JYeFPDamRhlQA", + "12oNKauicp8qkhnkm+lIDt98ALepNX5qSQ+hJcl0MqEBJUzlGYCXnMRsAvFHGFvG7P1QwbmjdNA21pLy", + "0karBdA87d2DO/Zkgz+8cuQy7D1OH3luomJCp47kh2G9PvK94UP/YZnzw+shjxnFjMBfBd0yI9qc2ATE", + "fgHhJRdXTTHPk4fzzhHw7qWT4gq/Q9lET48Uqhx+QxEFDm/jW6+Rpiy5PABBLiVX/ZYunQ4SVrk1QZGU", + "TfM6l1TNeGqyqozsQ5OVTVOFrSYDIk9ge/3W7EWP/gAC6BuuEI2TiMQEsrZ1DTZBcdE0SbjI6o9RWUhF", + "fDP2p8mm6GBrktvYKsAdZBM2g7HObVgb7PbL2+XlmhGfrg+qzQZ3EaSeqNohO5cmyculEYUvUcZkkeJI", + "kogECl3PaDCDCFv9DPo3Abg4SS6zlBobrlhqMbMIDN6WRFAcQZVHHpl6pZfzOL4cLGeAuzg5gY9McK3J", + "9XY5QC7rW3ZASN2qGDGrVxFhqdAbGwfc1pgkeBSZHb3Up1BhfRs2ljZPeTJkvrhaRq5th3SCLgshtpc1", + "MbaOob7mU/mt5KVOfaIqsxbFkQDAGdwkLGzVGXZo5I+u3er3fflTGkb6mmncc6Dv0mRe82mWJKuEyjhJ", + "mqKvnSZg8TyOV+AwaheSmUsV8lT9VaqQCAEfW+yuQ27UxoH5Q+ErjajMliJz6eAB/bzmS5O1xgsqzVQL", + "+aTNX/M4bnVadj6e6rlfHzFd7XDZzKZ3phAW/VPSvknAc5nZFyKeKyeHrVtRL3Lbchw/vL7nyl1/YzT8", + "BvaxfBaUOVEF9javI/64IidNpZaqLGaS5/toJCv1Uk8lZaPyWZ6m/3+gimrWWq3P88BKagZin2ZWKm/x", + "zbXTrNrGTw0101C5QGFqhqvUu/lh1c6MoaCUlTRPK57eVvfMksxlYIY6hGzlhUDO8zY/u5/HtxAXvhNO", + "2Kmt+lKXzihf9PfAcmtqojXiud9ITrLHakFA+IYs2FVne2gOnEFFq3sZl/su2LAhuIwbF3kOVN6nrvDi", + "T2ZcMgMaS+ltmbETPpdsgQX2TFk3iXAdX7Zyai0DtlWgfnh9LddVfnCNLeBCGNcxcEZ7TKGLhTvDgurZ", + "TnAqSScjmI67t744OdmoIxqhVpKM+D4utG8nOVTKcsahvy6yoKFLUn94cmRT2lOJRMp66G1MIXP8FSEJ", + "pKSkPJUIfAB7xXpjdVXQsoJihCmxSDhlau0s8qb3M5kvt0rS/cB8ygZv//BmJVto97ExKeAd+vS2C1it", + "VClTZs97TeeurSgzmfW18IHHPNW9L9VDQxMaEbmQisTmzm6SRkBEkN7DZn+13xnftQ6iSkL18A74+iRE", + "xFRKypkcsjGZaKkkIUKPDQUnaUQK1w++m60zhTOueWpY3/dxtQUl0uA2B6s6qJWro+EkcdXRfNcnWUG3", + "W0/pJdxVIbmIxzyiAYoou5KoHdErI4OjuUSR/rGx8rJrBN/ddW7b21OWhvQxm3Bv+j+Dsxky/wgc7rjC", + "1txl/qNja69IkVgc/4GN9rM1uZavCYIjKAKaudmiVNGIfjKsTndCpaKBqZmEM9hBuRczXm/ITogSug0W", + "BAU8ikignK1hMxE82Bym/f5OkFCIh9ghMDlgePWvYxjx8PQc2pmSNJ0h039Ax+8PThHVMJ1gqzIXJmoL", + "26Pjzbdrrv/PAEz/g/Uxs8BVZOHf8J83uzf3oaylIVlDojxZpQDx5Ic3GFgJ7qe14HFaC8CJPVtNeypw", + "AEKxnKUq5NfMbxkwFVLl5mfz43hdKITCwezClYr+PqRdWy123TBugY+CKO2aQmLSk34Te70t6PtI0zlp", + "wLklgBBTDOrwnwKmUPiPht13f1lXhON3eFNnIepS/343tPXQJ5+dg4vwK8LjsZC5wTS3EihZWbQ+ZeGM", + "a3WzIBWCMAWpYHLRMsAJDqhadBCOXDVVWx4psyHlheDHguArfdL2huxdFkhpyzNp7arjVCsUUnllerDa", + "Uw+9nRMh03E2OQSMyeh5AHxbUDXAUWAqkZLJhASKzokpESprtK9sKveZljcfxLPR7qUF3WNTOfw4AbuX", + "o4XVOkqecrXpG86yVs3SN2S9FrxhCp4iK32eR66hqYJ/E5OdZ/ArWusWb1/dzHvtN/1Rw7HLXlL+SdhX", + "X7nKHyUr3lnBOaVp0occwx9b/oXCzEukWnLwWh8I3tij6z49rNYFgmeDP3Qg+JnXyeeRpaPCJbetugjw", + "7w8R+g/rXfzQEeCPG7e0KCGXQFfPiRpEgn8XGHg/IeDf2Lv+FiHg35W/J4Twfju/++/K09N6LGaenj+D", + "vO/TwdNEekNAa52Dp+F61vK8UlG6sG2aqUm2xx9JgrfGyhvI7w7sP1O2NVAZCsByp3CF3QDvlxbhSZyo", + "hbNG8Qn43eQ5BSX9BN57vsC5zOh8f/Fqt7DH3h16ODyttcb+TPX2YAbfPB/28dHjz+9WpLnSwbKpT50u", + "FsGMzkvxWqso2IIoEaSb8ATsrKEBmIWHO8sUFr3pJ2S77w3Z+xlxfyHqsmWQEIVUkEBFC0SZ4sARzBh/", + "lkhwrQnAey4WPvNtkXJfCh4f2NWsOQ8tTVljWO7mFy+6IVa4O3fcZoUJ7SuurE7wRxqnMTA8RBl69Ry1", + "yUclTPIGNNGaD6KTDKTkY0BIKAEnN4oT3urXWDbpJzKajpvMckUajrc2zQkKUql47Pb++Ai1cap4d0qY", + "3gst6k9Akk0En9PQ5MjNgTrnkYHqVg1Ab2p31UKF9QfPlQszuW8iwzQ5kKafaFJmC8btsTVojSnDMLm1", + "CS/KNGU8cPV4mIIfXE47DnNaP4+wapVtjYlayXFAVJyjSEv0Gz+Pucd8zBU9GdyZVjrtmmUxbebc0NDn", + "4D4ymGaOLw9rtr74fu7jC1WJH6HpfJ4ppHVm8+8LBfsPdz48tLn84hH7b70iTvkumMqhA92jD2Fe8wBH", + "KCRzEvEk1mKladvqtFIRtQatmVLJYHMz0u1mXKrBfn+/3/ry4cv/DwAA//+gy9hgdyoBAA==", } // GetSwagger returns the content of the embedded swagger specification file diff --git a/lib/system/init/egress_proxy.go b/lib/system/init/egress_proxy.go new file mode 100644 index 00000000..50644aff --- /dev/null +++ b/lib/system/init/egress_proxy.go @@ -0,0 +1,42 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + + "github.com/kernel/hypeman/lib/vmconfig" +) + +func installEgressProxyCA(log *Logger, cfg *vmconfig.Config) { + if cfg == nil || cfg.EgressProxy == nil || !cfg.EgressProxy.Enabled || cfg.EgressProxy.CACertPEM == "" { + return + } + + caPath := "/usr/local/share/ca-certificates/hypeman-egress-proxy.crt" + if err := os.MkdirAll(filepath.Dir(caPath), 0755); err != nil { + log.Error("hypeman-init:egress-proxy", "failed to create CA directory", err) + return + } + if err := os.WriteFile(caPath, []byte(cfg.EgressProxy.CACertPEM), 0644); err != nil { + log.Error("hypeman-init:egress-proxy", "failed to write proxy CA", err) + return + } + + updateCACertificatesPath, err := lookupUpdateCACertificatesPath() + if err != nil { + log.Info("hypeman-init:egress-proxy", "installed egress proxy CA certificate, but update-ca-certificates was not found; guest trust store refresh skipped") + return + } + + cmd := exec.Command(updateCACertificatesPath) + if err := cmd.Run(); err != nil { + log.Error("hypeman-init:egress-proxy", "failed to run update-ca-certificates", err) + return + } + log.Info("hypeman-init:egress-proxy", "installed egress proxy CA certificate and refreshed the guest trust store") +} + +func lookupUpdateCACertificatesPath() (string, error) { + return exec.LookPath("update-ca-certificates") +} diff --git a/lib/system/init/egress_proxy_test.go b/lib/system/init/egress_proxy_test.go new file mode 100644 index 00000000..6370f1ff --- /dev/null +++ b/lib/system/init/egress_proxy_test.go @@ -0,0 +1,30 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestLookupUpdateCACertificatesPath(t *testing.T) { + t.Run("returns not found when tool is absent from PATH", func(t *testing.T) { + t.Setenv("PATH", t.TempDir()) + + _, err := lookupUpdateCACertificatesPath() + require.Error(t, err) + }) + + t.Run("finds update-ca-certificates from PATH", func(t *testing.T) { + binDir := t.TempDir() + t.Setenv("PATH", binDir) + + toolPath := filepath.Join(binDir, "update-ca-certificates") + require.NoError(t, os.WriteFile(toolPath, []byte("#!/bin/sh\nexit 0\n"), 0755)) + + foundPath, err := lookupUpdateCACertificatesPath() + require.NoError(t, err) + require.Equal(t, toolPath, foundPath) + }) +} diff --git a/lib/system/init/mode_exec.go b/lib/system/init/mode_exec.go index 1e1792e8..013512f6 100644 --- a/lib/system/init/mode_exec.go +++ b/lib/system/init/mode_exec.go @@ -41,6 +41,8 @@ func runExecMode(log *Logger, cfg *vmconfig.Config) { dropToShell() } + installEgressProxyCA(log, cfg) + // Set up environment os.Setenv("PATH", "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") os.Setenv("HOME", "/root") diff --git a/lib/system/init/mode_systemd.go b/lib/system/init/mode_systemd.go index eb1ffd61..249a9cd2 100644 --- a/lib/system/init/mode_systemd.go +++ b/lib/system/init/mode_systemd.go @@ -54,6 +54,8 @@ func runSystemdMode(log *Logger, cfg *vmconfig.Config) { dropToShell() } + installEgressProxyCA(log, cfg) + // Build effective command from entrypoint + cmd argv := append(cfg.Entrypoint, cfg.Cmd...) if len(argv) == 0 { diff --git a/lib/vmconfig/config.go b/lib/vmconfig/config.go index df1bf6b7..9d073077 100644 --- a/lib/vmconfig/config.go +++ b/lib/vmconfig/config.go @@ -29,6 +29,9 @@ type Config struct { // Boot optimizations SkipKernelHeaders bool `json:"skip_kernel_headers,omitempty"` SkipGuestAgent bool `json:"skip_guest_agent,omitempty"` + + // Optional egress MITM proxy configuration. + EgressProxy *EgressProxyConfig `json:"egress_proxy,omitempty"` } // VolumeMount represents a volume mount configuration. @@ -38,3 +41,10 @@ type VolumeMount struct { Mode string `json:"mode"` // "ro", "rw", or "overlay" OverlayDevice string `json:"overlay_device,omitempty"` } + +// EgressProxyConfig configures guest-side trust and proxy endpoint wiring. +type EgressProxyConfig struct { + Enabled bool `json:"enabled"` + ProxyURL string `json:"proxy_url"` + CACertPEM string `json:"ca_cert_pem,omitempty"` +} diff --git a/openapi.yaml b/openapi.yaml index cc61fc8f..1f0b5e1a 100644 --- a/openapi.yaml +++ b/openapi.yaml @@ -117,6 +117,92 @@ components: team: backend env: staging + CreateInstanceRequestNetworkEgressEnforcement: + type: object + description: Egress enforcement policy applied when mediation is enabled. + properties: + mode: + type: string + enum: [all, http_https_only] + default: all + description: | + `all` (default) rejects direct non-mediated TCP egress from the VM, + while `http_https_only` rejects direct egress only on TCP ports 80 and 443. + example: all + + CreateInstanceRequestNetworkEgress: + type: object + description: | + Host-mediated outbound network policy. + Omit this object, or set `enabled: false`, to preserve normal direct outbound networking + when `network.enabled` is true. + properties: + enabled: + type: boolean + description: | + Whether to enable the mediated egress path. + When false or omitted, the instance keeps normal direct outbound networking and + host-managed credential rewriting is disabled. + default: false + example: true + enforcement: + $ref: "#/components/schemas/CreateInstanceRequestNetworkEgressEnforcement" + + CreateInstanceRequestCredentialSource: + type: object + required: [env] + properties: + env: + type: string + description: | + Name of the real credential in the request `env` map. + The guest-visible env var key can receive a mock placeholder, while the mediated + egress path resolves that placeholder back to this real value only on the host. + example: OUTBOUND_OPENAI_KEY + + CreateInstanceRequestCredentialInjectAs: + type: object + required: [header, format] + description: | + Current v1 transform shape. Header templating is supported now; other transform + types (for example request signing) can be added in future revisions. + properties: + header: + type: string + description: Header name to set/mutate for matching outbound requests. + example: Authorization + format: + type: string + description: Template that must include `${value}`. + example: "Bearer ${value}" + + CreateInstanceRequestCredentialInject: + type: object + required: [as] + properties: + hosts: + type: array + items: + type: string + description: | + Optional destination host patterns (`api.example.com`, `*.example.com`). + Omit to allow injection on all destinations. + example: [api.openai.com, "*.openai.com"] + as: + $ref: "#/components/schemas/CreateInstanceRequestCredentialInjectAs" + + CreateInstanceRequestCredential: + type: object + required: [source, inject] + properties: + source: + $ref: "#/components/schemas/CreateInstanceRequestCredentialSource" + inject: + type: array + items: + $ref: "#/components/schemas/CreateInstanceRequestCredentialInject" + minItems: 1 + CreateInstanceRequest: type: object required: [name, image] @@ -162,6 +248,24 @@ components: example: PORT: "3000" NODE_ENV: production + credentials: + type: object + description: | + Host-managed credential brokering policies keyed by guest-visible env var name. + Those guest env vars receive mock placeholder values, while the real values remain + host-scoped in the request `env` map and are only materialized on the mediated + egress path according to each credential's `source` and `inject` rules. + additionalProperties: + $ref: "#/components/schemas/CreateInstanceRequestCredential" + example: + OUTBOUND_OPENAI_KEY: + source: + env: OUTBOUND_OPENAI_KEY + inject: + - hosts: [api.openai.com, "*.openai.com"] + as: + header: Authorization + format: "Bearer ${value}" tags: $ref: "#/components/schemas/Tags" network: @@ -181,6 +285,8 @@ components: type: string description: Upload bandwidth limit (VM→external, e.g., "1Gbps", "125MB/s"). Defaults to proportional share based on CPU allocation. example: "1Gbps" + egress: + $ref: "#/components/schemas/CreateInstanceRequestNetworkEgress" devices: type: array items: diff --git a/scripts/e2e-install-test.sh b/scripts/e2e-install-test.sh index 764d0353..1ebd78db 100755 --- a/scripts/e2e-install-test.sh +++ b/scripts/e2e-install-test.sh @@ -139,14 +139,34 @@ pass "hypeman ps works" # VM lifecycle test E2E_VM_NAME="e2e-test-vm" +E2E_IMAGE="${HYPEMAN_E2E_IMAGE:-nginx:alpine}" -$HYPEMAN_CMD pull nginx:alpine || fail "hypeman pull failed" +# Pull is async and can fail transiently on first attempt (registry/network startup race). +PULL_OK=false +for i in $(seq 1 5); do + if $HYPEMAN_CMD pull "$E2E_IMAGE"; then + PULL_OK=true + break + fi + warn "hypeman pull attempt ${i}/5 failed for ${E2E_IMAGE}" + sleep 2 +done +if [ "$PULL_OK" != true ]; then + if [ "$OS" = "darwin" ]; then + LOG_FILE="$HOME/Library/Application Support/hypeman/logs/hypeman.log" + if [ -f "$LOG_FILE" ]; then + warn "Service logs (last 100 lines):" + tail -100 "$LOG_FILE" || true + fi + fi + fail "hypeman pull failed" +fi pass "hypeman pull works" # Wait for image to be available (pull is async) IMAGE_READY=false for i in $(seq 1 30); do - if $HYPEMAN_CMD run --name "$E2E_VM_NAME" nginx:alpine 2>&1; then + if $HYPEMAN_CMD run --name "$E2E_VM_NAME" "$E2E_IMAGE" 2>&1; then IMAGE_READY=true break fi