diff --git a/cmd/gcs/main.go b/cmd/gcs/main.go index fd9575ec3b..e5eafdff6b 100644 --- a/cmd/gcs/main.go +++ b/cmd/gcs/main.go @@ -113,6 +113,58 @@ func readMemoryEvents(startTime time.Time, efdFile *os.File, cgName string, thre } } +// Sanity checks the memory limit value +func calculateContainersMemoryLimit(totalMemoryBytes, rootMemReserveBytes uint64) (int64, error) { + if totalMemoryBytes <= rootMemReserveBytes { + return 0, errors.Errorf("total memory %d must be greater than root memory reserve %d", totalMemoryBytes, rootMemReserveBytes) + } + + limit := totalMemoryBytes - rootMemReserveBytes + const maxInt64 = uint64(1<<63 - 1) + if limit > maxInt64 { + return 0, errors.Errorf("containers cgroup memory limit %d exceeds maximum supported value %d", limit, maxInt64) + } + return int64(limit), nil +} + +// Returns the total system memory in bytes. +func currentTotalMemoryBytes() (uint64, error) { + sinfo := syscall.Sysinfo_t{} + if err := syscall.Sysinfo(&sinfo); err != nil { + return 0, errors.Wrap(err, "failed to get sys info") + } + + totalMemoryBytes := uint64(sinfo.Totalram) + unit := uint64(sinfo.Unit) + + // If the unit is non-zero, multiply the total memory by the unit to get the actual memory in bytes. + if unit != 0 { + const maxUint64 = ^uint64(0) + if totalMemoryBytes > maxUint64/unit { + return 0, errors.Errorf("total memory %d with unit %d exceeds maximum supported value", totalMemoryBytes, unit) + } + totalMemoryBytes *= unit + } + return totalMemoryBytes, nil +} + +// Start a periodic goroutine to monitor and update the pods cgroup memory limit. +func monitorCgroupMemoryLimits(h *hcsv2.Host, interval time.Duration) { + ticker := time.NewTicker(interval) + defer ticker.Stop() + for range ticker.C { + podLimit, updated, err := h.UpdateCgroupMemoryLimits() + if err != nil { + logrus.WithError(err).Error("failed to refresh cgroup memory limits") + continue + } + if !updated { + continue + } + logrus.WithField("memoryLimitBytes", podLimit).Debug("refreshed cgroup memory limits") + } +} + // runWithRestartMonitor starts a command with given args and waits for it to exit. If the // command exit code is non-zero the command is restarted with with some back off delay. // Any stdout or stderr of the command will be split into lines and written as a log with @@ -351,11 +403,14 @@ func main() { // // The gcs cgroup is not limited but an event will get logged if memory // usage exceeds 50 MB. - sinfo := syscall.Sysinfo_t{} - if err := syscall.Sysinfo(&sinfo); err != nil { - logrus.WithError(err).Fatal("failed to get sys info") + totalMemoryBytes, err := currentTotalMemoryBytes() + if err != nil { + logrus.WithError(err).Fatal("failed to get total system memory") + } + podsLimit, err := calculateContainersMemoryLimit(totalMemoryBytes, *rootMemReserveBytes) + if err != nil { + logrus.WithError(err).Fatal("failed to calculate pods cgroup memory limit") } - podsLimit := int64(sinfo.Totalram - *rootMemReserveBytes) podsControl, err := cgroup.NewManager("/pods", &oci.LinuxResources{ Memory: &oci.LinuxMemory{ Limit: &podsLimit, @@ -382,6 +437,10 @@ func main() { } h := hcsv2.NewHost(rtime, tport, initialEnforcer, logWriter) + h.InitializeCgroupMemoryLimitUpdater(podsControl, *rootMemReserveBytes, podsLimit) + + // Start periodic task to update the pods cgroup memory limit. + go monitorCgroupMemoryLimits(h, 60*time.Second) // During live migration the VM is frozen and only wakes up when the host // shim is ready, so the vsock port should be immediately available. We diff --git a/cmd/gcs/main_test.go b/cmd/gcs/main_test.go new file mode 100644 index 0000000000..46a74fc2e3 --- /dev/null +++ b/cmd/gcs/main_test.go @@ -0,0 +1,42 @@ +//go:build linux +// +build linux + +package main + +import ( + "strings" + "testing" +) + +func TestCalculateContainersMemoryLimit(t *testing.T) { + tests := []struct { + name string + total uint64 + reserve uint64 + want int64 + wantErrText string + }{ + {name: "subtracts reserve", total: 4096, reserve: 1024, want: 3072}, + {name: "rejects equal reserve", total: 1024, reserve: 1024, wantErrText: "must be greater"}, + {name: "rejects below reserve", total: 512, reserve: 1024, wantErrText: "must be greater"}, + {name: "rejects int64 overflow", total: uint64(1 << 63), wantErrText: "exceeds maximum"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := calculateContainersMemoryLimit(test.total, test.reserve) + if test.wantErrText != "" { + if err == nil || !strings.Contains(err.Error(), test.wantErrText) { + t.Fatalf("expected error containing %q, got %v", test.wantErrText, err) + } + return + } + if err != nil { + t.Fatalf("calculateContainersMemoryLimit returned error: %v", err) + } + if got != test.want { + t.Fatalf("calculateContainersMemoryLimit returned %d, want %d", got, test.want) + } + }) + } +} diff --git a/internal/guest/prot/protocol.go b/internal/guest/prot/protocol.go index ef88bd278e..a09933d924 100644 --- a/internal/guest/prot/protocol.go +++ b/internal/guest/prot/protocol.go @@ -599,6 +599,8 @@ func UnmarshalContainerModifySettings(b []byte) (*containerModifySettings, error return &request, errors.Wrap(err, "failed to unmarshal settings as SecurityPolicyFragment") } msr.Settings = fragment + case guestresource.ResourceTypePodCgroupMemoryLimit: + msr.Settings = nil default: return &request, errors.Errorf("invalid ResourceType '%s'", msr.ResourceType) } diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 45266e4193..49c0f352aa 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -95,9 +95,13 @@ type pod struct { // and processes. type Host struct { // containersMutex guards both containers and pods maps. - containersMutex sync.Mutex - containers map[string]*Container - pods map[string]*pod + containersMutex sync.Mutex + containers map[string]*Container + pods map[string]*pod + cgroupMemoryMutex sync.Mutex + podsCgroup cgroupUpdater + rootMemReserveBytes uint64 + lastAppliedCgroupMemLimit int64 externalProcessesMutex sync.Mutex externalProcesses map[int]*externalProcess @@ -131,6 +135,89 @@ type Host struct { uvmError uvmConsistencyError } +type cgroupUpdater interface { + Update(resources *specs.LinuxResources) error +} + +func calculateCgroupMemoryLimit(totalMemoryBytes, rootMemReserveBytes uint64) (int64, error) { + if totalMemoryBytes <= rootMemReserveBytes { + return 0, errors.Errorf("total memory %d must be greater than root memory reserve %d", totalMemoryBytes, rootMemReserveBytes) + } + + limit := totalMemoryBytes - rootMemReserveBytes + const maxInt64 = uint64(1<<63 - 1) + if limit > maxInt64 { + return 0, errors.Errorf("pods cgroup memory limit %d exceeds maximum supported value %d", limit, maxInt64) + } + return int64(limit), nil +} + +func currentTotalMemoryBytes() (uint64, error) { + sinfo := syscall.Sysinfo_t{} + if err := syscall.Sysinfo(&sinfo); err != nil { + return 0, errors.Wrap(err, "failed to get sys info") + } + + totalMemoryBytes := uint64(sinfo.Totalram) + unit := uint64(sinfo.Unit) + if unit != 0 { + const maxUint64 = ^uint64(0) + if totalMemoryBytes > maxUint64/unit { + return 0, errors.Errorf("total memory %d with unit %d exceeds maximum supported value", totalMemoryBytes, unit) + } + totalMemoryBytes *= unit + } + return totalMemoryBytes, nil +} + +func setCgroupMemoryLimit(podsCgroup cgroupUpdater, totalMemoryBytes, rootMemReserveBytes uint64, lastAppliedLimit int64) (int64, bool, error) { + limit, err := calculateCgroupMemoryLimit(totalMemoryBytes, rootMemReserveBytes) + if err != nil { + return lastAppliedLimit, false, err + } + if limit == lastAppliedLimit { + return limit, false, nil + } + + resources := &specs.LinuxResources{Memory: &specs.LinuxMemory{Limit: &limit}} + if err := podsCgroup.Update(resources); err != nil { + return lastAppliedLimit, false, errors.Wrap(err, "failed to update pods cgroup memory limit") + } + return limit, true, nil +} + +// InitializeCgroupMemoryLimitUpdater configures the cgroup updated when the UVM memory changes. +func (h *Host) InitializeCgroupMemoryLimitUpdater(podsCgroup cgroup.Manager, rootMemReserveBytes uint64, initialLimit int64) { + h.cgroupMemoryMutex.Lock() + defer h.cgroupMemoryMutex.Unlock() + + h.podsCgroup = podsCgroup + h.rootMemReserveBytes = rootMemReserveBytes + h.lastAppliedCgroupMemLimit = initialLimit +} + +// UpdateCgroupMemoryLimits refreshes the pods cgroup limit from the UVM's current memory size. +func (h *Host) UpdateCgroupMemoryLimits() (int64, bool, error) { + h.cgroupMemoryMutex.Lock() + defer h.cgroupMemoryMutex.Unlock() + + if h.podsCgroup == nil { + return h.lastAppliedCgroupMemLimit, false, errors.New("cgroup memory limit updater is not initialized") + } + totalMemoryBytes, err := currentTotalMemoryBytes() + if err != nil { + return h.lastAppliedCgroupMemLimit, false, err + } + limit, updated, err := setCgroupMemoryLimit(h.podsCgroup, totalMemoryBytes, h.rootMemReserveBytes, h.lastAppliedCgroupMemLimit) + if err != nil { + return h.lastAppliedCgroupMemLimit, false, err + } + if updated { + h.lastAppliedCgroupMemLimit = limit + } + return limit, updated, nil +} + type uvmConsistencyError struct { mu sync.Mutex // The error describing why the UVM has entered an inconsistent state. If @@ -925,6 +1012,14 @@ func (h *Host) modifyHostSettings(ctx context.Context, containerID string, req * return errors.New("the request settings are not of type SecurityPolicyFragment") } return h.securityOptions.InjectFragment(ctx, r) + case guestresource.ResourceTypePodCgroupMemoryLimit: + switch req.RequestType { + case guestrequest.RequestTypeUpdate: + _, _, err := h.UpdateCgroupMemoryLimits() + return err + default: + return newInvalidRequestTypeError(req.RequestType) + } default: return errors.Errorf("the ResourceType %q is not supported for UVM", req.ResourceType) } diff --git a/internal/guest/runtime/hcsv2/uvm_state_test.go b/internal/guest/runtime/hcsv2/uvm_state_test.go index 23dd1e50c3..f06db4c492 100644 --- a/internal/guest/runtime/hcsv2/uvm_state_test.go +++ b/internal/guest/runtime/hcsv2/uvm_state_test.go @@ -4,9 +4,97 @@ package hcsv2 import ( + "context" + "strings" "testing" + + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" + oci "github.com/opencontainers/runtime-spec/specs-go" ) +func TestCalculateCgroupMemoryLimit(t *testing.T) { + tests := []struct { + name string + total uint64 + reserve uint64 + want int64 + wantErrText string + }{ + {name: "subtracts reserve", total: 4096, reserve: 1024, want: 3072}, + {name: "rejects equal reserve", total: 1024, reserve: 1024, wantErrText: "must be greater"}, + {name: "rejects below reserve", total: 512, reserve: 1024, wantErrText: "must be greater"}, + {name: "rejects int64 overflow", total: uint64(1 << 63), wantErrText: "exceeds maximum"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + got, err := calculateCgroupMemoryLimit(test.total, test.reserve) + if test.wantErrText != "" { + if err == nil || !strings.Contains(err.Error(), test.wantErrText) { + t.Fatalf("expected error containing %q, got %v", test.wantErrText, err) + } + return + } + if err != nil { + t.Fatalf("calculateCgroupMemoryLimit returned error: %v", err) + } + if got != test.want { + t.Fatalf("calculateCgroupMemoryLimit returned %d, want %d", got, test.want) + } + }) + } +} + +func TestSetCgroupMemoryLimit(t *testing.T) { + podsControl := &testCgroupUpdater{} + limit, updated, err := setCgroupMemoryLimit(podsControl, 4096, 1024, 2048) + if err != nil { + t.Fatalf("setCgroupMemoryLimit returned error: %v", err) + } + if !updated || limit != 3072 || podsControl.limit != 3072 { + t.Fatalf("setCgroupMemoryLimit returned limit=%d, updated=%t, applied=%d; want 3072, true, 3072", limit, updated, podsControl.limit) + } + + limit, updated, err = setCgroupMemoryLimit(podsControl, 4096, 1024, 3072) + if err != nil { + t.Fatalf("setCgroupMemoryLimit returned error: %v", err) + } + if updated || limit != 3072 || podsControl.updates != 1 { + t.Fatalf("unchanged set returned limit=%d, updated=%t, updates=%d; want 3072, false, 1", limit, updated, podsControl.updates) + } +} + +func TestModifyHostSettingsRejectsInvalidPodCgroupMemoryLimitRequestType(t *testing.T) { + for _, requestType := range []guestrequest.RequestType{ + guestrequest.RequestTypeAdd, + guestrequest.RequestTypeRemove, + guestrequest.RequestTypePreAdd, + } { + t.Run(string(requestType), func(t *testing.T) { + host := &Host{} + err := host.modifyHostSettings(context.Background(), UVMContainerID, &guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypePodCgroupMemoryLimit, + RequestType: requestType, + }) + if err == nil || !strings.Contains(err.Error(), "RequestType") { + t.Fatalf("modifyHostSettings returned %v, want invalid RequestType error", err) + } + }) + } +} + +type testCgroupUpdater struct { + limit int64 + updates int +} + +func (cgroup *testCgroupUpdater) Update(resources *oci.LinuxResources) error { + cgroup.updates++ + cgroup.limit = *resources.Memory.Limit + return nil +} + func Test_Add_Remove_RWDevice(t *testing.T) { hm := newHostMounts() mountPath := "/run/gcs/c/abcd" diff --git a/internal/protocol/guestresource/resources.go b/internal/protocol/guestresource/resources.go index c7eecfe140..932b97290f 100644 --- a/internal/protocol/guestresource/resources.go +++ b/internal/protocol/guestresource/resources.go @@ -55,6 +55,8 @@ const ( ResourceTypeSecurityPolicy guestrequest.ResourceType = "SecurityPolicy" // ResourceTypePolicyFragment is the modify resource type for injecting policy fragments. ResourceTypePolicyFragment guestrequest.ResourceType = "SecurityPolicyFragment" + // ResourceTypePodCgroupMemoryLimit is used to force a cgroup memory limit update in the guest. + ResourceTypePodCgroupMemoryLimit guestrequest.ResourceType = "PodCgroupMemoryLimit" ) // This class is used by a modify request to add or remove a combined layers diff --git a/internal/uvm/update_uvm.go b/internal/uvm/update_uvm.go index 86b988e090..129d690010 100644 --- a/internal/uvm/update_uvm.go +++ b/internal/uvm/update_uvm.go @@ -7,6 +7,8 @@ import ( "fmt" hcsschema "github.com/Microsoft/hcsshim/internal/hcs/schema2" + "github.com/Microsoft/hcsshim/internal/protocol/guestrequest" + "github.com/Microsoft/hcsshim/internal/protocol/guestresource" "github.com/Microsoft/hcsshim/pkg/annotations" "github.com/Microsoft/hcsshim/pkg/ctrdtaskapi" "github.com/opencontainers/runtime-spec/specs-go" @@ -54,6 +56,14 @@ func (uvm *UtilityVM) Update(ctx context.Context, data interface{}, annots map[s if err := uvm.UpdateMemory(ctx, *memoryLimitInBytes); err != nil { return err } + if uvm.operatingSystem == "linux" { + if err := uvm.GuestRequest(ctx, guestrequest.ModificationRequest{ + ResourceType: guestresource.ResourceTypePodCgroupMemoryLimit, + RequestType: guestrequest.RequestTypeUpdate, + }); err != nil { + return err + } + } } if processorLimits != nil { if err := uvm.UpdateCPULimits(ctx, processorLimits); err != nil {