diff --git a/Makefile b/Makefile index 8170ff22..73bdb892 100644 --- a/Makefile +++ b/Makefile @@ -154,12 +154,12 @@ docker-run: docker-build ## Build and Run docker image for the Frisbee controlle ##@ Deployment docker-push: docker-build ## Push the latest docker image for Frisbee controller. @echo "===> Tag ${IMG} as frisbee-operator:latest <===" - docker tag ${IMG} $(IMAGE_TAG_BASE)/frisbee-operator:${FrisbeeVersion} - docker tag ${IMG} $(IMAGE_TAG_BASE)/frisbee-operator:latest + docker tag ${IMG} ${IMAGE_TAG_BASE}/frisbee-operator:${FrisbeeVersion} + docker tag ${IMG} ${IMAGE_TAG_BASE}/frisbee-operator:latest @echo "===> Push frisbee operator ${FrisbeeVersion} as latest <===" - docker push $(IMAGE_TAG_BASE)/frisbee-operator:${FrisbeeVersion} - docker push $(IMAGE_TAG_BASE)/frisbee-operator:latest + docker push ${IMAGE_TAG_BASE}/frisbee-operator:${FrisbeeVersion} + docker push ${IMAGE_TAG_BASE}/frisbee-operator:latest install: generate ## Deploy platform to the K8s cluster specified in ~/.kube/config. diff --git a/charts/federated-learning/flower-custom/.gitignore b/charts/federated-learning/flower-custom/.gitignore new file mode 100644 index 00000000..127925ba --- /dev/null +++ b/charts/federated-learning/flower-custom/.gitignore @@ -0,0 +1,3 @@ +# ignore local charts (used for testing) +charts +Chart.lock diff --git a/charts/federated-learning/flower-custom/.helmignore b/charts/federated-learning/flower-custom/.helmignore new file mode 100644 index 00000000..6b0f281e --- /dev/null +++ b/charts/federated-learning/flower-custom/.helmignore @@ -0,0 +1,29 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ + +# Docs +*.md + +# Ignore any figures used in the README.md +examples/README.assets \ No newline at end of file diff --git a/charts/federated-learning/flower-custom/Chart.yaml b/charts/federated-learning/flower-custom/Chart.yaml new file mode 100644 index 00000000..1184c76f --- /dev/null +++ b/charts/federated-learning/flower-custom/Chart.yaml @@ -0,0 +1,31 @@ +--- +apiVersion: v2 +name: flower +description: Flower - A Friendly Federated Learning Framework + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" + +maintainers: + - name: Fotis Nikolaidis + email: nikolaidis.fotis@gmail.com + url: https://www.linkedin.com/in/fotis-nikolaidis-444a6634/ + + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.0 diff --git a/charts/federated-learning/flower-custom/README.md b/charts/federated-learning/flower-custom/README.md new file mode 100644 index 00000000..9442f144 --- /dev/null +++ b/charts/federated-learning/flower-custom/README.md @@ -0,0 +1,5 @@ +# Flower + + + +## Parameters diff --git a/charts/federated-learning/flower/examples/1.baseline.yml b/charts/federated-learning/flower-custom/examples/1.baseline.yml similarity index 100% rename from charts/federated-learning/flower/examples/1.baseline.yml rename to charts/federated-learning/flower-custom/examples/1.baseline.yml diff --git a/charts/federated-learning/flower/examples/10.scheduled-join.yml b/charts/federated-learning/flower-custom/examples/10.scheduled-join.yml similarity index 100% rename from charts/federated-learning/flower/examples/10.scheduled-join.yml rename to charts/federated-learning/flower-custom/examples/10.scheduled-join.yml diff --git a/charts/federated-learning/flower/examples/11.scaled-baseline.yml b/charts/federated-learning/flower-custom/examples/11.scaled-baseline.yml similarity index 100% rename from charts/federated-learning/flower/examples/11.scaled-baseline.yml rename to charts/federated-learning/flower-custom/examples/11.scaled-baseline.yml diff --git a/charts/federated-learning/flower/examples/12.coverage.yml b/charts/federated-learning/flower-custom/examples/12.coverage.yml similarity index 100% rename from charts/federated-learning/flower/examples/12.coverage.yml rename to charts/federated-learning/flower-custom/examples/12.coverage.yml diff --git a/charts/federated-learning/flower/examples/13.scaled-with-logs.yml b/charts/federated-learning/flower-custom/examples/13.scaled-with-logs.yml similarity index 100% rename from charts/federated-learning/flower/examples/13.scaled-with-logs.yml rename to charts/federated-learning/flower-custom/examples/13.scaled-with-logs.yml diff --git a/charts/federated-learning/flower/examples/2.throttle-server.yml b/charts/federated-learning/flower-custom/examples/2.throttle-server.yml similarity index 100% rename from charts/federated-learning/flower/examples/2.throttle-server.yml rename to charts/federated-learning/flower-custom/examples/2.throttle-server.yml diff --git a/charts/federated-learning/flower/examples/3.throttle-client.yml b/charts/federated-learning/flower-custom/examples/3.throttle-client.yml similarity index 100% rename from charts/federated-learning/flower/examples/3.throttle-client.yml rename to charts/federated-learning/flower-custom/examples/3.throttle-client.yml diff --git a/charts/federated-learning/flower/examples/4a.partition-server-init.yml b/charts/federated-learning/flower-custom/examples/4a.partition-server-init.yml similarity index 100% rename from charts/federated-learning/flower/examples/4a.partition-server-init.yml rename to charts/federated-learning/flower-custom/examples/4a.partition-server-init.yml diff --git a/charts/federated-learning/flower/examples/4b.partition-server-progress.yml b/charts/federated-learning/flower-custom/examples/4b.partition-server-progress.yml similarity index 100% rename from charts/federated-learning/flower/examples/4b.partition-server-progress.yml rename to charts/federated-learning/flower-custom/examples/4b.partition-server-progress.yml diff --git a/charts/federated-learning/flower/examples/5.partition-client.yml b/charts/federated-learning/flower-custom/examples/5.partition-client.yml similarity index 100% rename from charts/federated-learning/flower/examples/5.partition-client.yml rename to charts/federated-learning/flower-custom/examples/5.partition-client.yml diff --git a/charts/federated-learning/flower/examples/6.partition-p2p.yml b/charts/federated-learning/flower-custom/examples/6.partition-p2p.yml similarity index 100% rename from charts/federated-learning/flower/examples/6.partition-p2p.yml rename to charts/federated-learning/flower-custom/examples/6.partition-p2p.yml diff --git a/charts/federated-learning/flower/examples/7.network-loss.yml b/charts/federated-learning/flower-custom/examples/7.network-loss.yml similarity index 100% rename from charts/federated-learning/flower/examples/7.network-loss.yml rename to charts/federated-learning/flower-custom/examples/7.network-loss.yml diff --git a/charts/federated-learning/flower/examples/8.network-duplicates.yml b/charts/federated-learning/flower-custom/examples/8.network-duplicates.yml similarity index 100% rename from charts/federated-learning/flower/examples/8.network-duplicates.yml rename to charts/federated-learning/flower-custom/examples/8.network-duplicates.yml diff --git a/charts/federated-learning/flower/examples/9.network-delay.yml b/charts/federated-learning/flower-custom/examples/9.network-delay.yml similarity index 100% rename from charts/federated-learning/flower/examples/9.network-delay.yml rename to charts/federated-learning/flower-custom/examples/9.network-delay.yml diff --git a/charts/federated-learning/flower/templates/client-with-coverage.yml b/charts/federated-learning/flower-custom/templates/client-with-coverage.yml similarity index 100% rename from charts/federated-learning/flower/templates/client-with-coverage.yml rename to charts/federated-learning/flower-custom/templates/client-with-coverage.yml diff --git a/charts/federated-learning/flower/templates/client-with-logs.yml b/charts/federated-learning/flower-custom/templates/client-with-logs.yml similarity index 100% rename from charts/federated-learning/flower/templates/client-with-logs.yml rename to charts/federated-learning/flower-custom/templates/client-with-logs.yml diff --git a/charts/federated-learning/flower/templates/client.yml b/charts/federated-learning/flower-custom/templates/client.yml similarity index 100% rename from charts/federated-learning/flower/templates/client.yml rename to charts/federated-learning/flower-custom/templates/client.yml diff --git a/charts/federated-learning/flower/templates/datasets.yml b/charts/federated-learning/flower-custom/templates/datasets.yml similarity index 100% rename from charts/federated-learning/flower/templates/datasets.yml rename to charts/federated-learning/flower-custom/templates/datasets.yml diff --git a/charts/federated-learning/flower/templates/server-with-logs.yml b/charts/federated-learning/flower-custom/templates/server-with-logs.yml similarity index 100% rename from charts/federated-learning/flower/templates/server-with-logs.yml rename to charts/federated-learning/flower-custom/templates/server-with-logs.yml diff --git a/charts/federated-learning/flower/templates/server.yml b/charts/federated-learning/flower-custom/templates/server.yml similarity index 100% rename from charts/federated-learning/flower/templates/server.yml rename to charts/federated-learning/flower-custom/templates/server.yml diff --git a/charts/federated-learning/flower-custom/values.yaml b/charts/federated-learning/flower-custom/values.yaml new file mode 100644 index 00000000..e69de29b diff --git a/charts/federated-learning/flower/containers/README.md b/charts/federated-learning/flower/containers/README.md new file mode 100644 index 00000000..7ae41138 --- /dev/null +++ b/charts/federated-learning/flower/containers/README.md @@ -0,0 +1,13 @@ + + +== Advanced Pytorch == +``` shell +docker build -t icsforth/advanced_pytorch -f advanced_pytorch.Dockerfile . +docker push icsforth/advanced_pytorch +``` + +== Advanced Tensorflow == +``` shell +docker build -t icsforth/advanced_tensorflow -f advanced_tensorflow.Dockerfile . +docker push icsforth/advanced_tensorflow +``` \ No newline at end of file diff --git a/charts/federated-learning/flower/containers/advanced_pytorch.Dockerfile b/charts/federated-learning/flower/containers/advanced_pytorch.Dockerfile new file mode 100644 index 00000000..b12f9dc2 --- /dev/null +++ b/charts/federated-learning/flower/containers/advanced_pytorch.Dockerfile @@ -0,0 +1,29 @@ +# Build container for the advanced_pytorch example according to the following instructions: +# https://github.com/adap/flower/tree/main/examples/advanced_pytorch +FROM bitnami/pytorch + +USER root + +RUN apt-get update && apt-get install -y git + +# Install dependencies +RUN pip install poetry + +# Clone the example project +RUN git clone --depth=1 https://github.com/adap/flower.git \ + && mv flower/examples/advanced_pytorch . \ + && rm -rf flower + +WORKDIR ./advanced_pytorch + +# Download Flower dependencies +RUN poetry install + +# Download the EfficientNetB0 model +RUN python -c "import torch; torch.hub.load( \ + 'NVIDIA/DeepLearningExamples:torchhub', \ + 'nvidia_efficientnet_b0', pretrained=True)" + +# Download the CIFAR-10 dataset +RUN python -c "from torchvision.datasets import CIFAR10; CIFAR10('./dataset', download=True)" + diff --git a/charts/federated-learning/flower/containers/advanced_tensorflow.Dockerfile b/charts/federated-learning/flower/containers/advanced_tensorflow.Dockerfile new file mode 100644 index 00000000..03bfeac7 --- /dev/null +++ b/charts/federated-learning/flower/containers/advanced_tensorflow.Dockerfile @@ -0,0 +1,23 @@ +# Build container for the advanced_pytorch example according to the following instructions: +# https://github.com/adap/flower/tree/main/examples/advanced_tensorflow +FROM tensorflow/tensorflow + +USER root + +RUN apt-get update && apt-get install -y git + +# Install dependencies +RUN pip install poetry + +# Clone the example project +RUN git clone --depth=1 https://github.com/adap/flower.git \ + && mv flower/examples/advanced_tensorflow . \ + && rm -rf flower + +WORKDIR ./advanced_tensorflow + +# Download Flower dependencies +RUN poetry install + +# Download the CIFAR-10 dataset +RUN python -c "import tensorflow as tf; tf.keras.datasets.cifar10.load_data()" \ No newline at end of file diff --git a/charts/federated-learning/flower/examples/advanced_pytorch.yaml b/charts/federated-learning/flower/examples/advanced_pytorch.yaml new file mode 100644 index 00000000..ebe5c27d --- /dev/null +++ b/charts/federated-learning/flower/examples/advanced_pytorch.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: frisbee.dev/v1alpha1 +kind: Scenario +metadata: + name: advanced-pytoch +spec: + actions: + # Run the default script + - action: Service + name: script + service: + templateRef: flower.advanced-pytorch.standalone + + # Teardown + - action: Delete + name: teardown + depends: { success: [ script ] } + delete: + jobs: [] + diff --git a/charts/federated-learning/flower/examples/advanced_tensorflow.yaml b/charts/federated-learning/flower/examples/advanced_tensorflow.yaml new file mode 100644 index 00000000..473a7c83 --- /dev/null +++ b/charts/federated-learning/flower/examples/advanced_tensorflow.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: frisbee.dev/v1alpha1 +kind: Scenario +metadata: + name: advanced-tensorflow +spec: + actions: + # Run the default script + - action: Service + name: script + service: + templateRef: flower.advanced-tensorflow.standalone + + # Teardown + - action: Delete + name: teardown + depends: { success: [ script ] } + delete: + jobs: [] + diff --git a/charts/federated-learning/flower/templates/advanced_pytorch.yaml b/charts/federated-learning/flower/templates/advanced_pytorch.yaml new file mode 100644 index 00000000..4da95c4f --- /dev/null +++ b/charts/federated-learning/flower/templates/advanced_pytorch.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: frisbee.dev/v1alpha1 +kind: Template +metadata: + name: flower.advanced-pytorch.standalone +spec: + service: + decorators: + telemetry: [ system.telemetry.agent] + + containers: + - name: app + image: icsforth/advanced_pytorch + command: ["/bin/sh", "-c"] + args: + - | # Script + set -eum + cut -d ' ' -f 4 /proc/self/stat > /dev/shm/app # Sidecar: use it for entering the cgroup + + poetry run ./run.sh \ No newline at end of file diff --git a/charts/federated-learning/flower/templates/advanced_tensorflow.yaml b/charts/federated-learning/flower/templates/advanced_tensorflow.yaml new file mode 100644 index 00000000..d09abdb1 --- /dev/null +++ b/charts/federated-learning/flower/templates/advanced_tensorflow.yaml @@ -0,0 +1,20 @@ +--- +apiVersion: frisbee.dev/v1alpha1 +kind: Template +metadata: + name: flower.advanced-tensorflow.standalone +spec: + service: + decorators: + telemetry: [ system.telemetry.agent] + + containers: + - name: app + image: icsforth/advanced_tensorflow + command: ["/bin/sh", "-c"] + args: + - | # Script + set -eum + cut -d ' ' -f 4 /proc/self/stat > /dev/shm/app # Sidecar: use it for entering the cgroup + + poetry run ./run.sh \ No newline at end of file diff --git a/charts/system/templates/telemetry/agent/agent.yml b/charts/system/templates/telemetry/agent/agent.yml index 7cc2bd06..896fd031 100644 --- a/charts/system/templates/telemetry/agent/agent.yml +++ b/charts/system/templates/telemetry/agent/agent.yml @@ -50,7 +50,6 @@ spec: # Tap into the cgroup of the main container, and mount it locally nsenter -t $(cat /dev/shm/app) -C -- mount -t cgroup2 none /sys/fs/cgroup - cadvisor --port=9442 \ --docker_only=true \ --store_container_labels=false \ diff --git a/cmd/kubectl-frisbee/commands/tests/report.go b/cmd/kubectl-frisbee/commands/tests/report.go index 0abf9de9..36400e85 100644 --- a/cmd/kubectl-frisbee/commands/tests/report.go +++ b/cmd/kubectl-frisbee/commands/tests/report.go @@ -44,8 +44,8 @@ const ( SummaryDashboardUID = "summary" ) -func GenerateQuotedURL(grafanaEndpoint string, dashboard string, from int64, to int64) string { - return fmt.Sprintf("http://%s/d/%s?orgId=1&from=%d&to=%d&kiosk=", grafanaEndpoint, dashboard, from, to) +func GenerateQuotedURL(grafanaEndpoint string, dashboard string, from int64, to int64, postfix string) string { + return fmt.Sprintf("http://%s/d/%s?orgId=1&from=%d&to=%d%s", grafanaEndpoint, dashboard, from, to, postfix) } type TestReportOptions struct { @@ -95,9 +95,9 @@ func NewReportTestsCmd() *cobra.Command { ui.Fail(errors.Wrap(err, "Cannot chdir to Frisbee cache")) } - /* - Inspect the Scenario for Grafana Endpoints. - */ + /*---------------------------------------------------* + * Inspect the Scenario for Grafana Endpoints. + *---------------------------------------------------*/ scenario, err := env.Default.GetFrisbeeClient().GetScenario(cmd.Context(), testName) ui.ExitOnError("Getting test information", err) @@ -113,19 +113,20 @@ func NewReportTestsCmd() *cobra.Command { } } - /* - Filter time to the beginning/ending of the scenario. - */ + /*-- Filter time to the beginning/ending of the scenario. --*/ from, to := FindTimeline(scenario) - uri := GenerateQuotedURL(scenario.Status.GrafanaEndpoint, options.DashboardUID, from, to) - /* - Generate PDFs for each panel, in parallel. - */ + /*---------------------------------------------------* + * Generate PDFs for the specific timeline + *---------------------------------------------------*/ if options.Aggregate { + uri := GenerateQuotedURL(scenario.Status.GrafanaEndpoint, options.DashboardUID, from, to, "") + err = SavePDF(&options, uri, destination) ui.ExitOnError("Saving aggregated report to: "+destination, err) } else { + uri := GenerateQuotedURL(scenario.Status.GrafanaEndpoint, options.DashboardUID, from, to, "&kiosk") + err = SavePDFs(&options, uri, destination, scenario.Status.GrafanaEndpoint, options.DashboardUID) ui.ExitOnError("Saving reports to: "+destination, err) } @@ -173,16 +174,16 @@ var ( ) func SavePDFs(options *TestReportOptions, dashboardURI, destDir string, grafanaEndpoint string, dashboardUID string) error { - /* - Ensure destination exists - */ + /*---------------------------------------------------* + * Ensure destination exists + *---------------------------------------------------*/ if err := os.MkdirAll(destDir, os.ModePerm); err != nil { return errors.Wrapf(err, "destination error") } - /* - Open Connection to Grafana. - */ + /*---------------------------------------------------* + * Query Grafna for Available Pannels. + *---------------------------------------------------*/ ctx := context.Background() grafanaClient, err := grafana.New(ctx, grafana.WithHTTP(grafanaEndpoint)) @@ -190,20 +191,18 @@ func SavePDFs(options *TestReportOptions, dashboardURI, destDir string, grafanaE return errors.Wrapf(err, "cannot get Grafana client") } - /* - List Available Panels - */ panels, err := grafanaClient.ListPanelsWithData(ctx, dashboardUID) if err != nil { return err } - /* - Generate PDF for each Panel. - */ + /*---------------------------------------------------* + * Generate PDF for each Panel. + *---------------------------------------------------*/ for i, panel := range panels { panelURI := fmt.Sprintf("%s&viewPanel=%d", dashboardURI, panel.ID) - // replace special characters with underscore + + /*-- replace special characters with underscore --*/ title := nonAlphanumericRegex.ReplaceAllString(panel.Title, "_") title = removeDuplicatesRegex.ReplaceAllString(title, "_") @@ -217,14 +216,11 @@ func SavePDFs(options *TestReportOptions, dashboardURI, destDir string, grafanaE return nil } -/* -****************************************************************** - - Install PDF-Exporter. +/*---------------------------------------------------* + Install PDF-Exporter. This is required for generating pdfs from Grafana. + *---------------------------------------------------*/ -****************************************************************** -*/ const ( puppeteer = "puppeteer" ) @@ -238,9 +234,9 @@ var ( ) func InstallPDFExporter(location string) { - /* - Ensure that the Cache Dir exists. - */ + /*---------------------------------------------------* + * Ensure that the Cache Dir exists. + *---------------------------------------------------*/ _, err := os.Open(location) if err != nil && !os.IsNotExist(err) { ui.Failf("failed to open cache directory " + location) @@ -249,9 +245,9 @@ func InstallPDFExporter(location string) { err = os.MkdirAll(location, os.ModePerm) ui.ExitOnError("create cache directory:"+location, err) - /* - Install NodeJS dependencies - */ + /*---------------------------------------------------* + * Install NodeJS dependencies + *---------------------------------------------------*/ ui.Info("Installing PDFExporter ...") oldPwd, _ := os.Getwd() @@ -268,19 +264,18 @@ func InstallPDFExporter(location string) { _, err = process.Execute("sh", "-c", strings.Join(command, " ")) ui.ExitOnError(" --> Installing Puppeteer", err) - /* - Copy the embedded pdf exporter in the underlying filesystem. - */ + /*---------------------------------------------------* + * Copy the embedded pdf exporter into fs + *---------------------------------------------------*/ err = embed.CopyLocallyIfNotExists(embed.Hack, location) ui.ExitOnError(" --> Install PDF Renderer", err) err = os.Chdir(oldPwd) ui.ExitOnError("Returning to "+oldPwd, err) - /* - Update path to binary - */ - + /*---------------------------------------------------* + * Update path to the pdf-exporter binary + *---------------------------------------------------*/ FastPDFExporter = filepath.Join(location, "hack/pdf-exporter/fast-generator.js") LongPDFExporter = filepath.Join(location, "hack/pdf-exporter/long-dashboards.js") @@ -290,42 +285,42 @@ func InstallPDFExporter(location string) { ui.Success("PDFExporter is installed at ", location) } -/* -FindTimeline parses the scenario to find timeline that make sense (formatted into time.UnixMilli). -For the starting time we adhere to these rules: - 1. If possible, we use the time that the first job was scheduled. - 2. Otherwise, we use the Creation time. - -For the ending time we adhere to these rules: - 1. If the scenario is successful, we return the ConditionAllJobsAreCompleted time. - 2. If the scenario has failed, we return the Failure time. - 3. Otherwise, we report time.Now(). -*/ -func FindTimeline(in *v1alpha1.Scenario) (from int64, to int64) { - initialized := meta.FindStatusCondition(in.Status.Conditions, v1alpha1.ConditionCRInitialized.String()) +// FindTimeline parses the scenario to find timeline that make sense (formatted into time.UnixMilli). +/*---------------------------------------------------* + For the starting time we adhere to these rules: + 1. If possible, we use the time that the first job was scheduled. + 2. Otherwise, we use the Creation time. + + For the ending time we adhere to these rules: + 1. If the scenario is successful, we return the ConditionAllJobsAreCompleted time. + 2. If the scenario has failed, we return the Failure time. + 3. Otherwise, we report time.Now(). + *---------------------------------------------------*/ +func FindTimeline(scenario *v1alpha1.Scenario) (from int64, to int64) { + initialized := meta.FindStatusCondition(scenario.Status.Conditions, v1alpha1.ConditionCRInitialized.String()) if initialized != nil { from = initialized.LastTransitionTime.Time.UnixMilli() } else { - from = in.GetCreationTimestamp().Time.UnixMilli() + from = scenario.GetCreationTimestamp().Time.UnixMilli() } to = time.Now().UnixMilli() - switch in.Status.Phase { + switch scenario.Status.Phase { case v1alpha1.PhaseSuccess: - success := meta.FindStatusCondition(in.Status.Conditions, v1alpha1.ConditionAllJobsAreCompleted.String()) + success := meta.FindStatusCondition(scenario.Status.Conditions, v1alpha1.ConditionAllJobsAreCompleted.String()) to = success.LastTransitionTime.Time.UnixMilli() case v1alpha1.PhaseFailed: // Failure may come from various reasons. Unfortunately we have to go through all of them. - unexpected := meta.FindStatusCondition(in.Status.Conditions, v1alpha1.ConditionJobUnexpectedTermination.String()) + unexpected := meta.FindStatusCondition(scenario.Status.Conditions, v1alpha1.ConditionJobUnexpectedTermination.String()) if unexpected != nil { to = unexpected.LastTransitionTime.Time.UnixMilli() break } - assert := meta.FindStatusCondition(in.Status.Conditions, v1alpha1.ConditionAssertionError.String()) + assert := meta.FindStatusCondition(scenario.Status.Conditions, v1alpha1.ConditionAssertionError.String()) if assert != nil { to = assert.LastTransitionTime.Time.UnixMilli() diff --git a/controllers/common/controller.go b/controllers/common/controller.go index fcc2b329..faa3c379 100644 --- a/controllers/common/controller.go +++ b/controllers/common/controller.go @@ -56,6 +56,26 @@ func RequeueWithError(err error) (ctrl.Result, error) { return ctrl.Result{}, err } +type Logger interface { + // Info logs a non-error message with the given key/value pairs as context. + // The level argument is provided for optional logging. This method will + // only be called when Enabled(level) is true. See Logger.Info for more + // details. + Info(msg string, keysAndValues ...interface{}) + + // Error logs an error, with the given message and key/value pairs as + // context. See Logger.Error for more details. + Error(err error, msg string, keysAndValues ...interface{}) + + // WithValues returns a new LogSink with additional key/value pairs. See + // Logger.WithValues for more details. + WithValues(keysAndValues ...interface{}) logr.Logger + + // WithName returns a new LogSink with the specified name appended. See + // Logger.WithName for more details. + WithName(name string) logr.Logger +} + // Reconciler implements basic functionality that is common to every solid reconciler (e.g, finalizers). type Reconciler interface { GetClient() client.Client @@ -63,11 +83,7 @@ type Reconciler interface { GetEventRecorderFor(name string) record.EventRecorder - // Logging - - Error(err error, msg string, keysAndValues ...interface{}) - Info(msg string, keysAndValues ...interface{}) - V(level int) logr.Logger + Logger Finalizer() string @@ -87,17 +103,20 @@ type Reconciler interface { // Bool indicate whether the caller should return immediately (true) or continue (false). // The reconciliation cycle is where the framework gives us back control after a watch has passed up an event. func Reconcile(ctx context.Context, r Reconciler, req ctrl.Request, obj client.Object, requeue *bool) (ctrl.Result, error) { - // make the calling controller to return + /*-- make the calling controller to return --*/ *requeue = true - /* - ### 1: Retrieve the CR by name - */ + /*--------------------------------------------------- + * Retrieve CR by name + *---------------------------------------------------*/ + if err := r.GetClient().Get(ctx, req.NamespacedName, obj); err != nil { - // Request object not found, could have been deleted after reconcile request. - // We'll ignore not-found errors, since they can't be fixed by an immediate - // requeue (we'll need to wait for a new notification), and we can get them - // on added / deleted requests. + /*-- + Request object not found, could have been deleted after reconcile request. + We'll ignore not-found errors, since they can't be fixed by an immediate + requeue (we'll need to wait for a new notification), and we can get them + on added / deleted requests. + --*/ if k8errors.IsNotFound(err) { return Stop() } @@ -105,19 +124,19 @@ func Reconcile(ctx context.Context, r Reconciler, req ctrl.Request, obj client.O return RequeueWithError(err) } + logger := r.WithValues("obj", client.ObjectKeyFromObject(obj)) + + /*--------------------------------------------------- + * Set Finalizers for CR + *---------------------------------------------------*/ /* - ### 2: Manage Resource initialization Finalizers provide a mechanism to inform the Kubernetes control plane that an action needs to take place before the standard Kubernetes garbage collection logic can be performed. */ - if obj.GetDeletionTimestamp().IsZero() { - // The object is not being deleted, so if it does not have our finalizer, - // then lets add the finalizer and Update the object. This is equivalent - // registering our finalizer. + /*-- Add Finalizers to a new CR--*/ if controllerutil.AddFinalizer(obj, r.Finalizer()) { - r.Info("AddFinalizer", - "obj", client.ObjectKeyFromObject(obj), + logger.Info("AddFinalizer", "finalizer", r.Finalizer(), "current", obj.GetFinalizers()) @@ -130,30 +149,28 @@ func Reconcile(ctx context.Context, r Reconciler, req ctrl.Request, obj client.O return true, nil }, ); err != nil { - r.Error(err, "Abort retrying to add finalizer", "obj", client.ObjectKeyFromObject(obj)) + logger.Error(err, "Abort retrying to add finalizer") } return Stop() } } else { - // The object is being deleted + /*-- Handle and Remove Finalizers for a deleted CR-*/ if controllerutil.ContainsFinalizer(obj, r.Finalizer()) { - // our finalizer is present, so lets handle any external dependency. + /*-- Handle finalization logic to remove external dependencies. --*/ if err := r.Finalize(obj); err != nil { - // Run finalization logic to remove external dependencies. - // If the finalization logic fails, don't remove the finalizer - // so that we can retry during the next reconciliation. - r.Error(err, "Finalize error", - "obj", client.ObjectKeyFromObject(obj), - "finalizer", r.Finalizer()) + logger.Error(err, "Finalize error", "finalizer", r.Finalizer()) + /*-- + If the finalization logic fails, don't remove the finalizer + so that we can retry during the next reconciliation. + --*/ return RequeueWithError(err) } - // Once all finalizers have been removed, the object will be deleted. + /*-- Remove Finalizer. Once all finalizers have been removed, the object will be deleted. --*/ if controllerutil.RemoveFinalizer(obj, r.Finalizer()) { - r.Info("RemoveFinalizer", - "obj", client.ObjectKeyFromObject(obj), + logger.Info("RemoveFinalizer", "finalizer", r.Finalizer(), "current", obj.GetFinalizers(), ) @@ -167,7 +184,7 @@ func Reconcile(ctx context.Context, r Reconciler, req ctrl.Request, obj client.O return true, nil }, ); err != nil { - r.Error(err, "Abort retrying to remove finalizer", "obj", client.ObjectKeyFromObject(obj)) + logger.Error(err, "Abort retrying to remove finalizer") } return Stop() diff --git a/hack/pdf-exporter/fast-generator.js b/hack/pdf-exporter/fast-generator.js index 62956dae..b1fe5e90 100644 --- a/hack/pdf-exporter/fast-generator.js +++ b/hack/pdf-exporter/fast-generator.js @@ -13,7 +13,7 @@ const outfile = process.argv[4]; // Set the browser width in pixels. The paper size will be calculated on the basus of 96dpi, // so 1200 corresponds to 12.5". -const width_px = 1200; +const width_px = 1632; // Note that to get an actual paper size, e.g. Letter, you will want to *not* simply set the pixel // size here, since that would lead to a "mobile-sized" screen (816px), and mess up the rendering. // Instead, set e.g. double the size here (1632px), and call page.pdf() with format: 'Letter' and @@ -38,7 +38,7 @@ const auth_header = 'Basic ' + new Buffer.from(auth_string).toString('base64'); await page.setViewport({ width: width_px, height: 800, - deviceScaleFactor: 2, + deviceScaleFactor: 1, isMobile: false }) @@ -64,6 +64,7 @@ const auth_header = 'Basic ' + new Buffer.from(auth_string).toString('base64'); // Get the height of the main canvas, and add a margin var height_px = await page.evaluate(() => { + document.querySelector('.react-grid-layout').style.height = 'auto'; return document.getElementsByClassName('react-grid-layout')[0].getBoundingClientRect().bottom; }) + 20; @@ -71,7 +72,7 @@ const auth_header = 'Basic ' + new Buffer.from(auth_string).toString('base64'); path: outfile, width: width_px + 'px', height: height_px + 'px', -// format: 'Letter', <-- see note above for generating "paper-sized" outputs + format: 'Letter', // <-- see note above for generating "paper-sized" outputs scale: 1, displayHeaderFooter: false, printBackground: true, diff --git a/pkg/configuration/configuration.go b/pkg/configuration/configuration.go index aba4cbb4..93755a1e 100644 --- a/pkg/configuration/configuration.go +++ b/pkg/configuration/configuration.go @@ -68,7 +68,7 @@ func namesOfItems(list corev1.ConfigMapList) []string { return names } -// Get returns the system configuration +// Get returns the system configuration. func Get(ctx context.Context, cli client.Client, logger logr.Logger) (Configuration, error) { // 1. Discovery the configuration across the various namespaces. var list corev1.ConfigMapList diff --git a/pkg/lifecycle/calculator.go b/pkg/lifecycle/stateMapper.go similarity index 100% rename from pkg/lifecycle/calculator.go rename to pkg/lifecycle/stateMapper.go