[Elastic Agent] Disable modules.d in metricbeat

Currently when Elastic Agent is started, on post-install of metricbeat the system.yml file in the modules.d directoy is renamed to system.yml.disabled to make sure it is not run. It turns out there might be some edge cases where this does not work. To not rely on renaming of files, instead the modules are fully disabled.

Closes #27857

x-pack/elastic-agent/spec/metricbeat.yml

@@ -1,101 +1,98 @@
 name: Metricbeat
 cmd: metricbeat
 args: [
-  "-E", "setup.ilm.enabled=false",
-  "-E", "setup.template.enabled=false",
-  "-E", "management.enabled=true",
-  "-E", "logging.level=debug",
-  "-E", "gc_percent=${METRICBEAT_GOGC:100}"
+    "-E", "setup.ilm.enabled=false",
+    "-E", "setup.template.enabled=false",
+    "-E", "management.enabled=true",
+    "-E", "logging.level=debug",
+    "-E", "gc_percent=${METRICBEAT_GOGC:100}",
+    "-E", "metricbeat.config.modules.enabled=false"
 ]
 artifact: beats/metricbeat
 restart_on_output_change: true
-post_install:
-  - move_file:
-      path: "modules.d/system.yml"
-      target: "modules.d/system.yml.disabled"
 rules:
-- fix_stream: {}
-- inject_index:
-    type: metrics
+  - fix_stream: {}
+  - inject_index:
+      type: metrics
 
-- inject_stream_processor:
-    on_conflict: insert_after
-    type: metrics
+  - inject_stream_processor:
+      on_conflict: insert_after
+      type: metrics
 
-- rename:
-    from: inputs
-    to: inputsstreams
+  - rename:
+      from: inputs
+      to: inputsstreams
 
-- map:
-    path: inputsstreams
-    rules:
-    - copy_all_to_list:
-        to: streams
-        on_conflict: noop
-        except: ["streams", "id", "enabled", "processors"]
-    - copy_to_list:
-        item: processors
-        to: streams
-        on_conflict: insert_before
+  - map:
+      path: inputsstreams
+      rules:
+        - copy_all_to_list:
+            to: streams
+            on_conflict: noop
+            except: ["streams", "id", "enabled", "processors"]
+        - copy_to_list:
+            item: processors
+            to: streams
+            on_conflict: insert_before
 
-- extract_list_items:
-    path: inputsstreams
-    item: streams
-    to: inputs
+  - extract_list_items:
+      path: inputsstreams
+      item: streams
+      to: inputs
 
-- filter_values_with_regexp:
-    key: type
-    re: ^.+/metrics$
-    selector: inputs
+  - filter_values_with_regexp:
+      key: type
+      re: ^.+/metrics$
+      selector: inputs
 
-- filter_values:
-    selector: inputs
-    key: enabled
-    values:
-    - true
+  - filter_values:
+      selector: inputs
+      key: enabled
+      values:
+        - true
 
-- map:
-    path: inputs
-    rules:
-    - translate_with_regexp:
-        path: type
-        re: ^(?P<type>.+)/metrics$
-        with: $type
-    - rename:
-        from: type
-        to: module
-    - make_array:
-        item: metricset
-        to: metricsets
-    - remove_key:
-        key: metricset
-    - remove_key:
-        key: enabled
-    - remove_key:
-        key: data_stream
-    - remove_key:
-        key: data_stream.dataset
-    - remove_key:
-        key: data_stream.namespace
-    - remove_key:
-        key: use_output
+  - map:
+      path: inputs
+      rules:
+        - translate_with_regexp:
+            path: type
+            re: ^(?P<type>.+)/metrics$
+            with: $type
+        - rename:
+            from: type
+            to: module
+        - make_array:
+            item: metricset
+            to: metricsets
+        - remove_key:
+            key: metricset
+        - remove_key:
+            key: enabled
+        - remove_key:
+            key: data_stream
+        - remove_key:
+            key: data_stream.dataset
+        - remove_key:
+            key: data_stream.namespace
+        - remove_key:
+            key: use_output
 
-- inject_agent_info: {}
+  - inject_agent_info: {}
 
-- copy:
-    from: inputs
-    to: metricbeat
+  - copy:
+      from: inputs
+      to: metricbeat
 
-- rename:
-    from: metricbeat.inputs
-    to: modules
+  - rename:
+      from: metricbeat.inputs
+      to: modules
 
-- filter:
-    selectors:
-    - metricbeat
-    - output
-    - keystore
-- inject_headers: {}
+  - filter:
+      selectors:
+        - metricbeat
+        - output
+        - keystore
+  - inject_headers: {}
 
 when: length(${metricbeat.modules}) > 0 and hasKey(${output}, 'elasticsearch',
   'redis', 'kafka', 'logstash')