feat: Remove app-pods-secure scrape job

docs/user/04-metrics.md

@@ -313,14 +313,16 @@ spec:
 
 The Metric agent is configured with a generic scrape configuration, which uses annotations to specify the endpoints to scrape in the cluster.
 
-For metrics ingestion to start automatically, simply apply the following annotations either to a Service that resolves your metrics port, or directly to the Pod:
-
-| Annotation Key                     | Example Values    | Default Value | Description                                                                                                                                                                                                                                                                                                                                 |
-|------------------------------------|-------------------|-------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
-| `prometheus.io/scrape` (mandatory) | `true`, `false` | none | Controls whether Prometheus automatically scrapes metrics from this target.                                                                                                                                                                                                                                                             |
-| `prometheus.io/port` (mandatory)   | `8080`, `9100` | none | Specifies the port where the metrics are exposed.                                                                                                                                                                                                                                                                                           |
-| `prometheus.io/path`               | `/metrics`, `/custom_metrics` | `/metrics` | Defines the HTTP path where Prometheus can find metrics data.                                                                                                                                                                                                                                                                               |
-| `prometheus.io/scheme`             | `http`, `https` | If Istio is active, `https` is supported; otherwise, only `http` is available. The default scheme is `http` unless an Istio sidecar is present, denoted by the label `security.istio.io/tlsMode=istio`, in which case `https` becomes the default. | Determines the protocol used for scraping metrics — either HTTPS with mTLS or plain HTTP. |
+For metrics ingestion to start automatically, use the annotations of the following table.
+If an Istio sidecar is present, apply them to a Service that resolves your metrics port.
+Only if Istio sidecar is not present, you can alternatively apply the annotations directly to the Pod.
+
+| Annotation Key                                                   | Example Values    | Default Value | Description                                                                                                                                                                                                                                                                                                                                 |
+|------------------------------------------------------------------|-------------------|-------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| `prometheus.io/scrape` (mandatory)                               | `true`, `false` | none | Controls whether Prometheus automatically scrapes metrics from this target.                                                                                                                                                                                                                                                             |
+| `prometheus.io/port` (mandatory)                                 | `8080`, `9100` | none | Specifies the port where the metrics are exposed.                                                                                                                                                                                                                                                                                           |
+| `prometheus.io/path`                                             | `/metrics`, `/custom_metrics` | `/metrics` | Defines the HTTP path where Prometheus can find metrics data.                                                                                                                                                                                                                                                                               |
+| `prometheus.io/scheme` (only relevant when annotating a Service) | `http`, `https` | If Istio is active, `https` is supported; otherwise, only `http` is available. The default scheme is `http` unless an Istio sidecar is present, denoted by the label `security.istio.io/tlsMode=istio`, in which case `https` becomes the default. | Determines the protocol used for scraping metrics — either HTTPS with mTLS or plain HTTP. |
 
 If you're running the Pod targeted by a Service with Istio, Istio must be able to derive the [appProtocol](https://kubernetes.io/docs/concepts/services-networking/service/#application-protocol) from the Service port definition; otherwise the communication for scraping the metric endpoint cannot be established. You must either prefix the port name with the protocol like in `http-metrics`, or explicitly define the `appProtocol` attribute.
 

internal/otelcollector/config/metric/agent/prometheus_receiver.go

@@ -32,13 +32,13 @@ func makePrometheusConfigForPods(opts BuildOptions) *PrometheusReceiver {
 	return makePrometheusConfig(opts, "app-pods", RolePod, makePrometheusPodsRelabelConfigs)
 }
 
-// makePrometheusConfigForPods creates a Prometheus configuration for scraping Services that are annotated with prometheus.io annotations.
+// makePrometheusConfigForServices creates a Prometheus configuration for scraping Services that are annotated with prometheus.io annotations.
 func makePrometheusConfigForServices(opts BuildOptions) *PrometheusReceiver {
 	return makePrometheusConfig(opts, "app-services", RoleEndpoints, makePrometheusEndpointsRelabelConfigs)
 }
 
 // makePrometheusConfig generates a Prometheus receiver configuration for scraping either annotated Pods or Services (based on the provided role and relabelConfigFn).
-// If Istio is enabled, an additional scrape config is generated (prefixed with -secure) to scrape targets over HTTPS using Istio certificate.
+// If Istio is enabled, an additional scrape config is generated (suffixed with -secure) to scrape annotated Services over HTTPS using Istio certificate.
 // Istio certificate is expected to be mounted at the provided path using the proxy.istio.io/config annotation.
 // See more: https://istio.io/latest/docs/ops/integrations/prometheus/#tls-settings
 func makePrometheusConfig(opts BuildOptions, jobNamePrefix string, role Role, relabelConfigFn func(keepSecure bool) []RelabelConfig) *PrometheusReceiver {
@@ -55,7 +55,8 @@ func makePrometheusConfig(opts BuildOptions, jobNamePrefix string, role Role, re
 	httpScrapeConfig.RelabelConfigs = relabelConfigFn(false)
 	config.Config.ScrapeConfigs = append(config.Config.ScrapeConfigs, httpScrapeConfig)
 
-	if opts.IstioEnabled {
+	// If Istio is enabled, generate an additional scrape config for scraping annotated Services (role == RoleEndpoints) over HTTPS
+	if opts.IstioEnabled && role == RoleEndpoints {
 		httpsScrapeConfig := baseScrapeConfig
 		httpsScrapeConfig.JobName = jobNamePrefix + "-secure"
 		httpsScrapeConfig.RelabelConfigs = relabelConfigFn(true)
@@ -70,28 +71,19 @@ func makePrometheusConfig(opts BuildOptions, jobNamePrefix string, role Role, re
 // They restrict Pods that are selected for scraping and set internal labels (__address__, __scheme__, etc.).
 // See more: https://prometheus.io/docs/prometheus/latest/configuration/configuration/#pod.
 //
-// If requireHTTPS is true, only Pods with Istio sidecars or those explicitly marked with prometheus.io/scheme=http annotations are selected.
-// If requireHTTPS is false, only Pods without Istio sidecars or those marked with prometheus.io/scheme=https annotation are selected.
-func makePrometheusPodsRelabelConfigs(requireHTTPS bool) []RelabelConfig {
-	relabelConfigs := []RelabelConfig{
+// Only Pods without Istio sidecars are selected.
+func makePrometheusPodsRelabelConfigs(_ bool) []RelabelConfig {
+	return []RelabelConfig{
 		keepIfRunningOnSameNode(NodeAffiliatedPod),
 		keepIfScrapingEnabled(AnnotatedPod),
 		dropIfPodNotRunning(),
 		dropIfInitContainer(),
 		dropIfIstioProxy(),
 		inferSchemeFromIstioInjectedLabel(),
-		inferSchemeFromAnnotation(AnnotatedPod),
-	}
-
-	if requireHTTPS {
-		relabelConfigs = append(relabelConfigs, dropIfSchemeHTTP())
-	} else {
-		relabelConfigs = append(relabelConfigs, dropIfSchemeHTTPS())
-	}
-
-	return append(relabelConfigs,
+		dropIfSchemeHTTPS(),
 		inferMetricsPathFromAnnotation(AnnotatedPod),
-		inferAddressFromAnnotation(AnnotatedPod))
+		inferAddressFromAnnotation(AnnotatedPod),
+	}
 }
 
 // makePrometheusEndpointsRelabelConfigs generates a set of relabel configs for the Endpoint role type.

internal/otelcollector/config/metric/agent/receivers_test.go

@@ -211,7 +211,6 @@ func TestReceivers(t *testing.T) {
 				istioEnabled: true,
 				expectedPodScrapeJobs: []string{
 					"app-pods",
-					"app-pods-secure",
 				},
 				expectedServiceScrapeJobs: []string{
 					"app-services",

internal/otelcollector/config/metric/agent/testdata/config_istio_enabled.yaml

@@ -191,10 +191,6 @@ receivers:
                       target_label: __scheme__
                       replacement: https
                       action: replace
-                    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
-                      regex: (https?)
-                      target_label: __scheme__
-                      action: replace
                     - source_labels: [__scheme__]
                       regex: (https)
                       action: drop
@@ -209,53 +205,6 @@ receivers:
                       action: replace
                   kubernetes_sd_configs:
                     - role: pod
-                - job_name: app-pods-secure
-                  sample_limit: 50000
-                  scrape_interval: 30s
-                  relabel_configs:
-                    - source_labels: [__meta_kubernetes_pod_node_name]
-                      regex: ${MY_NODE_NAME}
-                      action: keep
-                    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape]
-                      regex: "true"
-                      action: keep
-                    - source_labels: [__meta_kubernetes_pod_phase]
-                      regex: Pending|Succeeded|Failed
-                      action: drop
-                    - source_labels: [__meta_kubernetes_pod_container_init]
-                      regex: (true)
-                      action: drop
-                    - source_labels: [__meta_kubernetes_pod_container_name]
-                      regex: (istio-proxy)
-                      action: drop
-                    - source_labels: [__meta_kubernetes_pod_label_security_istio_io_tlsMode]
-                      regex: (istio)
-                      target_label: __scheme__
-                      replacement: https
-                      action: replace
-                    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
-                      regex: (https?)
-                      target_label: __scheme__
-                      action: replace
-                    - source_labels: [__scheme__]
-                      regex: (http)
-                      action: drop
-                    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path]
-                      regex: (.+)
-                      target_label: __metrics_path__
-                      action: replace
-                    - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port]
-                      regex: ([^:]+)(?::\d+)?;(\d+)
-                      target_label: __address__
-                      replacement: $$1:$$2
-                      action: replace
-                  kubernetes_sd_configs:
-                    - role: pod
-                  tls_config:
-                    ca_file: /etc/istio-output-certs/root-cert.pem
-                    cert_file: /etc/istio-output-certs/cert-chain.pem
-                    key_file: /etc/istio-output-certs/key.pem
-                    insecure_skip_verify: true
     prometheus/app-services:
         config:
             scrape_configs:

internal/otelcollector/config/metric/agent/testdata/config_istio_not_enabled.yaml

@@ -180,10 +180,6 @@ receivers:
                       target_label: __scheme__
                       replacement: https
                       action: replace
-                    - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scheme]
-                      regex: (https?)
-                      target_label: __scheme__
-                      action: replace
                     - source_labels: [__scheme__]
                       regex: (https)
                       action: drop

test/integration/istio/metrics_prometheus_input_test.go

@@ -49,10 +49,12 @@ var _ = Describe(suite.ID(), Label(suite.LabelIntegration), Ordered, func() {
 		objs = append(objs, []client.Object{
 			httpsAnnotatedMetricProducer.Pod().WithSidecarInjection().WithPrometheusAnnotations(prommetricgen.SchemeHTTPS).K8sObject(),
 			httpsAnnotatedMetricProducer.Service().WithPrometheusAnnotations(prommetricgen.SchemeHTTPS).K8sObject(),
+
 			httpAnnotatedMetricProducer.Pod().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(),
 			httpAnnotatedMetricProducer.Service().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(),
-			unannotatedMetricProducer.Pod().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(),
-			unannotatedMetricProducer.Service().WithPrometheusAnnotations(prommetricgen.SchemeHTTP).K8sObject(),
+
+			unannotatedMetricProducer.Pod().WithPrometheusAnnotations(prommetricgen.SchemeNone).K8sObject(),
+			unannotatedMetricProducer.Service().WithPrometheusAnnotations(prommetricgen.SchemeNone).K8sObject(),
 		}...)
 
 		metricPipeline := testutils.NewMetricPipelineBuilder().
@@ -92,8 +94,8 @@ var _ = Describe(suite.ID(), Label(suite.LabelIntegration), Ordered, func() {
 			// here we are discovering the same metric-producer workload twice: once via the annotated service and once via the annotated pod
 			// targets discovered via annotated pods must have no service label
 			Context("Annotated pods", func() {
-				It("Should scrape if prometheus.io/scheme=https", func() {
-					podScrapedMetricsShouldBeDelivered(backendExportURL, httpsAnnotatedMetricProducerName)
+				It("Should NOT scrape if istio sidecar exists", func() {
+					podMetricsShouldNotBeDelivered(backendExportURL, httpsAnnotatedMetricProducerName)
 				})
 
 				It("Should scrape if prometheus.io/scheme=http", func() {
@@ -124,8 +126,27 @@ var _ = Describe(suite.ID(), Label(suite.LabelIntegration), Ordered, func() {
 	})
 })
 
-func podScrapedMetricsShouldBeDelivered(proxyURL, podName string) {
+func podMetricsShouldNotBeDelivered(proxyURL, podName string) {
+	Consistently(func(g Gomega) {
+		resp, err := proxyClient.Get(proxyURL)
+		g.Expect(err).NotTo(HaveOccurred())
+		g.Expect(resp).To(HaveHTTPStatus(http.StatusOK))
 
+		bodyContent, err := io.ReadAll(resp.Body)
+		resp.Body.Close()
+		g.Expect(err).NotTo(HaveOccurred())
+
+		g.Expect(bodyContent).To(HaveFlatMetrics(
+			Not(ContainElement(SatisfyAll(
+				HaveName(BeElementOf(prommetricgen.MetricNames)),
+				Not(HaveMetricAttributes(HaveKey("service"))),
+				HaveResourceAttributes(HaveKeyWithValue("k8s.pod.name", podName)),
+			))),
+		))
+	}, 3*periodic.TelemetryConsistentlyTimeout, periodic.TelemetryInterval).Should(Succeed())
+}
+
+func podScrapedMetricsShouldBeDelivered(proxyURL, podName string) {
 	Eventually(func(g Gomega) {
 		resp, err := proxyClient.Get(proxyURL)
 		g.Expect(err).NotTo(HaveOccurred())
@@ -136,11 +157,16 @@ func podScrapedMetricsShouldBeDelivered(proxyURL, podName string) {
 		g.Expect(err).NotTo(HaveOccurred())
 
 		g.Expect(bodyContent).To(HaveFlatMetrics(
-			SatisfyAll(
-				ContainElement(HaveResourceAttributes(HaveKeyWithValue("k8s.pod.name", podName))),
-				ContainElement(HaveName(BeElementOf(prommetricgen.MetricNames))),
-				ContainElement(HaveScopeName(ContainSubstring(InstrumentationScopePrometheus))),
-			),
+			ContainElement(SatisfyAll(
+				HaveName(BeElementOf(prommetricgen.MetricNames)),
+				Not(HaveMetricAttributes(HaveKey("service"))),
+				HaveResourceAttributes(HaveKeyWithValue("k8s.pod.name", podName)),
+				HaveScopeName(Equal(InstrumentationScopePrometheus)),
+				HaveScopeVersion(SatisfyAny(
+					Equal("main"),
+					MatchRegexp("[0-9]+.[0-9]+.[0-9]+"),
+				)),
+			)),
 		))
 	}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 }
@@ -156,10 +182,15 @@ func serviceScrapedMetricsShouldBeDelivered(proxyURL, serviceName string) {
 		g.Expect(err).NotTo(HaveOccurred())
 
 		g.Expect(bodyContent).To(HaveFlatMetrics(
-			SatisfyAll(
-				ContainElement(HaveName(BeElementOf(prommetricgen.MetricNames))),
-				ContainElement(HaveMetricAttributes(HaveKeyWithValue("service", serviceName))),
-			),
+			ContainElement(SatisfyAll(
+				HaveName(BeElementOf(prommetricgen.MetricNames)),
+				HaveMetricAttributes(HaveKeyWithValue("service", serviceName)),
+				HaveScopeName(Equal(InstrumentationScopePrometheus)),
+				HaveScopeVersion(SatisfyAny(
+					Equal("main"),
+					MatchRegexp("[0-9]+.[0-9]+.[0-9]+"),
+				)),
+			)),
 		))
 	}, periodic.TelemetryEventuallyTimeout, periodic.TelemetryInterval).Should(Succeed())
 }

test/testkit/mocks/prommetricgen/metric_producer.go

@@ -28,6 +28,7 @@ type ScrapingScheme string
 const (
 	SchemeHTTP  ScrapingScheme = "http"
 	SchemeHTTPS ScrapingScheme = "https"
+	SchemeNone  ScrapingScheme = "none"
 )
 
 var (
@@ -149,12 +150,16 @@ func (p *Pod) WithLabel(key, value string) *Pod {
 }
 
 func makePrometheusAnnotations(scheme ScrapingScheme) map[string]string {
-	return map[string]string{
+	annotations := map[string]string{
 		"prometheus.io/scrape": "true",
 		"prometheus.io/path":   metricsEndpoint,
 		"prometheus.io/port":   strconv.Itoa(int(metricsPort)),
-		"prometheus.io/scheme": string(scheme),
 	}
+	if scheme != SchemeNone {
+		annotations["prometheus.io/scheme"] = string(scheme)
+	}
+
+	return annotations
 }
 
 func (p *Pod) K8sObject() *corev1.Pod {