From 01de93b3cd6a7993ae692ecec0eac4010bfff7a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20M=C3=BCller?= Date: Thu, 24 Feb 2022 18:34:47 +0100 Subject: [PATCH] Mentix PromSD extensions (#2560) --- changelog/unreleased/mentix-prom-ext.md | 5 + .../config/http/services/mentix/_index.md | 6 +- .../http/services/mentix/localfile/_index.md | 19 --- .../http/services/mentix/promsd/_index.md | 14 +- .../http/services/mentix/sitereg/_index.md | 37 ----- examples/mentix/mentix.toml | 18 +- internal/http/services/mentix/mentix.go | 5 - pkg/mentix/config/config.go | 18 +- pkg/mentix/connectors/localfile.go | 144 ---------------- pkg/mentix/exchangers/exporters/promsd.go | 19 ++- pkg/mentix/exchangers/importers/importers.go | 3 + pkg/mentix/exchangers/importers/sitereg.go | 62 ------- .../exchangers/importers/sitereg/query.go | 157 ------------------ .../exchangers/importers/sitereg/types.go | 142 ---------------- pkg/mentix/mentix.go | 2 +- pkg/mentix/meshdata/endpoints.go | 18 ++ 16 files changed, 50 insertions(+), 619 deletions(-) create mode 100644 changelog/unreleased/mentix-prom-ext.md delete mode 100644 docs/content/en/docs/config/http/services/mentix/localfile/_index.md delete mode 100644 docs/content/en/docs/config/http/services/mentix/sitereg/_index.md delete mode 100755 pkg/mentix/connectors/localfile.go delete mode 100644 pkg/mentix/exchangers/importers/sitereg.go delete mode 100755 pkg/mentix/exchangers/importers/sitereg/query.go delete mode 100644 pkg/mentix/exchangers/importers/sitereg/types.go diff --git a/changelog/unreleased/mentix-prom-ext.md b/changelog/unreleased/mentix-prom-ext.md new file mode 100644 index 0000000000..03df3192d4 --- /dev/null +++ b/changelog/unreleased/mentix-prom-ext.md @@ -0,0 +1,5 @@ +Enhancement: Mentix PromSD extensions + +The Mentix Prometheus SD scrape targets are now split into one file per service type, making health checks configuration easier. Furthermore, the local file connector for mesh data and the site registration endpoint have been dropped, as they aren't needed anymore. + +https://github.com/cs3org/reva/pull/2560 diff --git a/docs/content/en/docs/config/http/services/mentix/_index.md b/docs/content/en/docs/config/http/services/mentix/_index.md index dd73b4f51a..c71eb414da 100644 --- a/docs/content/en/docs/config/http/services/mentix/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/_index.md @@ -43,17 +43,13 @@ _Supported connectors:_ - **gocdb** The [GOCDB](https://wiki.egi.eu/wiki/GOCDB/Documentation_Index) is a database specifically designed to organize the topology of a mesh of distributed sites and services. In order to use GOCDB with Mentix, its instance address has to be configured (see [here](gocdb)). - -- **localfile** -The [localfile](localfile) connector reads sites from a local JSON file. The file must contain an array of sites adhering to the `meshdata.Site` structure. ## Importers Mentix can import mesh data from various sources and write it to one or more targets through the corresponding connectors. __Supported importers:__ -- **sitereg** -Mentix can import new sites via an HTTP endpoint using the `sitereg` importer. Data can be sent to the configured relative endpoint (see [here](sitereg)). +- **None** ## Exporters Mentix exposes its gathered data by using one or more _exporters_. Such exporters can, for example, write the data to a file in a specific format, or offer the data via an HTTP endpoint. diff --git a/docs/content/en/docs/config/http/services/mentix/localfile/_index.md b/docs/content/en/docs/config/http/services/mentix/localfile/_index.md deleted file mode 100644 index 0a85c0059a..0000000000 --- a/docs/content/en/docs/config/http/services/mentix/localfile/_index.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -title: "localfile" -linkTitle: "localfile" -weight: 10 -description: > - Configuration for the local file connector of the Mentix service ---- - -{{% pageinfo %}} -The local file connector reads sites from a local JSON file adhering to the `meshdata.Site` structure. -{{% /pageinfo %}} - -{{% dir name="file" type="string" default="" %}} -The file path. -{{< highlight toml >}} -[http.services.mentix.connectors.localfile] -file = "/etc/reva/sites.json" -{{< /highlight >}} -{{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/promsd/_index.md b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md index 20cb66952a..4ff4a64528 100644 --- a/docs/content/en/docs/config/http/services/mentix/promsd/_index.md +++ b/docs/content/en/docs/config/http/services/mentix/promsd/_index.md @@ -10,19 +10,11 @@ description: > When using the Prometheus SD exporter, the output filenames have to be configured first. {{% /pageinfo %}} -{{% dir name="metrics_output_file" type="string" default="" %}} -The target filename of the generated Prometheus File SD scrape config for metrics. +{{% dir name="output_path" type="string" default="" %}} +The target path of the generated Prometheus File SD scrape configs for metrics. {{< highlight toml >}} [http.services.mentix.exporters.promsd] -metrics_output_file = "/var/shared/prometheus/sciencemesh.json" -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="blackbox_output_file" type="string" default="" %}} -The target filename of the generated Prometheus File SD scrape config for the blackbox exporter. -{{< highlight toml >}} -[http.services.mentix.exporters.promsd] -blackbox_output_file = "/var/shared/prometheus/blackbox.json" +output_path = "/var/shared/prometheus/sciencemesh" {{< /highlight >}} {{% /dir %}} diff --git a/docs/content/en/docs/config/http/services/mentix/sitereg/_index.md b/docs/content/en/docs/config/http/services/mentix/sitereg/_index.md deleted file mode 100644 index 3591732b9e..0000000000 --- a/docs/content/en/docs/config/http/services/mentix/sitereg/_index.md +++ /dev/null @@ -1,37 +0,0 @@ ---- -title: "sitereg" -linkTitle: "sitereg" -weight: 10 -description: > - Configuration for site registration service ---- - -{{% pageinfo %}} -The site registration service is used to register new and unregister existing sites. -{{% /pageinfo %}} - -The site registration service is used to register new and unregister existing sites. - -{{% dir name="endpoint" type="string" default="/sitereg" %}} -The endpoint of the service. -{{< highlight toml >}} -[http.services.mentix.importers.sitereg] -endpoint = "/reg" -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="enabled_connectors" type="[]string" default="" %}} -A list of all enabled connectors for the importer. -{{< highlight toml >}} -[http.services.mentix.importers.sitereg] -enabled_connectors = ["localfile"] -{{< /highlight >}} -{{% /dir %}} - -{{% dir name="ignore_sm_sites" type="bool" default="false" %}} -If set to true, registrations from ScienceMesh sites will be ignored. -{{< highlight toml >}} -[http.services.mentix.importers.sitereg] -ignore_sm_sites = true -{{< /highlight >}} -{{% /dir %}} diff --git a/examples/mentix/mentix.toml b/examples/mentix/mentix.toml index 1da40b784f..340ff4c508 100644 --- a/examples/mentix/mentix.toml +++ b/examples/mentix/mentix.toml @@ -8,10 +8,6 @@ update_interval = "15m" address = "http://sciencemesh-test.uni-muenster.de" apikey = "abc123" -# Sites can also be stored in a local file -[http.services.mentix.connectors.localfile] -file = "/usr/share/revad/sites.json" - # Configure the service types that are considered as critical/essential [http.services.mentix.services] critical_types = ["REVAD"] @@ -28,13 +24,6 @@ enabled_connectors = ["gocdb"] # Enable the Metrics exporter [http.services.mentix.exporters.metrics] -# Enable the site registration importer -[http.services.mentix.importers.sitereg] -# For importers, this is obligatory; the connectors will be used as the target for data updates -enabled_connectors = ["localfile"] -# If set to true, ScienceMesh sites will be ignored when they try to register -ignore_sm_sites = false - # Set up the accounts service used to query information about accounts associated with registered sites [http.services.mentix.accounts] # Depending on where the service is running, localhost may also be used here @@ -44,7 +33,6 @@ password = "userpass" # Configure the Prometheus Service Discovery: [http.services.mentix.exporters.promsd] -# The following files must be made available to Prometheus. -# They can then be used as the file_sd source of a job. -metrics_output_file = "/usr/share/prom/sciencemesh_metrics.json" -blackbox_output_file = "/usr/share/prom/sciencemesh_blackbox.json" +# The following path must be made available to Prometheus. +# The contained target files can then be used as the file_sd sources of a job. +output_path = "/usr/share/prom" diff --git a/internal/http/services/mentix/mentix.go b/internal/http/services/mentix/mentix.go index 4a9332164e..af9adb90ed 100644 --- a/internal/http/services/mentix/mentix.go +++ b/internal/http/services/mentix/mentix.go @@ -137,11 +137,6 @@ func applyDefaultConfig(conf *config.Configuration) { conf.Connectors.GOCDB.Scope = "SM" // TODO(Daniel-WWU-IT): This might change in the future } - // Importers - if conf.Importers.SiteRegistration.Endpoint == "" { - conf.Importers.SiteRegistration.Endpoint = "/sitereg" - } - // Exporters addDefaultConnector := func(enabledList *[]string) { if len(*enabledList) == 0 { diff --git a/pkg/mentix/config/config.go b/pkg/mentix/config/config.go index 4bf5eeb40d..6ea8abb156 100644 --- a/pkg/mentix/config/config.go +++ b/pkg/mentix/config/config.go @@ -28,10 +28,6 @@ type Configuration struct { Scope string `mapstructure:"scope"` APIKey string `mapstructure:"apikey"` } `mapstructure:"gocdb"` - - LocalFile struct { - File string `mapstructure:"file"` - } `mapstructure:"localfile"` } `mapstructure:"connectors"` UpdateInterval string `mapstructure:"update_interval"` @@ -40,15 +36,6 @@ type Configuration struct { CriticalTypes []string `mapstructure:"critical_types"` } `mapstructure:"services"` - Importers struct { - SiteRegistration struct { - Endpoint string `mapstructure:"endpoint"` - EnabledConnectors []string `mapstructure:"enabled_connectors"` - IsProtected bool `mapstructure:"is_protected"` - IgnoreScienceMeshSites bool `mapstructure:"ignore_sm_sites"` - } `mapstructure:"sitereg"` - } `mapstructure:"importers"` - Exporters struct { WebAPI struct { Endpoint string `mapstructure:"endpoint"` @@ -70,9 +57,8 @@ type Configuration struct { } `mapstructure:"siteloc"` PrometheusSD struct { - MetricsOutputFile string `mapstructure:"metrics_output_file"` - BlackboxOutputFile string `mapstructure:"blackbox_output_file"` - EnabledConnectors []string `mapstructure:"enabled_connectors"` + OutputPath string `mapstructure:"output_path"` + EnabledConnectors []string `mapstructure:"enabled_connectors"` } `mapstructure:"promsd"` Metrics struct { diff --git a/pkg/mentix/connectors/localfile.go b/pkg/mentix/connectors/localfile.go deleted file mode 100755 index 8ed918ec31..0000000000 --- a/pkg/mentix/connectors/localfile.go +++ /dev/null @@ -1,144 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package connectors - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - - "github.com/pkg/errors" - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/meshdata" -) - -// LocalFileConnector is used to read sites from a local file. -type LocalFileConnector struct { - BaseConnector - - filePath string -} - -// Activate activates the connector. -func (connector *LocalFileConnector) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := connector.BaseConnector.Activate(conf, log); err != nil { - return err - } - - // Check and store GOCDB specific settings - connector.filePath = conf.Connectors.LocalFile.File - if len(connector.filePath) == 0 { - return fmt.Errorf("no file configured") - } - - // Create the file directory if necessary - dir := filepath.Dir(connector.filePath) - _ = os.MkdirAll(dir, 0755) - - // Create an empty file if it doesn't exist - if _, err := os.Stat(connector.filePath); os.IsNotExist(err) { - _ = ioutil.WriteFile(connector.filePath, []byte("[]"), 0755) - } - - return nil -} - -// RetrieveMeshData fetches new mesh data. -func (connector *LocalFileConnector) RetrieveMeshData() (*meshdata.MeshData, error) { - jsonData, err := ioutil.ReadFile(connector.filePath) - if err != nil { - return nil, errors.Wrapf(err, "unable to read file '%v'", connector.filePath) - } - - meshData := &meshdata.MeshData{} - if err := json.Unmarshal(jsonData, &meshData.Sites); err != nil { - return nil, errors.Wrapf(err, "invalid file '%v'", connector.filePath) - } - - connector.setSiteTypes(meshData) - meshData.InferMissingData() - - return meshData, nil -} - -// UpdateMeshData updates the provided mesh data on the target side. -func (connector *LocalFileConnector) UpdateMeshData(updatedData *meshdata.MeshData) error { - meshData, err := connector.RetrieveMeshData() - if err != nil { - // Ignore errors and start with an empty data set - meshData = &meshdata.MeshData{} - } - - err = nil - switch updatedData.Status { - case meshdata.StatusDefault: - err = connector.mergeData(meshData, updatedData) - - case meshdata.StatusObsolete: - err = connector.unmergeData(meshData, updatedData) - } - - if err != nil { - return err - } - - // Write the updated sites back to the file - jsonData, _ := json.MarshalIndent(meshData.Sites, "", "\t") - if err := ioutil.WriteFile(connector.filePath, jsonData, 0755); err != nil { - return fmt.Errorf("unable to write file '%v': %v", connector.filePath, err) - } - - return nil -} - -func (connector *LocalFileConnector) mergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error { - // Add/update data by merging - meshData.Merge(updatedData) - return nil -} - -func (connector *LocalFileConnector) unmergeData(meshData *meshdata.MeshData, updatedData *meshdata.MeshData) error { - // Remove data by unmerging - meshData.Unmerge(updatedData) - return nil -} - -func (connector *LocalFileConnector) setSiteTypes(meshData *meshdata.MeshData) { - for _, site := range meshData.Sites { - site.Type = meshdata.SiteTypeCommunity // Sites coming from a local file are always community sites - } -} - -// GetID returns the ID of the connector. -func (connector *LocalFileConnector) GetID() string { - return config.ConnectorIDLocalFile -} - -// GetName returns the display name of the connector. -func (connector *LocalFileConnector) GetName() string { - return "Local file" -} - -func init() { - registerConnector(&LocalFileConnector{}) -} diff --git a/pkg/mentix/exchangers/exporters/promsd.go b/pkg/mentix/exchangers/exporters/promsd.go index 1c4a596ef9..4d51e9b641 100755 --- a/pkg/mentix/exchangers/exporters/promsd.go +++ b/pkg/mentix/exchangers/exporters/promsd.go @@ -24,7 +24,9 @@ import ( "io/ioutil" "net/url" "os" + "path" "path/filepath" + "strings" "github.com/cs3org/reva/pkg/mentix/utils" "github.com/rs/zerolog" @@ -54,6 +56,7 @@ const ( labelSiteID = "__meta_mentix_site_id" labelSiteCountry = "__meta_mentix_site_country" labelType = "__meta_mentix_type" + labelURL = "__meta_mentix_url" labelScheme = "__meta_mentix_scheme" labelHost = "__meta_mentix_host" labelPort = "__meta_mentix_port" @@ -78,6 +81,7 @@ func getScrapeTargetLabels(site *meshdata.Site, service *meshdata.Service, endpo labelSiteID: site.ID, labelSiteCountry: site.CountryCode, labelType: endpoint.Type.Name, + labelURL: endpoint.URL, labelScheme: endpointURL.Scheme, labelHost: endpointURL.Hostname(), labelPort: endpointURL.Port(), @@ -110,12 +114,13 @@ func (exporter *PrometheusSDExporter) registerScrapeCreators(conf *config.Config } // Register all scrape creators - if err := registerCreator("metrics", conf.Exporters.PrometheusSD.MetricsOutputFile, createGenericScrapeConfig, []string{meshdata.EndpointMetrics}); err != nil { - return fmt.Errorf("unable to register the 'metrics' scrape config creator: %v", err) - } + for _, endpoint := range meshdata.GetServiceEndpoints() { + epName := strings.ToLower(endpoint) + filename := path.Join(conf.Exporters.PrometheusSD.OutputPath, "svc_"+epName+".json") - if err := registerCreator("blackbox", conf.Exporters.PrometheusSD.BlackboxOutputFile, createGenericScrapeConfig, []string{meshdata.EndpointGateway}); err != nil { - return fmt.Errorf("unable to register the 'blackbox' scrape config creator: %v", err) + if err := registerCreator(epName, filename, createGenericScrapeConfig, []string{endpoint}); err != nil { + return fmt.Errorf("unable to register the '%v' scrape config creator: %v", epName, err) + } } return nil @@ -199,6 +204,10 @@ func (exporter *PrometheusSDExporter) createScrapeConfigs(creatorCallback promet } } + if scrapes == nil { + scrapes = []*prometheus.ScrapeConfig{} + } + return scrapes } diff --git a/pkg/mentix/exchangers/importers/importers.go b/pkg/mentix/exchangers/importers/importers.go index 152a2a1b55..da957f1000 100644 --- a/pkg/mentix/exchangers/importers/importers.go +++ b/pkg/mentix/exchangers/importers/importers.go @@ -85,6 +85,9 @@ func AvailableImporters(conf *config.Configuration) (*Collection, error) { return &Collection{Importers: importers}, nil } +// TODO: Uncomment once an importer is actually implemented +/* func registerImporter(importer Importer) { registeredImporters.Register(importer) } +*/ diff --git a/pkg/mentix/exchangers/importers/sitereg.go b/pkg/mentix/exchangers/importers/sitereg.go deleted file mode 100644 index 34c642527c..0000000000 --- a/pkg/mentix/exchangers/importers/sitereg.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package importers - -import ( - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/exchangers/importers/sitereg" -) - -// SiteRegistrationImporter implements the external site registration importer. -type SiteRegistrationImporter struct { - BaseRequestImporter -} - -// Activate activates the importer. -func (importer *SiteRegistrationImporter) Activate(conf *config.Configuration, log *zerolog.Logger) error { - if err := importer.BaseRequestImporter.Activate(conf, log); err != nil { - return err - } - - // Store SiteRegistration specifics - importer.SetEndpoint(conf.Importers.SiteRegistration.Endpoint, conf.Importers.SiteRegistration.IsProtected) - importer.SetEnabledConnectors(conf.Importers.SiteRegistration.EnabledConnectors) - importer.SetAllowUnauthorizedSites(true) - - importer.RegisterExtendedActionHandler("register", sitereg.HandleRegisterSiteQuery) - importer.RegisterExtendedActionHandler("unregister", sitereg.HandleUnregisterSiteQuery) - - return nil -} - -// GetID returns the ID of the importer. -func (importer *SiteRegistrationImporter) GetID() string { - return config.ImporterIDSiteRegistration -} - -// GetName returns the display name of the importer. -func (importer *SiteRegistrationImporter) GetName() string { - return "SiteRegistration" -} - -func init() { - registerImporter(&SiteRegistrationImporter{}) -} diff --git a/pkg/mentix/exchangers/importers/sitereg/query.go b/pkg/mentix/exchangers/importers/sitereg/query.go deleted file mode 100755 index bf43a53ceb..0000000000 --- a/pkg/mentix/exchangers/importers/sitereg/query.go +++ /dev/null @@ -1,157 +0,0 @@ -// Copyright 2018-2021 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package sitereg - -import ( - "encoding/json" - "net/http" - "net/url" - "strings" - - "github.com/pkg/errors" - "github.com/rs/zerolog" - - "github.com/cs3org/reva/pkg/mentix/accservice" - "github.com/cs3org/reva/pkg/mentix/config" - "github.com/cs3org/reva/pkg/mentix/key" - "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/utils/network" -) - -func decodeQueryData(data []byte) (*siteRegistrationData, error) { - siteData := &siteRegistrationData{} - if err := json.Unmarshal(data, siteData); err != nil { - return nil, err - } - - if err := siteData.Verify(); err != nil { - return nil, errors.Wrap(err, "verifying the imported site data failed") - } - - return siteData, nil -} - -func extractQueryInformation(params url.Values) (key.SiteIdentifier, int, string, error) { - apiKey := params.Get("apiKey") - if len(apiKey) == 0 { - return "", 0, "", errors.Errorf("no API key specified") - } - - // Try to get an account that is associated with the given API key; if none exists, return an error - resp, err := accservice.Query("find", network.URLParams{"by": "apikey", "value": apiKey}) - if err != nil { - return "", 0, "", errors.Wrap(err, "error while querying the accounts service") - } - if !resp.Success { - return "", 0, "", errors.Errorf("unable to fetch account associated with the provided API key: %v", resp.Error) - } - - // Extract email from account data; this is needed to calculate the site ID from the API key - email := "" - if value := accservice.GetResponseValue(resp, "account.email"); value != nil { - email, _ = value.(string) - } - if len(email) == 0 { - return "", 0, "", errors.Errorf("could not get the email address of the user account") - } - - _, flags, _, err := key.SplitAPIKey(apiKey) - if err != nil { - return "", 0, "", errors.Errorf("sticky API key specified") - } - - siteID, err := key.CalculateSiteID(apiKey, strings.ToLower(email)) - if err != nil { - return "", 0, "", errors.Wrap(err, "unable to get site ID") - } - - return siteID, flags, email, nil -} - -func createErrorResponse(msg string, err error) (meshdata.Vector, int, []byte, error) { - return nil, http.StatusBadRequest, network.CreateResponse(msg, network.ResponseParams{"error": err.Error()}), nil -} - -// HandleRegisterSiteQuery registers a site. -func HandleRegisterSiteQuery(meshData *meshdata.MeshData, data []byte, params url.Values, conf *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - siteID, flags, email, err := extractQueryInformation(params) - if err != nil { - return createErrorResponse("INVALID_API_KEY", err) - } - - msg := "SITE_REGISTERED" - if meshData.FindSite(siteID) != nil { - msg = "SITE_UPDATED" - } - - // Decode the site registration data and convert it to a meshdata object - siteData, err := decodeQueryData(data) - if err != nil { - return createErrorResponse("INVALID_SITE_DATA", err) - } - - siteType := meshdata.SiteTypeCommunity - if flags&key.FlagScienceMesh == key.FlagScienceMesh { - siteType = meshdata.SiteTypeScienceMesh - } - - // If the corresponding setting is set, ignore registrations of ScienceMesh sites - if siteType == meshdata.SiteTypeScienceMesh && conf.Importers.SiteRegistration.IgnoreScienceMeshSites { - return meshdata.Vector{}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": siteID}), nil - } - - site, err := siteData.ToMeshDataSite(siteID, siteType, email) - if err != nil { - return createErrorResponse("INVALID_SITE_DATA", err) - } - - meshDataUpdate := &meshdata.MeshData{Sites: []*meshdata.Site{site}} - if err := meshDataUpdate.Verify(); err != nil { - return createErrorResponse("INVALID_MESH_DATA", err) - } - meshDataUpdate.Status = meshdata.StatusDefault - meshDataUpdate.InferMissingData() - - return meshdata.Vector{meshDataUpdate}, http.StatusOK, network.CreateResponse(msg, network.ResponseParams{"id": siteID}), nil -} - -// HandleUnregisterSiteQuery unregisters a site. -func HandleUnregisterSiteQuery(meshData *meshdata.MeshData, _ []byte, params url.Values, _ *config.Configuration, _ *zerolog.Logger) (meshdata.Vector, int, []byte, error) { - siteID, _, _, err := extractQueryInformation(params) - if err != nil { - return createErrorResponse("INVALID_API_KEY", err) - } - - // The site ID must be provided in the call as well to enhance security further - if params.Get("siteId") != siteID { - return createErrorResponse("INVALID_SITE_ID", errors.Errorf("site ID mismatch")) - } - - // Check if the site to be removed actually exists - if meshData.FindSite(siteID) == nil { - return createErrorResponse("INVALID_SITE_ID", errors.Errorf("site not found")) - } - - // To remove a site, a meshdata object that contains a site with the given ID needs to be created - site := &meshdata.Site{ID: siteID} - meshDataUpdate := &meshdata.MeshData{Sites: []*meshdata.Site{site}} - meshDataUpdate.Status = meshdata.StatusObsolete - - return meshdata.Vector{meshDataUpdate}, http.StatusOK, network.CreateResponse("SITE_UNREGISTERED", network.ResponseParams{"id": siteID}), nil -} diff --git a/pkg/mentix/exchangers/importers/sitereg/types.go b/pkg/mentix/exchangers/importers/sitereg/types.go deleted file mode 100644 index 6350289f2a..0000000000 --- a/pkg/mentix/exchangers/importers/sitereg/types.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright 2018-2020 CERN -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// In applying this license, CERN does not waive the privileges and immunities -// granted to it by virtue of its status as an Intergovernmental Organization -// or submit itself to any jurisdiction. - -package sitereg - -import ( - "net/url" - - "github.com/pkg/errors" - - "github.com/cs3org/reva/pkg/mentix/key" - "github.com/cs3org/reva/pkg/mentix/meshdata" - "github.com/cs3org/reva/pkg/mentix/utils/countries" - "github.com/cs3org/reva/pkg/mentix/utils/network" -) - -type siteRegistrationData struct { - Name string `json:"name"` - URL string `json:"url"` - CountryCode string `json:"countryCode"` - - Reva struct { - Host string `json:"host"` - URL string `json:"url"` - MetricsPath string `json:"metricsPath"` - } `json:"reva"` -} - -/* Example JSON: -{ - "name": "Testsite", - "url": "https://test-site.de/owncloud", - "countryCode": "DE", - "reva": { - "url": "https://test-site.de/owncloud/reva", - "metricsPath": "/iop/metrics" - } -} -*/ - -// Verify checks whether the entered data is valid and complete. -func (siteData *siteRegistrationData) Verify() error { - if len(siteData.Name) == 0 { - return errors.Errorf("no site name provided") - } - if len(siteData.URL) > 0 { - if _, err := url.Parse(siteData.URL); err != nil { - return errors.Wrap(err, "invalid site URL provided") - } - } else { - return errors.Errorf("no site URL provided") - } - - if len(siteData.Reva.Host) == 0 && len(siteData.Reva.URL) == 0 { - return errors.Errorf("no Reva host or URL provided") - } - if len(siteData.Reva.URL) > 0 { - if _, err := url.Parse(siteData.Reva.URL); err != nil { - return errors.Wrap(err, "invalid Reva URL provided") - } - } - if len(siteData.Reva.MetricsPath) == 0 { - return errors.Errorf("no Reva metrics path provided") - } - - return nil -} - -// ToMeshDataSite converts the stored data into a meshdata site object, filling out as much data as possible. -func (siteData *siteRegistrationData) ToMeshDataSite(siteID key.SiteIdentifier, siteType meshdata.SiteType, email string) (*meshdata.Site, error) { - siteURL, err := url.Parse(siteData.URL) - if err != nil { - return nil, errors.Wrap(err, "invalid site URL") - } - - // Create the Reva service entry - revaHost := siteData.Reva.Host - revaURL := siteData.Reva.URL - - if len(revaHost) == 0 { // Infer host from URL - URL, _ := url.Parse(revaURL) - revaHost = network.ExtractDomainFromURL(URL, true) - } else if len(revaURL) == 0 { // Infer URL from host - URL, _ := network.GenerateURL(revaHost, "", network.URLParams{}) - revaURL = URL.String() - } - - properties := make(map[string]string) - - revaService := &meshdata.Service{ - ServiceEndpoint: &meshdata.ServiceEndpoint{ - Type: &meshdata.ServiceType{ - Name: "REVAD", - Description: "Reva Daemon", - }, - Name: revaHost + " - REVAD", - URL: revaURL, - IsMonitored: true, - Properties: properties, - }, - Host: revaHost, - AdditionalEndpoints: nil, - } - - // Create the site data - site := &meshdata.Site{ - Type: siteType, - ID: siteID, - Name: siteData.Name, - FullName: siteData.Name, - Organization: "", - Domain: network.ExtractDomainFromURL(siteURL, true), - Homepage: siteData.URL, - Email: email, - Description: siteData.Name + " @ " + siteData.URL, - Country: countries.LookupCountry(siteData.CountryCode), - CountryCode: siteData.CountryCode, - Location: "", - Latitude: 0, - Longitude: 0, - Services: []*meshdata.Service{revaService}, - Properties: nil, - } - - site.InferMissingData() - return site, nil -} diff --git a/pkg/mentix/mentix.go b/pkg/mentix/mentix.go index 856d857af2..16ace2cea7 100644 --- a/pkg/mentix/mentix.go +++ b/pkg/mentix/mentix.go @@ -52,7 +52,7 @@ type Mentix struct { } const ( - runLoopSleeptime = time.Millisecond * 500 + runLoopSleeptime = time.Millisecond * 1000 ) func (mntx *Mentix) initialize(conf *config.Configuration, log *zerolog.Logger) error { diff --git a/pkg/mentix/meshdata/endpoints.go b/pkg/mentix/meshdata/endpoints.go index 02228b2040..3f259489c5 100644 --- a/pkg/mentix/meshdata/endpoints.go +++ b/pkg/mentix/meshdata/endpoints.go @@ -19,6 +19,9 @@ package meshdata const ( + // EndpointRevad identifies the main Reva Daemon endpoint + EndpointRevad = "REVAD" + // EndpointGateway identifies the Gateway endpoint EndpointGateway = "GATEWAY" // EndpointMetrics identifies the Metrics endpoint @@ -27,4 +30,19 @@ const ( EndpointWebdav = "WEBDAV" // EndpointOCM identifies the OCM endpoint EndpointOCM = "OCM" + // EndpointMeshDir identifies the Mesh Directory endpoint + EndpointMeshDir = "MESHDIR" ) + +// GetServiceEndpoints returns an array of all service endpoint identifiers. +func GetServiceEndpoints() []string { + return []string{ + EndpointRevad, + + EndpointGateway, + EndpointMetrics, + EndpointWebdav, + EndpointOCM, + EndpointMeshDir, + } +}