Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add hierarchy group by's to Cloud realm #1309

Merged
merged 11 commits into from
May 13, 2020
Merged

Add hierarchy group by's to Cloud realm #1309

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
cea6426
adding hierarchy group bys
eiffel777 Apr 9, 2020
798a0c2
fixing merge conflicts
eiffel777 Apr 20, 2020
54bfdea
updating how fos_id is set and adding all tests
eiffel777 Apr 22, 2020
d74f238
merging latest xdmod9.0
eiffel777 May 12, 2020
2e238d0
documentation updates
eiffel777 May 12, 2020
e6d89bf
removing extraneous formatting
eiffel777 May 12, 2020
83c8d12
Merge branch 'xdmod9.0' into cloud-add-hierarchy-groupbys
eiffel777 May 12, 2020
aa54f96
Merge branch 'xdmod9.0' into cloud-add-hierarchy-groupbys
eiffel777 May 12, 2020
d6da177
add importing names.csv on upgrade
eiffel777 May 12, 2020
202747c
Merge branch 'cloud-add-hierarchy-groupbys' of https://github.com/eif…
eiffel777 May 12, 2020
7cc2541
updates to bootstrap.sh when doing an upgrade
eiffel777 May 13, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions configuration/datawarehouse.d/ref/Cloud-group-bys.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,5 +282,14 @@
},
"pi": {
"$ref": "datawarehouse.d/ref/group-by-common.json#/pi"
},
"fieldofscience": {
"$ref": "datawarehouse.d/ref/group-by-hierarchy.json#/fieldofscience"
},
"nsfdirectorate": {
"$ref": "datawarehouse.d/ref/group-by-hierarchy.json#/nsfdirectorate"
},
"parentscience": {
"$ref": "datawarehouse.d/ref/group-by-hierarchy.json#/parentscience"
}
}
7 changes: 5 additions & 2 deletions configuration/etl/etl_action_defs.d/cloud_common/cloud_metrics_aggregation.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@
"submission_venue_id": "sr.submission_venue_id",
"domain_id": "sr.domain_id",
"service_provider": "sr.service_provider",
"principalinvestigator_person_id": "a.principalinvestigator_person_id"
"principalinvestigator_person_id": "a.principalinvestigator_person_id",
"fos_id": "a.fos_id"
},
"groupby": [
"${AGGREGATION_UNIT}_id",
Expand Down Expand Up @@ -75,10 +76,12 @@
"schema": "${UTILITY_SCHEMA}",
"alias": "task_resource",
"on": "task_resource.id = sr.resource_id"
},{
},
{
"name": "account",
"schema": "${SOURCE_SCHEMA}",
"alias": "a",
"type": "LEFT",
"on": "instance.account_id = a.account_id"
}
],
Expand Down
5 changes: 4 additions & 1 deletion configuration/etl/etl_sql.d/cloud_common/pi_to_account.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ LEFT JOIN
modw.account as acc ON pi.pi_name = acc.charge_number
LEFT JOIN
modw.principalinvestigator as p ON acc.id = p.request_id
LEFT JOIN
mod_hpcdb.hpcdb_requests as req ON acc.id = req.request_id
SET
a.principalinvestigator_person_id = COALESCE(p.person_id, -1)
a.principalinvestigator_person_id = COALESCE(p.person_id, -1),
a.fos_id = COALESCE(req.primary_fos_id, 1)
//
7 changes: 7 additions & 0 deletions configuration/etl/etl_tables.d/cloud_common/account.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,13 @@
"type": "int(11)",
"nullable": true,
"default": null
},
{
"name": "fos_id",
"type": "int(11)",
"nullable": false,
"default": 1,
"comment": "The field of science of the project to which the instance belongs to"
}
],

Expand Down
6 changes: 6 additions & 0 deletions configuration/etl/etl_tables.d/cloud_common/cloudfact_by_.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,6 +127,12 @@
"nullable": false,
"default": -1,
"comment": "DIMENSION: The PI that owns the allocations that these VM's ran under. References principalinvestigator.person_id"
},{
"name": "fos_id",
"type": "int(11)",
"nullable": false,
"default": 1,
"comment": "DIMENSION: The field of science of the project to which the jobs belong"
}
],

Expand Down
24 changes: 24 additions & 0 deletions configuration/roles.d/cloud.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
],
"+summary_charts": [{
Expand Down Expand Up @@ -163,6 +175,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
]
}
Expand Down
8 changes: 8 additions & 0 deletions docs/cloud.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,14 @@ as seen in your resources event log files. The third column is the name of the r
If you want the first and last name of the PI to be shown instead of their username when viewing this data you should add the PI username and
first and last name to the `names.csv` file and ingested. Details on doing this can be found in the [`User/PI Names`](user-names.md) documentation.

## Hierarchy

Open XDMoD allows you to define a three level hierarchy that can be used to define various entities or groups and associate users with a group in
the hierarchy. These can be decanal units and their associated departments or any hierarchy that is desired. If defined, this hierarchy is used
to generate charts that aggregate cloud metrics into groups based on users assigned to one of the groups.

See the [Hierarchy Guide](hierarchy.html) for more details.

## Adding and enabling cloud resources

### Add a cloud resource
Expand Down
12 changes: 12 additions & 0 deletions tests/artifacts/xdmod/configuration/input/datawarehouse.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -373,6 +373,18 @@
{
"name": "pi",
"class": "GroupByPI"
},
{
"name": "fieldofscience",
"class": "GroupByScience"
},
{
"name": "nsfdirectorate",
"class": "GroupByNSFDirectorate"
},
{
"name": "parentscience",
"class": "GroupByParentScience"
}
],
"statistics": [
Expand Down
24 changes: 24 additions & 0 deletions tests/artifacts/xdmod/configuration/input/roles.d/module_configuration/cloud.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
]
},
Expand Down Expand Up @@ -85,6 +97,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
]
}
Expand Down
24 changes: 24 additions & 0 deletions tests/artifacts/xdmod/configuration/input/roles.d/multi_extends_chain/cloud.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
]
},
Expand Down Expand Up @@ -85,6 +97,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
]
},
Expand Down
24 changes: 24 additions & 0 deletions tests/artifacts/xdmod/configuration/input/roles.d/xdmod_config/cloud.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
]
},
Expand Down Expand Up @@ -85,6 +97,18 @@
{
"realm": "Cloud",
"group_by": "pi"
},
{
"realm": "Cloud",
"group_by": "fieldofscience"
},
{
"realm": "Cloud",
"group_by": "nsfdirectorate"
},
{
"realm": "Cloud",
"group_by": "parentscience"
}
]
}
Expand Down
12 changes: 6 additions & 6 deletions tests/artifacts/xdmod/configuration/output/module_config_roles-new_module.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

],
"query_descripters": {
"24": {
"27": {
"realm": "NewModule",
"group_by": "none"
}
Expand All @@ -24,7 +24,7 @@

],
"query_descripters": {
"24": {
"27": {
"realm": "NewModule",
"group_by": "none"
}
Expand All @@ -43,7 +43,7 @@

],
"query_descripters": {
"24": {
"27": {
"realm": "NewModule",
"group_by": "none"
}
Expand All @@ -65,7 +65,7 @@

],
"query_descripters": {
"24": {
"27": {
"realm": "NewModule",
"group_by": "none"
}
Expand All @@ -87,7 +87,7 @@

],
"query_descripters": {
"24": {
"27": {
"realm": "NewModule",
"group_by": "none"
}
Expand All @@ -109,7 +109,7 @@

],
"query_descripters": {
"24": {
"27": {
"realm": "NewModule",
"group_by": "none"
}
Expand Down
Loading