Skip to content

Commit

Permalink
Merge branch 'main' into observer-role
Browse files Browse the repository at this point in the history
  • Loading branch information
ItIsJordan committed Dec 18, 2024

Verified

This commit was created on GitHub.com and signed with GitHub’s verified signature. The key has expired.
2 parents d261bbb + 8a89c88 commit 949c59d
Showing 17 changed files with 175 additions and 21 deletions.
8 changes: 8 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -21,3 +21,11 @@ updates:
ignore:
- dependency-name: "pytest" # see https://github.com/HEPData/hepdata/issues/815
- dependency-name: "pytest-cov" # see https://github.com/HEPData/hepdata/issues/580
- dependency-name: "invenio-access" # see https://github.com/HEPData/hepdata/issues/848
- dependency-name: "invenio-assets" # see https://github.com/HEPData/hepdata/issues/848
- dependency-name: "invenio-logging" # see https://github.com/HEPData/hepdata/issues/848
- dependency-name: "invenio-oauthclient" # see https://github.com/HEPData/hepdata/issues/848
- dependency-name: "invenio-pidstore" # see https://github.com/HEPData/hepdata/issues/848
- dependency-name: "invenio-records" # see https://github.com/HEPData/hepdata/issues/848
- dependency-name: "invenio-search" # see https://github.com/HEPData/hepdata/issues/848
- dependency-name: "invenio-userprofiles" # see https://github.com/HEPData/hepdata/issues/848
2 changes: 1 addition & 1 deletion INSTALL.rst
Original file line number Diff line number Diff line change
@@ -249,7 +249,7 @@ Now start the HEPData web application in debug mode:
Then open your preferred web browser (Chrome, Firefox, Safari, etc.) at http://localhost:5000/ .

On macOS Monterey you might find that ControlCenter is already listening to port 5000
On macOS Monterey (and later) you might find that ControlCenter is already listening to port 5000
(check with ``lsof -i -P | grep 5000``). If this is the case,
`turn off AirPlay Receiver <https://support.apple.com/en-gb/guide/mac-help/mchl15c9e4b5/12.0/mac/12.0>`_.

4 changes: 2 additions & 2 deletions hepdata/cli.py
Original file line number Diff line number Diff line change
@@ -223,9 +223,9 @@ def do_unload(records_to_unload):

@utils.command()
@with_appcontext
@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS"). Omit for all.')
@click.option('--endpoint', '-e', type=str, help='Specific endpoint to update (e.g. "rivet" or "MadAnalysis" or "SModelS" or "Combine"). Omit for all.')
def find_and_add_record_analyses(endpoint):
"""Finds analyses such as Rivet, MadAnalysis 5 and SModelS and adds them to records."""
"""Finds analyses such as Rivet, MadAnalysis 5, SModelS and Combine and adds them to records."""
update_analyses(endpoint)


11 changes: 10 additions & 1 deletion hepdata/config.py
Original file line number Diff line number Diff line change
@@ -332,7 +332,16 @@ def _(x):
'endpoint_url': 'https://zenodo.org/records/13952092/files/smodels-analyses.hepdata.json?download=1',
'url_template': '{0}',
'subscribe_user_id': 7766
}
},
'Combine': {
'endpoint_url': 'https://cms-public-likelihoods-list.web.cern.ch/artifacts/output.json',
'url_template': 'https://doi.org/{0}',
'description': 'Statistical models',
'license': {
'name': 'cc-by-4.0',
'url': 'https://creativecommons.org/licenses/by/4.0'
},
},
#'ufo': {},
#'xfitter': {},
#'applgrid': {},
2 changes: 1 addition & 1 deletion hepdata/ext/opensearch/document_enhancers.py
Original file line number Diff line number Diff line change
@@ -94,7 +94,7 @@ def add_shortened_authors(doc):

def add_analyses(doc):
"""
Add analyses links such as Rivet, MadAnalysis 5, SModelS, HistFactory and NUISANCE to the index.
Add analyses links such as Rivet, MadAnalysis 5, SModelS, Combine, HistFactory and NUISANCE to the index.
:param doc:
:return:
1 change: 1 addition & 0 deletions hepdata/modules/records/assets/js/hepdata_common.js
Original file line number Diff line number Diff line change
@@ -46,6 +46,7 @@ HEPDATA.file_type_to_details = {
"rivet": {"icon": "area-chart", "description": "Rivet Analysis"},
"madanalysis": {"icon": "area-chart", "description": "MadAnalysis 5 Analysis"},
"smodels": {"icon": "area-chart", "description": "SModelS Analysis"},
"combine": {"icon": "area-chart", "description": "Combine Analysis"},
"xfitter": {"icon": "area-chart", "description": "xFitter Analysis"},
"applgrid": {"icon": "area-chart", "description": "APPLgrid Analysis"},
"ufo": {"icon": "rocket", "description": "Universal Feynrules Output (UFO)"},
Original file line number Diff line number Diff line change
@@ -41,6 +41,7 @@ <h4>Add Resource for <span id="selected_resource_item">Submission</span></h4>
<option value="applgrid">APPLgrid</option>
<option value="MadAnalysis">MadAnalysis 5</option>
<option value="SModelS">SModelS</option>
<option value="Combine">Combine</option>
<option value="rivet">Rivet</option>
<option value="fastnlo">fastNLO</option>
<option value="ufo">Universal Feynrules Output (UFO)</option>
13 changes: 11 additions & 2 deletions hepdata/modules/records/utils/analyses.py
Original file line number Diff line number Diff line change
@@ -35,6 +35,7 @@
from hepdata.utils.users import get_user_from_id
from hepdata.modules.records.subscribers.rest import subscribe
from hepdata.modules.records.subscribers.api import is_current_user_subscribed_to_record
from hepdata.modules.records.utils.common import get_license

logging.basicConfig()
log = logging.getLogger(__name__)
@@ -43,10 +44,11 @@
@shared_task
def update_analyses(endpoint=None):
"""
Update (Rivet, MadAnalysis 5 and SModelS) analyses and remove outdated resources.
Update (Rivet, MadAnalysis 5, SModelS and Combine) analyses and remove outdated resources.
Allow bulk subscription to record update notifications if "subscribe_user_id" in endpoint.
Add optional "description" and "license" fields if present in endpoint.
:param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or None (default) for both
:param endpoint: either "rivet" or "MadAnalysis" or "SModelS" or "Combine" or None (default) for both
"""
endpoints = current_app.config["ANALYSES_ENDPOINTS"]
for analysis_endpoint in endpoints:
@@ -86,6 +88,13 @@ def update_analyses(endpoint=None):
file_location=_resource_url,
file_type=analysis_endpoint)

if "description" in endpoints[analysis_endpoint]:
new_resource.file_description = str(endpoints[analysis_endpoint]["description"])

if "license" in endpoints[analysis_endpoint]:
resource_license = get_license(endpoints[analysis_endpoint]["license"])
new_resource.file_license = resource_license.id

submission.resources.append(new_resource)
num_new_resources += 1

Original file line number Diff line number Diff line change
@@ -243,6 +243,13 @@ <h4>Other useful searches</h4>
(SModelS analysis)
</span>
</li>
<li>
<a href='/search?q=analysis:Combine&sort_by=latest'
target="_new">analysis:Combine</a>
<span class="text-muted">
(CMS statistical models in Combine format)
</span>
</li>
<li>
<a href='/search?q=analysis:HistFactory&sort_by=latest'
target="_new">analysis:HistFactory</a>
2 changes: 1 addition & 1 deletion hepdata/version.py
Original file line number Diff line number Diff line change
@@ -28,4 +28,4 @@
and parsed by ``setup.py``.
"""

__version__ = "0.9.4dev20241112"
__version__ = "0.9.4dev20241211"
18 changes: 7 additions & 11 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,22 +1,18 @@
beautifulsoup4==4.12.3
bleach==6.2.0
datacite==1.1.4
datacite==1.2.0
gunicorn==23.0.0
hepdata-converter-ws-client==0.2.2
hepdata-validator==0.3.5
invenio-access==2.0.0 # Indirect (needed by invenio-admin)
invenio-accounts==5.1.3
invenio-admin==1.5.1
invenio-assets==3.0.3
invenio-access[admin]==3.0.2
invenio-assets==3.1.0
invenio-config==1.0.4
invenio-db[postgresql]==1.1.5
invenio-logging[sentry_sdk]==2.1.1
invenio-oauthclient==4.0.2
invenio-pidstore==1.3.1
invenio-records==2.3.0
invenio-oauthclient==4.1.3
invenio-pidstore==1.3.4
invenio-records[postgresql]==2.4.1
invenio-search[opensearch2]==2.4.1
invenio-theme==3.4.3
invenio-userprofiles==3.0.0
invenio-userprofiles==3.0.1
python-twitter-v2==0.9.2
responses==0.25.3
unicodeit==0.7.5
19 changes: 18 additions & 1 deletion tests/records_test.py
Original file line number Diff line number Diff line change
@@ -1031,7 +1031,7 @@ def test_create_breadcrumb_text():


def test_update_analyses(app):
""" Test update of Rivet, MadAnalyses 5 and SModelS analyses """
""" Test update of Rivet, MadAnalyses 5, SModelS and Combine analyses """

# Import a record that already has a Rivet analysis attached (but with '#' in the URL)
import_records(['ins1203852'], synchronous=True)
@@ -1074,6 +1074,23 @@ def test_update_analyses(app):
submission = get_latest_hepsubmission(inspire_id='1847779', overall_status='finished')
assert is_current_user_subscribed_to_record(submission.publication_recid, user)

# Import a record that has an associated Combine analysis
import_records(['ins2796231'], synchronous=True)
analysis_resources = DataResource.query.filter_by(file_type='Combine').all()
assert len(analysis_resources) == 0
analysis_resources = DataResource.query.filter_by(file_location='https://doi.org/10.17181/bp9fx-6qs64').all()
assert len(analysis_resources) == 1
db.session.delete(analysis_resources[0]) # delete resource so it can be re-added in next step
db.session.commit()
update_analyses('Combine')
analysis_resources = DataResource.query.filter_by(file_type='Combine').all()
assert len(analysis_resources) == 1
assert analysis_resources[0].file_location == 'https://doi.org/10.17181/bp9fx-6qs64'
assert analysis_resources[0].file_description == 'Statistical models'
license_data = License.query.filter_by(id=analysis_resources[0].file_license).first()
assert license_data.name == 'cc-by-4.0'
assert license_data.url == 'https://creativecommons.org/licenses/by/4.0'


def test_generate_license_data_by_id(app):
"""
86 changes: 85 additions & 1 deletion tests/search_test.py
Original file line number Diff line number Diff line change
@@ -21,14 +21,16 @@
from opensearch_dsl import Search, Index
from datetime import datetime
import pytest
import os as op_s
from invenio_db import db
from unittest.mock import call

from hepdata.ext.opensearch.config.os_config import \
add_default_aggregations, sort_fields_mapping
from hepdata.ext.opensearch import api as os_api
from hepdata.ext.opensearch.config.os_config import get_filter_field
from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies
from hepdata.ext.opensearch.document_enhancers import add_data_keywords, process_cmenergies, add_analyses
from hepdata.modules.records.utils.submission import process_submission_directory
from hepdata.utils.miscellaneous import get_resource_data
from hepdata.ext.opensearch.process_results import merge_results, match_tables_to_papers, \
get_basic_record_information, is_datatable
@@ -753,6 +755,88 @@ def test_add_data_keywords():
assert 'NOTAREALKEYWORD' not in doc['data_keywords']


def test_add_analyses(app):
"""
Tests the add_analyses function to ensure that DataSubmission data
is properly added to the doc object during document enhancement.
Currently testing against: NUISANCE, HistFactory, MadAnalysis
"""
# Here, test_data should match the contents of the test_folder
test_folder = "test_data/test_analysis_submission"
test_data = [
{ # ProSelecta/NUISANCE
"type": "NUISANCE",
"filename": "test.ProSelecta"
},
{ # HistFactory entry
"type": "HistFactory",
"filename": "test.tar.gz"
},
]
# This should probably be changed to use SITE_URL or some similar concept
analysis_url = "http://localhost:5000/record/resource/%s?landing_page=true"

with app.app_context():
# Creating and submitting the test submission containing resources
# op_s is os module
base_dir = op_s.path.dirname(op_s.path.realpath(__file__))

hepsubmission = HEPSubmission(publication_recid=123456,
overall_status="finished",
version=1,
doi="10.17182/hepdata.123456")
db.session.add(hepsubmission)
db.session.commit()

# Setting directory and executing processing
directory = op_s.path.join(base_dir, test_folder)
errors = process_submission_directory(
directory,
op_s.path.join(directory, "submission.yaml"),
hepsubmission.publication_recid
)

# No errors should happen
assert not errors

# Add MadAnalysis DataResource object separately
mad_analysis_resource = DataResource(
file_location = "placeholder",
file_type = "MadAnalysis",
file_description = "placeholder"
)

# Adding object to database
hepsubmission.resources.append(mad_analysis_resource)
db.session.add(mad_analysis_resource)
db.session.add(hepsubmission)
db.session.commit()

# Set up a generic doc object to match what add_analyses expects
test_doc = {"analyses": [], "recid": hepsubmission.publication_recid}
# Run the test add_analyses function
add_analyses(test_doc)

# A sorted list of all DataResource object IDs from submission
data_ids = sorted([r.id for r in hepsubmission.resources])

# There should be 3 analyses and 3 resources
assert len(data_ids) == len(test_doc["analyses"]) == 3

# There should be one entry into test_data per resource ID
# Looping through the test, resource IDs and the analysis outputs
for test, d_id, analysis in zip(test_data, data_ids, test_doc["analyses"]):
# Set the expected ID in the url to the sorted data_id entry
test["analysis"] = (analysis_url % d_id)
# Confirm data has been added to the doc
assert analysis == test

# Checking MadAnalysis added after submission
mad_analysis = test_doc["analyses"][-1]
assert mad_analysis["type"] == "MadAnalysis"


def test_process_cmenergies():
test_keywords = {
"cmenergies": [
10 changes: 10 additions & 0 deletions tests/test_data/test_analysis_submission/data1.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
dependent_variables:
- header: {name: TestData1-dependent-V1, units: NA}
qualifiers:
- {name: TestData1-qualifier-V1, value: 0}
values:
- {value: 0}
independent_variables:
- header: {name: TestData1-independent-V1, units: NA}
values:
- value: 0
12 changes: 12 additions & 0 deletions tests/test_data/test_analysis_submission/submission.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
---
description: "TestSubmission1-V1"
comment: TestComment1-V1
additional_resources:
- {description: Test, location: test.ProSelecta, type: ProSelecta }
- {description: Test, location: test.tar.gz, type: HistFactory }
---
name: "TestTable1-V1"
description: TestTable1-description-V1
keywords:
- {name: cmenergies, values: [0]}
data_file: data1.yaml
Empty file.
Empty file.

0 comments on commit 949c59d

Please sign in to comment.