Skip to content

Commit

Permalink
feat: update demodata loaders and scripts
Browse files Browse the repository at this point in the history
  • Loading branch information
@ChasNelson1990
ChasNelson1990 committed Sep 1, 2024
1 parent d0f4b7e commit fa1e4fc
Show file tree
Hide file tree
Showing 5 changed files with 414 additions and 178 deletions.
10 changes: 5 additions & 5 deletions util/demo_data_loader/ckan_loader.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
DATA_PATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), CONFIG['data_path'])
USERS_FILE = os.path.join(DATA_PATH, 'users.json')
ORGANIZATIONS_FILE = os.path.join(DATA_PATH, 'organizations.json')
DOCUMENTS_FILE = os.path.join(DATA_PATH, CONFIG['documents_file'])
DATASETS_FILE = os.path.join(DATA_PATH, CONFIG['datasets_file'])
GROUPS_FILE = os.path.join(DATA_PATH, CONFIG['groups_file'])
RESOURCE_FOLDER = os.path.join(DATA_PATH, CONFIG['resource_folder'])

Expand Down Expand Up @@ -84,7 +84,7 @@ def load_users(ckan):

def load_datasets(ckan, documents):
"""
Helper method to load datasets from the DOCUMENTS_FILE config file
Helper method to load datasets from the DATASETS_FILE config file
:param ckan: ckanapi instance
:param documents: a list of documents built from the metadata import file
:return: None
Expand Down Expand Up @@ -120,7 +120,7 @@ def load_datasets(ckan, documents):

def load_resources(ckan, documents):
"""
Helper method to load resources from the DOCUMENTS_FILE config file
Helper method to load resources from the DATASETS_FILE config file
:param ckan: ckanapi instance
:param documents: a list of documents built from the metadata import file
:return: None
Expand Down Expand Up @@ -153,7 +153,7 @@ def load_resources(ckan, documents):
_upload_resource(ckan, file_path, resource_dict)


def load_groups(ckan, documents):
def load_groups(ckan):
"""
Helper method to load groups from the GROUPS_FILE config file
:param ckan: ckanapi instance
Expand Down Expand Up @@ -283,7 +283,7 @@ def _create_tags(tags_str):


def _load_documents():
with open(DOCUMENTS_FILE) as csvfile:
with open(DATASETS_FILE) as csvfile:
metadata_reader = csv.reader(csvfile)
start_table = False
documents = []
Expand Down
2 changes: 1 addition & 1 deletion util/demo_data_loader/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
"users_file": "users.json",
"organizations_file": "organizations.json",
"groups_file": "groups.json",
"documents_file": "demo_data.csv",
"datasets_file": "demo_data.csv",
"ckan_url": "http://zarr.minikube",
"ckan_api_key": "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJqdGkiOiJTOG9fTEJCalhUU2wtbDI0aURBelpoMG04bXpaTXRSNVN0cmE3Ry13UDJBb1NabHFMOWRUTmVGV0x5b2hLaWVmLW10Q19seFZHbXFVcmxnNSIsImlhdCI6MTY4MzgwNTc2NX0.XQ-34lYAqiQoj83SfX371XwrjxC17e9ZABiiDjFl_tI",
"resource_folder": "files"
Expand Down
208 changes: 107 additions & 101 deletions util/demo_data_loader/inputs/demo_data/groups.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,151 +2,157 @@
"groups": [
{
"users": [
{
"name": "test_admin",
{
"name": "fjelltopp_admin",
"capacity": "admin"
},
{
"name": "test_editor",
"capacity": "editor"
},
}
],
"display_name": "Collection",
"description": "An aggregation of resources. A collection is described as a group; its parts may also be separately described.",
"image_url": "",
"name": "collection",
"state": "active",
"title": "Collection"
},
{
"users": [
{
"name": "test_member",
"capacity": "member"
"name": "fjelltopp_admin",
"capacity": "admin"
}
],
"display_name": "Publication",
"description": "Department publications and research studies",
"image_url": "https://dms.hiv.health.gov.mw/uploads/group/2022-03-17-123757.433896243660articlesquareicon.png",
"name": "publication",
"display_name": "Dataset",
"description": "Data encoded in a defined structure. Examples include lists, tables, and databases. A dataset may be useful for direct machine processing.",
"image_url": "",
"name": "dataset",
"state": "active",
"title": "Publication"
}, {
"title": "Dataset"
},
{
"users": [
{
"name": "test_admin",
"name": "fjelltopp_admin",
"capacity": "admin"
},
{
"name": "test_editor",
"capacity": "editor"
},
{
"name": "test_member",
"capacity": "member"
}
],
"display_name": "Guideline",
"description": "Guidelines for health service staff",
"image_url": "https://dms.hiv.health.gov.mw/uploads/group/2022-03-17-123643.7336343533402businessmanagementplanprojectreporticon.png",
"name": "guideline",
"display_name": "Event",
"description": "A non-persistent, time-based occurrence. Metadata for an event provides descriptive information that is the basis for discovery of the purpose, location, duration, and responsible agents associated with an event. Examples include an exhibition, webcast, conference, workshop, open day, performance, battle, trial, wedding, tea party, conflagration.",
"image_url": "",
"name": "event",
"state": "active",
"title": "Guideline"
}, {
"title": "Event"
},
{
"users": [
{
"name": "test_admin",
"name": "fjelltopp_admin",
"capacity": "admin"
},
{
"name": "test_editor",
"capacity": "editor"
},
{
"name": "test_member",
"capacity": "member"
}
],
"display_name": "Circulars",
"description": "Department internal circulars and reports",
"image_url": "https://dms.hiv.health.gov.mw/uploads/group/2022-03-17-123603.1298852205240clusterdatagrouporganizeicon.png",
"name": "circular",
"display_name": "Image",
"description": "A visual representation other than text. Examples include images and photographs of physical objects, paintings, prints, drawings, other images and graphics, animations and moving pictures, film, diagrams, maps, musical notation. Note that Image may include both electronic and physical representations.",
"image_url": "",
"name": "still_image",
"state": "active",
"title": "Circulars"
}, {
"title": "Image"
},
{
"users": [
{
"name": "test_admin",
"name": "fjelltopp_admin",
"capacity": "admin"
},
{
"name": "test_editor",
"capacity": "editor"
},
{
"name": "test_member",
"capacity": "member"
}
],
"display_name": "Data",
"description": "Data group for demo purposes",
"image_url": "https://dms.hiv.health.gov.mw//uploads/group/2022-07-24-152420.906558circular.png",
"name": "data",
"display_name": "Interactive Resource",
"description": "A resource requiring interaction from the user to be understood, executed, or experienced. Examples include forms on Web pages, applets, multimedia learning objects, chat services, or virtual reality environments.",
"image_url": "",
"name": "interactive_resource",
"state": "active",
"title": "Data"
}, {
"title": "Interactive Resource"
},
{
"users": [
{
"name": "test_admin",
"name": "fjelltopp_admin",
"capacity": "admin"
},
{
"name": "test_editor",
"capacity": "editor"
},
{
"name": "test_member",
"capacity": "member"
}
],
"display_name": "Minutes",
"description": "Minutes group for demo purposes",
"image_url": "https://dms.hiv.health.gov.mw/uploads/group/2022-03-17-123701.0261814308050communicationconnectionmeetingofficeicon.png",
"name": "minutes",
"display_name": "Video",
"description": "A series of visual representations imparting an impression of motion when shown in succession. Examples include animations, movies, television programs, videos, zoetropes, or visual output from a simulation. Instances of the type Moving Image must also be describable as instances of the broader type Image.",
"image_url": "",
"name": "moving_image",
"state": "active",
"title": "Minutes"
}, {
"title": "Video"
},
{
"users": [
{
"name": "test_admin",
"name": "fjelltopp_admin",
"capacity": "admin"
},
{
"name": "test_editor",
"capacity": "editor"
},
}
],
"display_name": "Physical Object",
"description": "An inanimate, three-dimensional object or substance. Note that digital representations of, or surrogates for, these objects should use Image, Text or one of the other types.",
"image_url": "",
"name": "physical_object",
"state": "active",
"title": "Physical Object"
},
{
"users": [
{
"name": "test_member",
"capacity": "member"
"name": "fjelltopp_admin",
"capacity": "admin"
}
],
"display_name": "Monitoring Tool",
"description": "Monitoring tool group for demo purposes",
"image_url": "https://dms.hiv.health.gov.mw/uploads/group/2022-03-17-123717.467925211661eyeicon.png",
"name": "monitoring",
"display_name": "Service",
"description": "A system that provides one or more functions. Examples include a photocopying service, a banking service, an authentication service, interlibrary loans, a Z39.50 or Web server.",
"image_url": "",
"name": "service",
"state": "active",
"title": "Monitoring Tool"
}, {
"title": "Service"
},
{
"users": [
{
"name": "test_admin",
"name": "fjelltopp_admin",
"capacity": "admin"
},
}
],
"display_name": "Software",
"description": "A computer program in source or compiled form. Examples include a C source file, MS-Windows .exe executable, or Perl script.",
"image_url": "",
"name": "software",
"state": "active",
"title": "Software"
},
{
"users": [
{
"name": "test_editor",
"capacity": "editor"
},
"name": "fjelltopp_admin",
"capacity": "admin"
}
],
"display_name": "Audio",
"description": "A resource primarily intended to be heard. Examples include a music playback file format, an audio compact disc, and recorded speech or sounds.",
"image_url": "",
"name": "sound",
"state": "active",
"title": "Audio"
},
{
"users": [
{
"name": "test_member",
"capacity": "member"
"name": "fjelltopp_admin",
"capacity": "admin"
}
],
"display_name": "Plan",
"description": "Plan group for demo purposes",
"image_url": "https://dms.hiv.health.gov.mw/uploads/group/2022-03-17-123743.3695114308313economicsmarketplanstrategicstrategyicon.png",
"name": "plan",
"display_name": "Document",
"description": "A resource consisting primarily of words for reading. Examples include books, letters, dissertations, poems, newspapers, articles, archives of mailing lists. Note that facsimiles or images of texts are still of the genre Text.",
"image_url": "",
"name": "text",
"state": "active",
"title": "Plan"
"title": "Document"
}
]
}
Loading

0 comments on commit fa1e4fc

Please sign in to comment.