From 597fde560b4f32b00e026ce744ca009b6d2f2a32 Mon Sep 17 00:00:00 2001 From: Elliot Date: Sat, 1 Feb 2025 08:41:17 -0500 Subject: [PATCH 01/18] Import alerting and monitoring (#5062) * WIP; Scaffold for import thresholds * WIP; UI for import thresholds * Assignment UI functional * Assignment UI functional * Migration from import extension to import threshold * Pause imports based on import thresholds instead of data source config * WIP; import notifications and pausing; scaffold for tests * WIP; setup for testing * WIP; catch change counts before import * Fixes for failing tests * WIP; everything functioning in app * Working on tests * Test for sending email * Be a bit more lenient with checking delayed jobs * Cleanup * Review cleanup * Use active record uncached instead of hack * Centralize find safely logic; future work to expand the usage * Use bell instead of alert icon * Remove unused method --- .github/rspec_buckets.json | 10 +- .../import_thresholds_controller.rb | 40 +++ .../notification_configurations_controller.rb | 63 +++++ app/mailers/notify_user.rb | 11 + app/models/application_record.rb | 11 + .../concerns/service_history/builder.rb | 6 +- app/models/concerns/user_concern.rb | 3 +- app/models/grda_warehouse/client_match.rb | 2 +- app/models/grda_warehouse/data_source.rb | 97 ++++++- app/models/grda_warehouse/import_threshold.rb | 194 +++++++++++++ .../notification_configuration.rb | 28 ++ app/models/grda_warehouse/utility.rb | 4 + .../importers/hmis_auto_migrate/local.rb | 5 +- app/views/data_sources/_breadcrumbs.haml | 6 +- app/views/data_sources/_buttons.haml | 4 + app/views/import_thresholds/_table.haml | 20 ++ app/views/import_thresholds/show.haml | 43 +++ .../notification_configurations/_form.haml | 5 + .../notification_configurations/create.js.erb | 8 + .../destroy.js.erb | 5 + .../notification_configurations/edit.haml | 3 + .../notification_configurations/new.haml | 3 + .../notification_configurations/update.js.erb | 8 + .../notify_user/import_processing.html.haml | 15 + .../notify_user/import_processing.text.haml | 11 + config/routes.rb | 3 + ...45506_create_notification_configuration.rb | 24 ++ ..._data_source_config_to_import_threshold.rb | 20 ++ db/warehouse_structure.sql | 136 ++++++++- .../importer_extensions_controller.rb | 1 - .../hmis_csv_importer/importer/importer.rb | 259 ++++++++++++++++-- .../importer/importer_log.rb | 8 + .../import_logs/_import_log.haml | 23 +- .../_import_threshold_description.haml | 13 + .../importer_extensions/edit.haml | 7 - .../import_threshold_spec.rb | 73 +++++ .../importer_extensions_controller.rb | 1 - .../importer/importer.rb | 41 ++- .../importer_extensions/edit.haml | 7 - .../spec/models/hmis_auto_detect_spec.rb | 56 ---- .../models/prepend_organization_ids_spec.rb | 76 ----- .../grda_warehouse/import_thresholds.rb | 9 + .../notification_configurations.rb | 13 + spec/models/application_record_spec.rb | 33 +++ spec/support/hmis_csv_fixtures.rb | 54 ++-- 45 files changed, 1221 insertions(+), 241 deletions(-) create mode 100644 app/controllers/import_thresholds_controller.rb create mode 100644 app/controllers/notification_configurations_controller.rb create mode 100644 app/models/grda_warehouse/import_threshold.rb create mode 100644 app/models/grda_warehouse/notification_configuration.rb create mode 100644 app/views/import_thresholds/_table.haml create mode 100644 app/views/import_thresholds/show.haml create mode 100644 app/views/notification_configurations/_form.haml create mode 100644 app/views/notification_configurations/create.js.erb create mode 100644 app/views/notification_configurations/destroy.js.erb create mode 100644 app/views/notification_configurations/edit.haml create mode 100644 app/views/notification_configurations/new.haml create mode 100644 app/views/notification_configurations/update.js.erb create mode 100644 app/views/notify_user/import_processing.html.haml create mode 100644 app/views/notify_user/import_processing.text.haml create mode 100644 db/warehouse/migrate/20250116145506_create_notification_configuration.rb create mode 100644 db/warehouse/migrate/20250117174547_convert_data_source_config_to_import_threshold.rb create mode 100644 drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_threshold_description.haml create mode 100644 drivers/hmis_csv_importer/spec/models/importer/twenty_twenty_four/import_threshold_spec.rb delete mode 100644 drivers/hmis_csv_twenty_twenty/spec/models/hmis_auto_detect_spec.rb delete mode 100644 drivers/hmis_csv_twenty_twenty/spec/models/prepend_organization_ids_spec.rb create mode 100644 spec/factories/grda_warehouse/import_thresholds.rb create mode 100644 spec/factories/grda_warehouse/notification_configurations.rb create mode 100644 spec/models/application_record_spec.rb diff --git a/.github/rspec_buckets.json b/.github/rspec_buckets.json index b0a976c32c9..f380ab16ed4 100644 --- a/.github/rspec_buckets.json +++ b/.github/rspec_buckets.json @@ -201,14 +201,6 @@ "average": 0.6751237092105263, "location": "./drivers/hud_path_report/spec/models/fy2021/question_twenty_six_spec.rb:16" }, - { - "count": 4, - "start": "2024-07-20T16:36:53.099+00:00", - "description": "Prepend Organization IDs", - "total_time": 12.343842105, - "average": 3.08596052625, - "location": "./drivers/hmis_csv_twenty_twenty/spec/models/prepend_organization_ids_spec.rb:9" - }, { "count": 12, "start": "2024-07-20T16:59:17.391+00:00", @@ -463,4 +455,4 @@ } ] } -] \ No newline at end of file +] diff --git a/app/controllers/import_thresholds_controller.rb b/app/controllers/import_thresholds_controller.rb new file mode 100644 index 00000000000..66a9faa23f5 --- /dev/null +++ b/app/controllers/import_thresholds_controller.rb @@ -0,0 +1,40 @@ +### +# Copyright 2016 - 2025 Green River Data Analysis, LLC +# +# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md +### + +class ImportThresholdsController < ApplicationController + include AjaxModalRails::Controller + before_action :require_can_view_imports_projects_or_organizations!, only: [:show] + before_action :data_source + before_action :import_threshold + + def show + end + + def update + import_threshold.update!(import_threshold_params) + respond_with(import_threshold, location: data_source_import_threshold_path) + end + + private def import_threshold_params + params.require(:grda_warehouse_import_threshold). + permit(*GrdaWarehouse::ImportThreshold.known_params) + end + + private def data_source_scope + GrdaWarehouse::DataSource.viewable_by(current_user, permission: :can_view_projects) + end + + private def data_source + @data_source ||= data_source_scope.find_safely(params[:data_source_id]) + end + helper_method :data_source + + # Ensure the import threshold is saved so the related notifications can be added + private def import_threshold + @import_threshold ||= data_source.import_threshold || GrdaWarehouse::ImportThreshold.create!(data_source_id: data_source.id) + end + helper_method :import_threshold +end diff --git a/app/controllers/notification_configurations_controller.rb b/app/controllers/notification_configurations_controller.rb new file mode 100644 index 00000000000..97104180c9c --- /dev/null +++ b/app/controllers/notification_configurations_controller.rb @@ -0,0 +1,63 @@ +### +# Copyright 2016 - 2025 Green River Data Analysis, LLC +# +# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md +### + +class NotificationConfigurationsController < ApplicationController + include AjaxModalRails::Controller + before_action :require_can_view_imports_projects_or_organizations!, only: [:show] + before_action :data_source + before_action :import_threshold + + def new + @form_url = data_source_import_threshold_notification_configurations_path(notification_slug: import_threshold.valid_notification_slug(params[:notification_slug])) + end + + def edit + @form_url = data_source_import_threshold_notification_configuration_path(notification_slug: import_threshold.valid_notification_slug(params[:notification_slug])) + end + + def create + notification_configuration.update!(notification_configuration_params.merge(notification_slug: import_threshold.valid_notification_slug(params[:notification_slug]))) + respond_with(notification_configuration, location: data_source_import_threshold_path) + end + + def update + notification_configuration.update!(notification_configuration_params) + respond_with(notification_configuration, location: data_source_import_threshold_path) + end + + def destroy + notification_configuration.destroy! + respond_with(notification_configuration, location: data_source_import_threshold_path) + end + + private def notification_configuration_params + params.require(:grda_warehouse_notification_configuration). + permit(:user_id, :active) + end + + private def data_source_scope + GrdaWarehouse::DataSource.viewable_by(current_user, permission: :can_view_projects) + end + + private def data_source + @data_source ||= data_source_scope.find(params[:data_source_id].to_i) + end + helper_method :data_source + + private def import_threshold + @import_threshold ||= data_source.import_threshold + end + helper_method :import_threshold + + def notification_configuration + @notification_configuration ||= GrdaWarehouse::NotificationConfiguration.find_safely(params[:id]) || + GrdaWarehouse::NotificationConfiguration.new( + source: import_threshold, + notification_slug: import_threshold.valid_notification_slug(params[:notification_slug]), + ) + end + helper_method :notification_configuration +end diff --git a/app/mailers/notify_user.rb b/app/mailers/notify_user.rb index 4663247edfc..146257ec9bb 100644 --- a/app/mailers/notify_user.rb +++ b/app/mailers/notify_user.rb @@ -234,4 +234,15 @@ def new_account_created(new_user) mail(to: user.email, subject: 'Account Created') end end + + def import_processing + @user = params[:user] + @import = params[:import_log_id] + @data_source = params[:data_source] + @error = params[:error] + @count = params[:count] + @paused = params[:paused] + subject = 'HMIS Import Status Update' + mail(to: @user.email, subject: subject) + end end diff --git a/app/models/application_record.rb b/app/models/application_record.rb index 87669d92b0d..208fb4002e7 100644 --- a/app/models/application_record.rb +++ b/app/models/application_record.rb @@ -12,6 +12,17 @@ class ApplicationRecord < ActiveRecord::Base connects_to database: { writing: :primary, reading: :primary } + def self.find_safely(tainted_id) + safe_id = begin + Integer(tainted_id) + rescue ArgumentError, TypeError + nil + end + raise(ActiveRecord::RecordNotFound, "#{sti_name} Record not found for ID: #{tainted_id}") unless safe_id + + find(safe_id) + end + def self.needs_migration? ActiveRecord::Migration.check_pending! end diff --git a/app/models/concerns/service_history/builder.rb b/app/models/concerns/service_history/builder.rb index 7b95cbe3929..541e7daa15f 100644 --- a/app/models/concerns/service_history/builder.rb +++ b/app/models/concerns/service_history/builder.rb @@ -60,7 +60,7 @@ def wait_for_processing(interval: 30, max_wait_seconds: DEFAULT_MAX_WAIT_SECONDS else started = Time.current while builder_batch_job_scope.exists? - break if (Time.current - started) > max_wait_seconds + return if (Time.current - started) > max_wait_seconds sleep(interval) end @@ -121,7 +121,9 @@ def clients_still_processing?(client_ids:) # Class method private def builder_batch_job_scope - Delayed::Job.where(failed_at: nil).jobs_for_class('ServiceHistory::RebuildEnrollments') + Delayed::Job.uncached do + Delayed::Job.where(failed_at: nil).jobs_for_class('ServiceHistory::RebuildEnrollments') + end end # Class method diff --git a/app/models/concerns/user_concern.rb b/app/models/concerns/user_concern.rb index a95954aba62..80e486167c3 100644 --- a/app/models/concerns/user_concern.rb +++ b/app/models/concerns/user_concern.rb @@ -60,7 +60,8 @@ module UserConcern after_save :create_access_group # END_ACL - validates :email, presence: true, uniqueness: true, email_format: { check_mx: true }, length: { maximum: 250 } + # No longer validating MX record, just validate email format (MX check requires a network connection) + validates :email, presence: true, uniqueness: true, email_format: { check_mx: false }, length: { maximum: 250 } validate :password_cannot_be_sequential, on: :update validates :last_name, presence: true, length: { maximum: 40 } validates :first_name, presence: true, length: { maximum: 40 } diff --git a/app/models/grda_warehouse/client_match.rb b/app/models/grda_warehouse/client_match.rb index f9adff1ace9..c2746762d6f 100644 --- a/app/models/grda_warehouse/client_match.rb +++ b/app/models/grda_warehouse/client_match.rb @@ -58,7 +58,7 @@ class ClientMatch < GrdaWarehouseBase # In addition, if either the source or destination client no longer # exists, we'll delete the match def self.accept_exact_matches! - candidate. + candidate.preload(:source_client, :destination_client). find_each do |match| sc = match.source_client dc = match.destination_client diff --git a/app/models/grda_warehouse/data_source.rb b/app/models/grda_warehouse/data_source.rb index 9399cff059e..2d1c0a2a6f2 100644 --- a/app/models/grda_warehouse/data_source.rb +++ b/app/models/grda_warehouse/data_source.rb @@ -3,14 +3,19 @@ # # License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md ### +# +require 'memery' class GrdaWarehouse::DataSource < GrdaWarehouseBase include RailsDrivers::Extensions include EntityAccess include ArelHelper + include Memery self.primary_key = :id - require 'memery' + TodoOrDie('Enable the following after release-151', by: '2025-03-01') + # self.ignored_columns = ['refuse_imports_with_errors'] + TodoOrDie('Add a migration to remove refuse_imports_with_errors column from data source', by: '2025-04-01') acts_as_paranoid validates :name, presence: true @@ -35,6 +40,7 @@ class GrdaWarehouse::DataSource < GrdaWarehouseBase has_many :non_hmis_uploads has_one :hmis_import_config + has_one :import_threshold accepts_nested_attributes_for :organizations accepts_nested_attributes_for :projects @@ -568,6 +574,95 @@ def self.options_for_select(user:) end end + delegate :error_count_threshold_reached?, to: :import_threshold, allow_nil: true + ## + # Determines whether imports should be paused based on error thresholds. + # + # This method checks if an import threshold is set and whether the number of errors + # exceeds the allowed limit. If so, it returns `true`, indicating that imports should + # be paused. + # + # @param total [Integer] The total number of rows in the import file. Required. + # @param errors [Integer] The number of errors encountered. Required. + # @return [Boolean] `true` if imports should be paused due to errors, otherwise `false`. + # + memoize def pause_imports_with_errors?(total:, errors:) + return false unless import_threshold.present? + return false unless import_threshold.pause_on_error_threshold + + error_count_threshold_reached?(total, errors) + end + + ## + # Determines if imports should ever be paused due to errors. + # + # This method checks whether an error threshold is configured and if a minimum error count + # or percentage threshold is set. If both conditions are met, it indicates that the import + # process has a mechanism to pause based on errors. + # + # @return [Boolean] `true` if error-based pausing is possible, otherwise `false`. + # + memoize def ever_pause_imports_with_errors? + return false unless import_threshold.present? + return false unless import_threshold.pause_on_error_threshold + + import_threshold.error_count_min_threshold.present? && import_threshold.error_percent_threshold.present? + end + + delegate :record_count_threshold_reached?, to: :import_threshold, allow_nil: true + ## + # Determines whether imports should be paused based on record count change thresholds. + # + # This method checks if an import threshold is set and whether the number of changes to records + # (overall additions and removals) + # exceeds the allowed limit. If so, it returns `true`, indicating that imports should + # be paused. + # + # @param total [Integer] The total number of records in the data source for the given class. Required. + # @param errors [Integer] The absolute number of additions - removals encountered for the import file. Required. + # @return [Boolean] `true` if imports should be paused due to errors, otherwise `false`. + # + memoize def pause_imports_with_record_count_changes?(total:, changes:) + return false unless import_threshold.present? + return false unless import_threshold.pause_on_record_count_threshold + + record_count_threshold_reached?(total, changes) + end + + ## + # Determines if imports should ever be paused due to changes in record counts. + # + # This method checks whether a threshold for record count changes is configured and if + # minimum change or percentage thresholds are defined. If both conditions exist, the system + # can pause imports based on significant changes in the number of records. + # + # @return [Boolean] `true` if imports may be paused due to record count changes, otherwise `false`. + # + memoize def ever_pause_imports_with_record_changes? + return false unless import_threshold.present? + return false unless import_threshold.pause_on_error_threshold + + import_threshold.record_count_change_min_threshold.present? && import_threshold.record_count_change_percent_threshold.present? + end + + def notify_of_import_status(import_log_id:, error_threshold_met:, record_count_threshold_met:, paused:) + return unless import_threshold.present? + + import_threshold.send_status_notifications( + import_log_id: import_log_id, + error_threshold_met: error_threshold_met, + record_count_threshold_met: record_count_threshold_met, + paused: paused, + ) + end + + def ever_notify_for_imports? + return false unless import_threshold.present? + + # Do we have any thresholds set? + import_threshold.record_count_change_percent_threshold || import_threshold.error_percent_threshold + end + def manual_import_path "/tmp/uploaded#{file_path}" end diff --git a/app/models/grda_warehouse/import_threshold.rb b/app/models/grda_warehouse/import_threshold.rb new file mode 100644 index 00000000000..4e09cefea7b --- /dev/null +++ b/app/models/grda_warehouse/import_threshold.rb @@ -0,0 +1,194 @@ +### +# Copyright 2016 - 2025 Green River Data Analysis, LLC +# +# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md +### + +require 'memery' + +# This defines two thresholds for imports that will potentially trigger notifications +# 1. When an import is going to add or remove a significant percentage of the existing records, and that change is greater than the min count threshold +# 2. When an import contains a high percentage of errors, and that change is greater than the min count threshold +module GrdaWarehouse + class ImportThreshold < GrdaWarehouseBase + include Memery + belongs_to :data_source + + validates :error_count_min_threshold, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true + validates :error_percent_threshold, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 100 }, allow_nil: true + validates :record_count_change_min_threshold, numericality: { greater_than_or_equal_to: 0 }, allow_nil: true + validates :record_count_change_percent_threshold, numericality: { greater_than_or_equal_to: 0, less_than_or_equal_to: 100 }, allow_nil: true + + ## + # Determines whether the record count change during an import meets or exceeds + # the defined thresholds for triggering a notification. + # + # @param total [Integer] The total number of existing records before the import. + # @param count [Integer] The number of records being added or removed. + # @return [Boolean] Returns `true` if the count meets or exceeds both the + # minimum count threshold and the percentage change threshold, otherwise `false`. + # + # The method performs the following checks: + # - Returns `false` if any of the required threshold values (`record_count_change_min_threshold` + # or `record_count_change_percent_threshold`) or input parameters (`total` or `count`) are missing. + # - Returns `false` if `count` or `total` is zero (to avoid division errors and meaningless calculations). + # - Returns `false` if `count` is below the `record_count_change_min_threshold`. + # - Returns `false` if the percentage change is below the `record_count_change_percent_threshold`. + # - Returns `true` if all conditions are met. + # + def record_count_threshold_reached?(total, count) + return false unless record_count_change_min_threshold && record_count_change_percent_threshold && total && count + return false if count.zero? || total.zero? + return false if count < record_count_change_min_threshold + + percent_change = (count / total.to_f) * 100 + return false if percent_change < record_count_change_percent_threshold + + true + end + + ## + # Determines whether the count of errors during an import meets or exceeds + # the defined thresholds for triggering a notification. + # + # @param total [Integer] The total number of records in the import. + # @param count [Integer] The number of records containing an error. + # @return [Boolean] Returns `true` if the count meets or exceeds both the + # minimum count threshold and the percentage change threshold, otherwise `false`. + # + # The method performs the following checks: + # - Returns `false` if any of the required threshold values (`error_count_min_threshold` + # or `error_percent_threshold`) or input parameters (`total` or `count`) are missing. + # - Returns `false` if `count` or `total` is zero (to avoid division errors and meaningless calculations). + # - Returns `false` if `count` is below the `error_count_min_threshold`. + # - Returns `false` if the percentage change is below the `error_percent_threshold`. + # - Returns `true` if all conditions are met. + # + def error_count_threshold_reached?(total, count) + return false unless error_count_min_threshold && error_percent_threshold && total && count + return false if count.zero? || total.zero? + return false if count < error_count_min_threshold + + percent_change = (count / total.to_f) * 100 + return false if percent_change < error_percent_threshold + + true + end + + # Gather users from notification configurations into 3 categories + # 1. Those that have notifications for errors + # 2. Those that have notifications for count changes + # 3. Those that have both + def send_status_notifications(import_log_id:, error_threshold_met:, record_count_threshold_met:, paused:) + return unless error_threshold_met || record_count_threshold_met + + receive_both_user_ids = error_count_notification_user_ids & record_change_count_notification_user_ids + + # Handle users who subscribe to both types - they get one notification containing both status types + if error_threshold_met || record_count_threshold_met + User.where(id: receive_both_user_ids).find_each do |user| + NotifyUser.with( + user: user, + import_log_id: import_log_id, + data_source: data_source, + error: error_threshold_met, + count: record_count_threshold_met, + paused: paused, + ).import_processing.deliver_later + end + end + + only_error_user_ids = error_count_notification_user_ids - receive_both_user_ids + # Notify where the user receives only the error notification + if error_threshold_met + User.where(id: only_error_user_ids).find_each do |user| + NotifyUser.with( + user: user, + import_log_id: import_log_id, + data_source: data_source, + error: error_threshold_met, + count: false, # never notify on counts in this scenario + paused: paused, + ).import_processing.deliver_later + end + end + + only_count_user_ids = record_change_count_notification_user_ids - receive_both_user_ids + # Notify where the user receives only the record count notification + if record_count_threshold_met # rubocop:disable Style/GuardClause + User.where(id: only_count_user_ids).find_each do |user| + NotifyUser.with( + user: user, + import_log_id: import_log_id, + data_source: data_source, + error: false, # never notify on errors in this scenario + count: record_count_threshold_met, + paused: paused, + ).import_processing.deliver_later + end + end + end + + memoize def error_count_notification_user_ids + error_count_notifications.select(&:active).map(&:user_id) + end + + memoize def record_change_count_notification_user_ids + record_count_change_notifications.select(&:active).map(&:user_id) + end + + def error_count_notifications + @error_count_notifications ||= GrdaWarehouse::NotificationConfiguration.where( + notification_slug: error_count_notification_event, + source: self, + ).preload(:user). + to_a + end + + def record_count_change_notifications + @record_count_change_notifications ||= GrdaWarehouse::NotificationConfiguration.where( + notification_slug: record_count_change_notification_event, + source: self, + ).preload(:user). + to_a + end + + def valid_notification_slug(slug) + valid_slug = [ + record_count_change_notification_event, + error_count_notification_event, + ].detect { |m| m == slug } + return valid_slug if valid_slug.present? + + raise "Unknown slug #{slug}" + end + + def items_for(slug) + case slug + when record_count_change_notification_event + record_count_change_notifications + when error_count_notification_event + error_count_notifications + end + end + + def record_count_change_notification_event + 'count_threshold_exceeded' + end + + def error_count_notification_event + 'error_threshold_exceeded' + end + + def self.known_params + [ + :record_count_change_min_threshold, + :record_count_change_percent_threshold, + :error_count_min_threshold, + :error_percent_threshold, + :pause_on_record_count_threshold, + :pause_on_error_threshold, + ] + end + end +end diff --git a/app/models/grda_warehouse/notification_configuration.rb b/app/models/grda_warehouse/notification_configuration.rb new file mode 100644 index 00000000000..d5e5ca650ae --- /dev/null +++ b/app/models/grda_warehouse/notification_configuration.rb @@ -0,0 +1,28 @@ +### +# Copyright 2016 - 2025 Green River Data Analysis, LLC +# +# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md +### + +# This class holds links between: +# 1. The item that will trigger a notification (source) +# 2. The user who should receive the notification +# 3. The notification type +module GrdaWarehouse + class NotificationConfiguration < GrdaWarehouseBase + belongs_to :user # NOTE: this is a cross-database relationship + belongs_to :source, polymorphic: true + + def available_users + possible_users.where.not(id: unavailable_user_ids) + end + + private def possible_users + User.active.not_system.order(last_name: :asc, first_name: :asc) + end + + private def unavailable_user_ids + self.class.where(source: source, notification_slug: notification_slug).pluck(:user_id) - [user_id] + end + end +end diff --git a/app/models/grda_warehouse/utility.rb b/app/models/grda_warehouse/utility.rb index 76eebe7c6c0..8db2664c673 100644 --- a/app/models/grda_warehouse/utility.rb +++ b/app/models/grda_warehouse/utility.rb @@ -71,6 +71,7 @@ def self.clear! UserGroupMember, UserGroup, AccessControl, + User, HudReports::ReportInstance, HudReports::UniverseMember, HudReports::ReportCell, @@ -96,6 +97,8 @@ def self.clear! ActiveStorage::Blob, GrdaWarehouse::File, GrdaWarehouse::Config, + GrdaWarehouse::ImportThreshold, + GrdaWarehouse::NotificationConfiguration, ] if RailsDrivers.loaded.include?(:hud_apr) tables << HudApr::Fy2020::AprClient @@ -203,6 +206,7 @@ def self.modifier(model) GrdaWarehouse::ServiceHistoryEnrollment, ActiveStorage::Attachment, ActiveStorage::Blob, + User, ] return 'CASCADE' if cascade_models.include?(model) diff --git a/app/models/importers/hmis_auto_migrate/local.rb b/app/models/importers/hmis_auto_migrate/local.rb index 9161511ae48..8c457c855d7 100644 --- a/app/models/importers/hmis_auto_migrate/local.rb +++ b/app/models/importers/hmis_auto_migrate/local.rb @@ -8,6 +8,8 @@ require 'zip' module Importers::HmisAutoMigrate class Local < Base + attr_accessor :importer + def initialize( data_source_id:, deidentified: false, @@ -36,7 +38,8 @@ def import! ).import! end - delegate :loader_log, to: :@importer + delegate :loader_log, to: :importer + delegate :importer_log, to: :importer def pre_process compress_and_upload diff --git a/app/views/data_sources/_breadcrumbs.haml b/app/views/data_sources/_breadcrumbs.haml index 8bd8da43c28..2c8bc53bbb1 100644 --- a/app/views/data_sources/_breadcrumbs.haml +++ b/app/views/data_sources/_breadcrumbs.haml @@ -2,6 +2,6 @@ = link_to data_sources_path do « View Data Sources - if defined? data_source - = link_to data_source_path do - « View - = data_source.name + = link_to data_source_path(data_source) do + « View + = data_source.name diff --git a/app/views/data_sources/_buttons.haml b/app/views/data_sources/_buttons.haml index 6f1c02f79d4..1e490f9858f 100644 --- a/app/views/data_sources/_buttons.haml +++ b/app/views/data_sources/_buttons.haml @@ -20,6 +20,10 @@ = link_to new_data_source_hmis_import_config_path(data_source_id: @data_source), class: 'btn btn-secondary btn-sm mb-4 mr-2' do %i.icon-file-drawn Automate HMIS CSV Loads + .config-button-wrapper + = link_to data_source_import_threshold_path(data_source_id: @data_source), class: 'btn btn-secondary btn-sm mb-4 mr-2' do + %i.icon-bell + Configure Import Alerts - if can_edit_data_sources? && RailsDrivers.loaded.include?(:hmis_csv_importer) .config-button-wrapper diff --git a/app/views/import_thresholds/_table.haml b/app/views/import_thresholds/_table.haml new file mode 100644 index 00000000000..613735b0f09 --- /dev/null +++ b/app/views/import_thresholds/_table.haml @@ -0,0 +1,20 @@ +- items = import_threshold.items_for(slug) +.card + %table.table.table-striped + %thead + %tr + %th Name + %th Email + %th Receive Notifications + %th Actions + %tbody + - items.each do |item| + %tr + %td= item.user&.name + %td= item.user&.email + %td= checkmark_or_x(item.active) + %td + - items = [{ link_to: { path: edit_data_source_import_threshold_notification_configuration_path(@data_source, item, notification_slug: slug), data: { loads_in_ajax_modal: true }}, class: ['btn', 'btn-sm', 'btn-secondary'], icon: :pencil, label: 'Edit notification'}] + + - items << { link_to: { path: data_source_import_threshold_notification_configuration_path(@data_source, item, notification_slug: slug), remote: true, method: :delete, data: {confirm: "Are you sure you want to delete this notification?"}}, class: ['btn', 'btn-sm', 'btn-danger'], label: 'Delete notification'} + = action_menu_or_button(items: items) diff --git a/app/views/import_thresholds/show.haml b/app/views/import_thresholds/show.haml new file mode 100644 index 00000000000..f00235ae3a9 --- /dev/null +++ b/app/views/import_thresholds/show.haml @@ -0,0 +1,43 @@ += render 'data_sources/breadcrumbs', data_source: data_source +- content_for :title, 'Import Thresholds' +%h1= content_for :title + +%p Configure thresholds that will trigger alerts and optionally pause imports when specific criteria are met for this data source. + += simple_form_for import_threshold, url: data_source_import_threshold_path, method: :patch do |f| + .well + %h2 Changes in Record Counts + %p The following settings apply when an import significantly changes the number of rows in the data that already exists in the warehouse for the data source. This will look at the aggregate change, so if 100 rows are removed and 100 added, no change is indicated. + .row + .col + = f.input :record_count_change_percent_threshold, label: 'Record count change threshold (%)', hint: 'Imports will pause and alert if more than the chosen percentage have changed', input_html: { style: 'width: 5em' } + .col + = f.input :record_count_change_min_threshold, label: 'Minimum change count threshold', hint: 'Imports will not pause or alert below this threshold. This is designed to prevent alerting on a 30% change when only 1 of 3 rows changes.', input_html: { style: 'width: 8em' } + + = f.input :pause_on_record_count_threshold, label: 'Pause if threshold is met?', hint: 'When the above threshold is met, should the import be paused? Notifications will be sent even if the import is not paused.' + - slug = import_threshold.record_count_change_notification_event + %div{ class: slug.parameterize } + - if import_threshold.items_for(slug).any? + = render 'table', slug: slug + .none-found + = link_to new_data_source_import_threshold_notification_configuration_path(notification_slug: slug), class: 'btn btn-secondary btn-sm', data: {loads_in_ajax_modal: true} do + %i.icon-plus + Add Notification + .well + %h2 Error Counts + %p The following settings apply when an import contains a significant number of errors or validation issues. The determination of change is completely contained to the import in question, looking at the errors in a given table against the number of records being processed. + .row + .col + = f.input :error_percent_threshold, label: 'Error count threshold (%)', hint: 'Imports will pause and alert if more than the chosen percentage of rows have errors', input_html: { style: 'width: 5em' } + .col + = f.input :error_count_min_threshold, label: 'Minimum error count threshold', hint: 'Imports will not pause or alert below this threshold. This is designed to prevent alerting on a 30% error rate when only 1 of 3 rows contains an error.', input_html: { style: 'width: 8em' } + = f.input :pause_on_error_threshold, label: 'Pause if threshold is met?', hint: 'When the above threshold is met, should the import be paused? Notifications will be sent even if the import is not paused.' + - slug = import_threshold.error_count_notification_event + %div{ class: slug.parameterize } + - if import_threshold.items_for(slug).any? + = render 'table', slug: slug + .none-found + = link_to new_data_source_import_threshold_notification_configuration_path(notification_slug: slug), class: 'btn btn-secondary btn-sm', data: {loads_in_ajax_modal: true} do + %i.icon-plus + Add Notification + = f.submit 'Update Thresholds' diff --git a/app/views/notification_configurations/_form.haml b/app/views/notification_configurations/_form.haml new file mode 100644 index 00000000000..3f7a336012a --- /dev/null +++ b/app/views/notification_configurations/_form.haml @@ -0,0 +1,5 @@ +.mb-8 + = simple_form_for notification_configuration, url: @form_url, remote: ajax_modal_request? do |f| + = f.input :user_id, as: :select_two, label_method: :name_with_email, collection: notification_configuration.available_users, label: 'User', required: true + = f.input :active, as: :pretty_boolean, label: 'Receive Notifications' + = f.submit submit_label diff --git a/app/views/notification_configurations/create.js.erb b/app/views/notification_configurations/create.js.erb new file mode 100644 index 00000000000..a10e3780f71 --- /dev/null +++ b/app/views/notification_configurations/create.js.erb @@ -0,0 +1,8 @@ +<% if notification_configuration.errors.any? %> + $('.notification-configuration').html "<%=j render 'form' %>" +<% else %> + $('#ajax-modal').modal('hide') + html = "<%= j render('/import_thresholds/table', slug: params[:notification_slug]) %>" + slug = "<%= params[:notification_slug].parameterize %>" + $("." + slug).html(html) +<% end %> diff --git a/app/views/notification_configurations/destroy.js.erb b/app/views/notification_configurations/destroy.js.erb new file mode 100644 index 00000000000..800eb4cd8ba --- /dev/null +++ b/app/views/notification_configurations/destroy.js.erb @@ -0,0 +1,5 @@ +<% if notification_configuration.errors.blank? %> + html = "<%= j render('/import_thresholds/table', slug: params[:notification_slug]) %>" + slug = "<%= params[:notification_slug].parameterize %>" + $("." + slug).html(html) +<% end %> diff --git a/app/views/notification_configurations/edit.haml b/app/views/notification_configurations/edit.haml new file mode 100644 index 00000000000..2d4e90572e0 --- /dev/null +++ b/app/views/notification_configurations/edit.haml @@ -0,0 +1,3 @@ +- content_for :modal_title, 'Update Notification' +.notification-configuration + = render 'form', submit_label: 'Update Notification' diff --git a/app/views/notification_configurations/new.haml b/app/views/notification_configurations/new.haml new file mode 100644 index 00000000000..45ca07b2011 --- /dev/null +++ b/app/views/notification_configurations/new.haml @@ -0,0 +1,3 @@ +- content_for :modal_title, 'Add Notification' +.notification-configuration + = render 'form', submit_label: 'Add Notification' diff --git a/app/views/notification_configurations/update.js.erb b/app/views/notification_configurations/update.js.erb new file mode 100644 index 00000000000..a10e3780f71 --- /dev/null +++ b/app/views/notification_configurations/update.js.erb @@ -0,0 +1,8 @@ +<% if notification_configuration.errors.any? %> + $('.notification-configuration').html "<%=j render 'form' %>" +<% else %> + $('#ajax-modal').modal('hide') + html = "<%= j render('/import_thresholds/table', slug: params[:notification_slug]) %>" + slug = "<%= params[:notification_slug].parameterize %>" + $("." + slug).html(html) +<% end %> diff --git a/app/views/notify_user/import_processing.html.haml b/app/views/notify_user/import_processing.html.haml new file mode 100644 index 00000000000..beb6d146238 --- /dev/null +++ b/app/views/notify_user/import_processing.html.haml @@ -0,0 +1,15 @@ +%p + An import in the #{@data_source.name} data source has triggered notifications for the following reason(s): + +%ul + - if @paused + %li The import is paused + - if @error + %li The import crossed the configured error count threshold + - if @count + %li The import crossed the configured record count change threshold + +%p + View import: + %br + = import_url(@import) diff --git a/app/views/notify_user/import_processing.text.haml b/app/views/notify_user/import_processing.text.haml new file mode 100644 index 00000000000..2ef84995ac9 --- /dev/null +++ b/app/views/notify_user/import_processing.text.haml @@ -0,0 +1,11 @@ +An import in the #{@data_source.name} data source has triggered notifications for the following reason(s): + +- if @paused + * The import is paused +- if @error + * The import crossed the configured error count threshold +- if @count + * The import crossed the configured record count change threshold + +View import: += import_url(@import) diff --git a/config/routes.rb b/config/routes.rb index 0a12731eecb..df81626f255 100644 --- a/config/routes.rb +++ b/config/routes.rb @@ -610,6 +610,9 @@ def healthcare_routes resource :hmis_import_config do get :download end + resource :import_threshold, only: [:show, :update] do + resources :notification_configurations, only: [:new, :edit, :create, :update, :destroy] + end end resources :ad_hoc_data_sources do resources :uploads, except: [:edit], controller: 'ad_hoc_data_sources/uploads' do diff --git a/db/warehouse/migrate/20250116145506_create_notification_configuration.rb b/db/warehouse/migrate/20250116145506_create_notification_configuration.rb new file mode 100644 index 00000000000..76df8669109 --- /dev/null +++ b/db/warehouse/migrate/20250116145506_create_notification_configuration.rb @@ -0,0 +1,24 @@ +class CreateNotificationConfiguration < ActiveRecord::Migration[7.0] + def change + create_table :notification_configurations do |t| + t.references :user, null: false + t.references :source, polymorphic: true, null: false + t.string :notification_slug, null: false, description: 'Class name for notification logic' + t.boolean :active, default: :true + t.timestamps + t.timestamp :deleted_at + t.index [:user_id, :source_id, :source_type, :notification_slug], unique: true, where: 'deleted_at is NULL', name: 'nc_user_source_slug_uniq_idx' + end + create_table :import_thresholds do |t| + t.references :data_source, null: false + t.integer :record_count_change_min_threshold + t.integer :record_count_change_percent_threshold + t.integer :error_count_min_threshold + t.integer :error_percent_threshold + t.boolean :pause_on_record_count_threshold + t.boolean :pause_on_error_threshold + t.timestamps + t.timestamp :deleted_at + end + end +end diff --git a/db/warehouse/migrate/20250117174547_convert_data_source_config_to_import_threshold.rb b/db/warehouse/migrate/20250117174547_convert_data_source_config_to_import_threshold.rb new file mode 100644 index 00000000000..2338411f3ad --- /dev/null +++ b/db/warehouse/migrate/20250117174547_convert_data_source_config_to_import_threshold.rb @@ -0,0 +1,20 @@ +class ConvertDataSourceConfigToImportThreshold < ActiveRecord::Migration[7.0] + def up + GrdaWarehouse::DataSource.where(refuse_imports_with_errors: true).find_each do |ds| + # Create and update in individual queries, DS count should never be more than 50 + threshold = ds.import_threshold || GrdaWarehouse::ImportThreshold.new(data_source_id: ds.id) + threshold.error_count_min_threshold ||= 0 + threshold.error_percent_threshold ||= 0 + threshold.pause_on_error_threshold = true + threshold.save! + ds.update(refuse_imports_with_errors: false) + end + end + + def down + ds_ids = GrdaWarehouse::ImportThreshold. + where.not(error_count_min_threshold: nil, error_percent_threshold: nil, pause_on_error_threshold: true). + pluck(:data_source_id) + GrdaWarehouse::DataSource.where(id: ds_ids).update_all(refuse_imports_with_errors: true) + end +end diff --git a/db/warehouse_structure.sql b/db/warehouse_structure.sql index fadef047fc8..c2919e18607 100644 --- a/db/warehouse_structure.sql +++ b/db/warehouse_structure.sql @@ -22031,6 +22031,44 @@ CREATE SEQUENCE public.import_overrides_id_seq ALTER SEQUENCE public.import_overrides_id_seq OWNED BY public.import_overrides.id; +-- +-- Name: import_thresholds; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.import_thresholds ( + id bigint NOT NULL, + data_source_id bigint NOT NULL, + record_count_change_min_threshold integer, + record_count_change_percent_threshold integer, + error_count_min_threshold integer, + error_percent_threshold integer, + pause_on_record_count_threshold boolean, + pause_on_error_threshold boolean, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL, + deleted_at timestamp without time zone +); + + +-- +-- Name: import_thresholds_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.import_thresholds_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: import_thresholds_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.import_thresholds_id_seq OWNED BY public.import_thresholds.id; + + -- -- Name: inbound_api_configurations; Type: TABLE; Schema: public; Owner: - -- @@ -23037,6 +23075,42 @@ CREATE SEQUENCE public.non_hmis_uploads_id_seq ALTER SEQUENCE public.non_hmis_uploads_id_seq OWNED BY public.non_hmis_uploads.id; +-- +-- Name: notification_configurations; Type: TABLE; Schema: public; Owner: - +-- + +CREATE TABLE public.notification_configurations ( + id bigint NOT NULL, + user_id bigint NOT NULL, + source_type character varying NOT NULL, + source_id bigint NOT NULL, + notification_slug character varying NOT NULL, + active boolean DEFAULT true, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL, + deleted_at timestamp without time zone +); + + +-- +-- Name: notification_configurations_id_seq; Type: SEQUENCE; Schema: public; Owner: - +-- + +CREATE SEQUENCE public.notification_configurations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + + +-- +-- Name: notification_configurations_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - +-- + +ALTER SEQUENCE public.notification_configurations_id_seq OWNED BY public.notification_configurations.id; + + -- -- Name: organization_47_tes; Type: TABLE; Schema: public; Owner: - -- @@ -31241,6 +31315,13 @@ ALTER TABLE ONLY public.import_logs ALTER COLUMN id SET DEFAULT nextval('public. ALTER TABLE ONLY public.import_overrides ALTER COLUMN id SET DEFAULT nextval('public.import_overrides_id_seq'::regclass); +-- +-- Name: import_thresholds id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.import_thresholds ALTER COLUMN id SET DEFAULT nextval('public.import_thresholds_id_seq'::regclass); + + -- -- Name: inbound_api_configurations id; Type: DEFAULT; Schema: public; Owner: - -- @@ -31416,6 +31497,13 @@ ALTER TABLE ONLY public.nightly_census_by_projects ALTER COLUMN id SET DEFAULT n ALTER TABLE ONLY public.non_hmis_uploads ALTER COLUMN id SET DEFAULT nextval('public.non_hmis_uploads_id_seq'::regclass); +-- +-- Name: notification_configurations id; Type: DEFAULT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notification_configurations ALTER COLUMN id SET DEFAULT nextval('public.notification_configurations_id_seq'::regclass); + + -- -- Name: performance_measurement_goals id; Type: DEFAULT; Schema: public; Owner: - -- @@ -34946,6 +35034,14 @@ ALTER TABLE ONLY public.import_overrides ADD CONSTRAINT import_overrides_pkey PRIMARY KEY (id); +-- +-- Name: import_thresholds import_thresholds_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.import_thresholds + ADD CONSTRAINT import_thresholds_pkey PRIMARY KEY (id); + + -- -- Name: inbound_api_configurations inbound_api_configurations_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -35146,6 +35242,14 @@ ALTER TABLE ONLY public.non_hmis_uploads ADD CONSTRAINT non_hmis_uploads_pkey PRIMARY KEY (id); +-- +-- Name: notification_configurations notification_configurations_pkey; Type: CONSTRAINT; Schema: public; Owner: - +-- + +ALTER TABLE ONLY public.notification_configurations + ADD CONSTRAINT notification_configurations_pkey PRIMARY KEY (id); + + -- -- Name: performance_measurement_goals performance_measurement_goals_pkey; Type: CONSTRAINT; Schema: public; Owner: - -- @@ -57489,6 +57593,13 @@ CREATE INDEX index_import_logs_on_updated_at ON public.import_logs USING btree ( CREATE INDEX index_import_overrides_on_data_source_id ON public.import_overrides USING btree (data_source_id); +-- +-- Name: index_import_thresholds_on_data_source_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_import_thresholds_on_data_source_id ON public.import_thresholds USING btree (data_source_id); + + -- -- Name: index_inbound_api_configurations_on_hashed_api_key; Type: INDEX; Schema: public; Owner: - -- @@ -57853,6 +57964,20 @@ CREATE INDEX index_nightly_census_by_projects_on_project_id ON public.nightly_ce CREATE INDEX index_non_hmis_uploads_on_deleted_at ON public.non_hmis_uploads USING btree (deleted_at); +-- +-- Name: index_notification_configurations_on_source; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_notification_configurations_on_source ON public.notification_configurations USING btree (source_type, source_id); + + +-- +-- Name: index_notification_configurations_on_user_id; Type: INDEX; Schema: public; Owner: - +-- + +CREATE INDEX index_notification_configurations_on_user_id ON public.notification_configurations USING btree (user_id); + + -- -- Name: index_performance_metrics_clients_on_client_id; Type: INDEX; Schema: public; Owner: - -- @@ -62340,6 +62465,13 @@ CREATE UNIQUE INDEX involved_in_imports_by_id ON public.involved_in_imports USIN CREATE INDEX involved_in_imports_by_importer_log ON public.involved_in_imports USING btree (importer_log_id, record_type, record_action); +-- +-- Name: nc_user_source_slug_uniq_idx; Type: INDEX; Schema: public; Owner: - +-- + +CREATE UNIQUE INDEX nc_user_source_slug_uniq_idx ON public.notification_configurations USING btree (user_id, source_id, source_type, notification_slug) WHERE (deleted_at IS NULL); + + -- -- Name: one_entity_per_type_per_collection; Type: INDEX; Schema: public; Owner: - -- @@ -65558,6 +65690,8 @@ INSERT INTO "schema_migrations" (version) VALUES ('20241213204702'), ('20241213204837'), ('20241216184819'), -('20241217210211'); +('20241217210211'), +('20250116145506'), +('20250117174547'); diff --git a/drivers/hmis_csv_importer/app/controllers/hmis_csv_importer/importer_extensions_controller.rb b/drivers/hmis_csv_importer/app/controllers/hmis_csv_importer/importer_extensions_controller.rb index b8245d82ba9..60d58e2b524 100644 --- a/drivers/hmis_csv_importer/app/controllers/hmis_csv_importer/importer_extensions_controller.rb +++ b/drivers/hmis_csv_importer/app/controllers/hmis_csv_importer/importer_extensions_controller.rb @@ -16,7 +16,6 @@ def update config = { import_aggregators: {}, import_cleanups: {}, - refuse_imports_with_errors: params.dig(:extensions, :refuse_imports_with_errors), } allowed_extensions.each do |extension| next unless params[:extensions][extension.to_s] == '1' diff --git a/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer.rb b/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer.rb index e413b939c73..14b85698c1e 100644 --- a/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer.rb +++ b/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer.rb @@ -23,12 +23,15 @@ # # at this point, you can call any of the various import methods, usually, the last one that was attempted # imp.log_timing(:process_existing) +require 'memery' + module HmisCsvImporter::Importer class Importer include TsqlImport include NotifierConfig include HmisCsvImporter::HmisCsv include ArelHelper + include Memery attr_accessor :import, :range, :data_source, :importer_log @@ -76,15 +79,19 @@ def import!(import_log = nil) log_timing :aggregate! log_timing :cleanup_data_set! log_timing :analyze_tables + + # determine what changes will be made and make note for alerting and monitoring + log_timing :precalculate_change_counts + # Send any notifications that might be relevant to the error state of this import + notify_of_import_status + # refuse to proceed with the import if there are any errors and that setting is in effect - if should_pause? - pause_import - else - ingest! - log_timing :invalidate_aggregated_enrollments! - complete_import - post_process - end + return pause_import if should_pause? + + ingest! + log_timing :invalidate_aggregated_enrollments! + complete_import + post_process end def resume! @@ -101,22 +108,215 @@ def resume! post_process end - def should_pause? - return false unless @data_source.refuse_imports_with_errors + ## + # Determines whether the import process should be paused based on error thresholds. + # + # This method evaluates the configured settings for pausing imports due to errors and + # checks the current status of the loader log. It aggregates error counts from different + # sources and determines if they exceed the allowable threshold. + # + # The decision to pause is made based on the following conditions: + # - If the data source is configured to pause imports due to errors or record changes. + # - If the loader log status is not `'loaded'`, indicating the import has not progressed to that stage. + # - If any individual file has exceeded its error threshold, the import is paused. + # + # @return [Boolean] `true` if the import should be paused due to exceeding error thresholds, otherwise `false`. + # + memoize def should_pause? return true unless @loader_log.status == 'loaded' - loader_errors = @loader_log.summary.values.sum { |h| h['total_errors'].to_i } + return true if @data_source.ever_pause_imports_with_errors? && any_error_thresholds_met? + + @data_source.ever_pause_imports_with_record_changes? && any_record_count_thresholds_met? + end - db_errors = HmisCsvImporter::Importer::ImportError.where( + ## + # Determines if any file in the import process has exceeded its allowed error threshold. + # + # This method iterates over all import files and checks whether the number of errors + # in each file surpasses the defined threshold. If any file's error count exceeds the + # threshold set in the data source, the method returns `true`. + # + # @return [Boolean] `true` if any file has met or exceeded its error threshold, otherwise `false`. + # + memoize private def any_error_thresholds_met? + return false unless @data_source.ever_notify_for_imports? + + # counts of expected rows in each file + totals_by_filename = @loader_log.summary.map { |filename, data| [filename, data['total_lines'].to_i] }.to_h + totals_by_filename.any? do |filename, total_count| + @data_source.error_count_threshold_reached?(total_count, error_counts[filename]) + end + end + + ## + # Determines if any file in the import process has exceeded its allowed record count change threshold. + # + # This method evaluates the total number of records and absolute number of changes (additions minus removals) + # for each file. If the number of changes exceeds the configured threshold for any file, the method + # returns `true`. + # + # @return [Boolean] `true` if any file has met or exceeded its record count change threshold, otherwise `false`. + # + memoize private def any_record_count_thresholds_met? + return false unless @data_source.ever_notify_for_imports? + + change_counts.values.any? do |data| + total = data[:total_count] + changes = data[:change_count] + @data_source.record_count_threshold_reached?(total, changes) + end + end + + ## + # Collects and aggregates import errors from the loader and import processes. + # + # This method retrieves error counts from different parts of the import process: + # - Errors logged in the loader summary. + # - Errors recorded in the {HmisCsvImporter::Importer::ImportError} model. + # - Validation errors recorded in the {HmisCsvImporter::HmisCsvValidation::Base} model. + # + # It maps error counts to their respective filenames and ensures that the counts + # are aggregated correctly without raising additional errors if files are missing. + # + # @return [Hash{String => Integer}] A hash where keys are filenames and values are the total error counts for each file. + # + memoize private def error_counts + file_lookup = self.class.importable_files_map.invert + + # Serialized hash of processing data persisted on the log model + summary = @loader_log.summary + # Initialize error counts grouped by filename + counts = summary.map { |filename, data| [filename, data['total_errors'].to_i] }.to_h + + # Collect errors from ImportError model + HmisCsvImporter::Importer::ImportError.where( importer_log_id: importer_log.id, - ) + ). + group(:source_type). + count. + map do |k, count| + # convert to filename keys + filename = file_lookup[k.demodulize] + + counts[filename] += count + end - validation_errors = HmisCsvImporter::HmisCsvValidation::Base.where( + # Collect validation errors + HmisCsvImporter::HmisCsvValidation::Base.where( type: HmisCsvImporter::HmisCsvValidation::Error.subclasses.map(&:name), importer_log_id: importer_log.id, - ) + ). + group(:source_type). + count. + map do |k, count| + # convert to filename keys + filename = file_lookup[k.demodulize] + # every file _should_ be present, but let's try not to throw any errors here + next unless counts[filename] + + counts[filename] += count + end + counts + end + + ## + # Estimates the number of records that will be added and removed during the import process. + # + # This method calculates changes before ingestion to determine if the import + # exceeds configured thresholds. It compares existing records in the warehouse + # with incoming records from the import log to identify additions and removals. + # + # The results are stored in the `importer_log.summary` for each file, allowing these values + # to be used in import threshold checks before committing data changes. + # + # @return [void] + # + private def precalculate_change_counts + # This makes an estimate of the changes that will occur as it needs to be done before the + # actual processing so that it can pause the import if necessary. + # NOTE: as written this causes 2 additional, potentially slow queries + importable_files.map do |file, klass| + incoming_data = klass.incoming_data(importer_log_id: importer_log.id). + pluck(klass.hud_key).to_set + + to_add = (incoming_data - existing_hud_keys(klass)).count + # If we never delete, just pretend it will be 0 + to_remove = 0 if klass.prevent_import_deletions? + to_remove ||= (existing_hud_keys(klass) - incoming_data).count + + importer_log.summary[file]['added'] = to_add + importer_log.summary[file]['removed'] = to_remove + end + end + + ## + # Retrieves the set of existing HUD keys from the warehouse for a given class. + # + # This method queries the warehouse for records that match the data source, + # involved projects, and date range. It then extracts and returns the HUD keys + # as a set for efficient lookup and comparison. + # + # @param klass [Class] The class representing the data model being queried. + # @return [Set] A set of existing HUD keys for the given class. + # + memoize private def existing_hud_keys(klass) + klass.existing_data( + data_source_id: data_source.id, + project_ids: involved_project_ids, + date_range: date_range, + ).pluck(klass.hud_key).to_set + end - loader_errors.positive? || db_errors.count.positive? || validation_errors.count.positive? + ## + # Collects and aggregates changes in record counts during the import process. + # + # This method calculates changes in records (additions and removals) for each importable file. + # It determines the total number of changes by comparing additions and removals and associates + # the changes with the corresponding file. Additionally, it retrieves the total record count + # for of existing data that matches each file to provide more context about the scale of the changes. + # + # @return [Hash{String => Hash{Symbol => Integer}}] A hash where keys are filenames, and values + # are hashes containing: + # - `:change_count` (Integer): The absolute value of the difference between additions and removals. + # - `:total_count` (Integer): The total number of records for the corresponding file. + # + private def change_counts + importer_log.summary.map do |file, data| + klass = importable_files[file] + to_add = data['added'] + to_remove = data['removed'] + [ + file, + { + change_count: (to_add - to_remove).abs, + total_count: existing_hud_keys(klass).count, + }, + ] + end.to_h + end + + ## + # Sends notifications about the current import status. + # + # This method triggers a emails to users who are setup to receive status notifications for the data source + # with the following details: + # - Whether any error thresholds have been exceeded. + # - Whether any record count change thresholds have been exceeded. + # - Whether the import process should be paused. + # + # @return [void] + # + private def notify_of_import_status + # don't do anything if we don't have an import log to reference. + return unless @import_log + + @data_source.notify_of_import_status( + import_log_id: @import_log.id, + error_threshold_met: any_error_thresholds_met?, + record_count_threshold_met: any_record_count_thresholds_met?, + paused: should_pause?, + ) end private def analyze_tables @@ -301,6 +501,8 @@ def cleanup_data_set! # GrdaWarehouse::Tasks::ServiceHistory::Enrollment.batch_process_date_range!(range) # In here, add history_generated_on date to enrollment record def ingest! + # Reset add/remove counts used for import thresholds + reset_import_counts importer_log.update(status: :importing) # Mark everything that exists in the warehouse, that would be covered by this import # as pending deletion. We'll remove the pending where appropriate @@ -745,6 +947,25 @@ def note_processed(file, increment_by, type) importer_log.summary[file][type] += increment_by end + ## + # Resets the import counts for each importable file. + # + # This method ensures that the counts for 'added' and 'removed' records + # are set to zero before running ingest!. + # These numbers are pre-calculated and saved to determine if the import should trigger any + # notifications or should pause. They are re-added as we process batches of data and + # delete existing records, so we'll zero them out and let the re-accumulate + # + # @return [void] + # + private def reset_import_counts + importable_files.each_key do |file| + ['added', 'removed'].each do |type| + importer_log.summary[file][type] = 0 + end + end + end + def setup_summary(file) importer_log.summary ||= {} importer_log.summary[file] ||= { @@ -792,13 +1013,13 @@ def complete_import importer_log.status = :complete importer_log.completed_at = Time.current importer_log.upload_id = @upload.id if @upload.present? - importer_log.save - data_source.update(last_imported_at: Time.current) + importer_log.save! + data_source.update!(last_imported_at: Time.current) elapsed = importer_log.completed_at - @started_at Rails.logger.tagged({ task_name: 'HMIS CSV Importer', repeating_task: true, task_runtime: elapsed }) do log("Completed importing in #{elapsed_time(elapsed)} #{hash_as_log_str log_ids}. #{summary_as_log_str(importer_log.summary)}") end - @import_log&.update(importer_log: importer_log) + @import_log&.update!(importer_log: importer_log) end end diff --git a/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer_log.rb b/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer_log.rb index 5cf6186ce44..0cc3e2776fb 100644 --- a/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer_log.rb +++ b/drivers/hmis_csv_importer/app/models/hmis_csv_importer/importer/importer_log.rb @@ -34,6 +34,14 @@ def import_time end end + def post_processing_time(importer) + return unless completed_at + return unless importer&.completed_at + + seconds = ((importer.completed_at - completed_at) / 1.minute).round * 60 + "#{distance_of_time_in_words(seconds)} -#{importer.completed_at.strftime('%l:%M %P')} to #{completed_at.strftime('%l:%M %P')}" + end + def any_errors_or_validations? import_errors.exists? || import_validations.exists? end diff --git a/drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_log.haml b/drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_log.haml index 5320bd9fac0..d1e7dee6e69 100644 --- a/drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_log.haml +++ b/drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_log.haml @@ -2,12 +2,8 @@ - status = @import.import_time(details: true) - ds_name = @import.data_source.name -%h1 - = Translation.translate("Import Log from #{ds_name}") -- if @import.importer_log&.paused? - .alert.alert-warning - %i.icon-warning.mr-2 - %p #{ds_name} is currently set to prevent imports from loading if they contain any errors. This import contained at least one error, which you can review below. If the errors are unacceptable, fix them in the HMIS CSV files and re-upload. +%h1= Translation.translate("Import Log from #{ds_name}") += render 'hmis_csv_importer/import_logs/import_threshold_description' .row .col .card.mb-4 @@ -24,6 +20,9 @@ %tr %th Imported %td= import_time + %tr + %th Post-Processing + %td= @import&.importer_log&.post_processing_time(@import) %tr %th Completed In %td @@ -89,8 +88,16 @@ %tr %th File %th Rows Processed - %th Rows Added - %th Rows Removed + %th + - if @import.importer_log&.paused? + Rows to be Added + - else + Rows Added + %th + - if @import.importer_log&.paused? + Rows to be Removed + - else + Rows Removed %th Rows Updated %th Rows Unchanged %th Validation Flags diff --git a/drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_threshold_description.haml b/drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_threshold_description.haml new file mode 100644 index 00000000000..d4127758927 --- /dev/null +++ b/drivers/hmis_csv_importer/app/views/hmis_csv_importer/import_logs/_import_threshold_description.haml @@ -0,0 +1,13 @@ +- if @import.importer_log&.paused? && @import.data_source&.import_threshold + .alert.alert-warning + %i.icon-warning.mr-2 + .d-block + %p + %strong #{@import.data_source.name} + is currently set to prevent pause imports if the following conditions are met: + - threshold = @import.data_source.import_threshold + %ul + - if threshold.pause_on_error_threshold + %li If more than #{pluralize(threshold.error_count_min_threshold, 'record')} contain errors and constitute more than #{threshold.error_percent_threshold}% of the incoming records + - if threshold.pause_on_record_count_threshold + %li If more than #{pluralize(threshold.record_count_change_min_threshold, 'record')} change that would cause a change of more than #{threshold.record_count_change_percent_threshold}% of the data in any given table diff --git a/drivers/hmis_csv_importer/app/views/hmis_csv_importer/importer_extensions/edit.haml b/drivers/hmis_csv_importer/app/views/hmis_csv_importer/importer_extensions/edit.haml index 712dee57562..c0a5adc6bca 100644 --- a/drivers/hmis_csv_importer/app/views/hmis_csv_importer/importer_extensions/edit.haml +++ b/drivers/hmis_csv_importer/app/views/hmis_csv_importer/importer_extensions/edit.haml @@ -15,13 +15,6 @@ %th Extension %th Enabled? %tbody - %tr - %td All - %td Require review of imports with any errors. - - if editable - %td.text-center= f.input :refuse_imports_with_errors, as: :boolean, required: false, label: false, input_html: {checked: @data_source.refuse_imports_with_errors} - - else - %td.text-center= checkmark(@data_source.refuse_imports_with_errors) - allowed_extensions.each do |extension| %tr %td= extension.associated_model diff --git a/drivers/hmis_csv_importer/spec/models/importer/twenty_twenty_four/import_threshold_spec.rb b/drivers/hmis_csv_importer/spec/models/importer/twenty_twenty_four/import_threshold_spec.rb new file mode 100644 index 00000000000..f9410c980b9 --- /dev/null +++ b/drivers/hmis_csv_importer/spec/models/importer/twenty_twenty_four/import_threshold_spec.rb @@ -0,0 +1,73 @@ +### +# Copyright 2016 - 2025 Green River Data Analysis, LLC +# +# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md +### + +require 'rails_helper' + +RSpec.describe GrdaWarehouse::ImportThreshold, type: :model do + describe 'imports with errors' do + describe 'when import thresholds are present' do + # Using before all as imports are relatively expensive/time consuming + before(:all) do + HmisCsvImporter::Utility.clear! + GrdaWarehouse::Utility.clear! + + threshold = FactoryBot.create(:import_threshold, pause_on_error_threshold: true) + @data_source = threshold.data_source + @user = FactoryBot.create(:user) + FactoryBot.create(:notification_configuration_import_threshold, :error_count_notification_event, user: @user, source: threshold) + + # move to a time contemporaneous with the incoming data (this may not always be necessary) + travel_to Time.local(2020, 1, 1) do + @loader = import_hmis_csv_fixture( + 'drivers/hmis_csv_importer/spec/fixtures/files/twenty_twenty_four/loader_errors', + data_source: @data_source, + version: 'AutoMigrate', + run_jobs: false, + ) + end + end + + after(:all) do + HmisCsvImporter::Utility.clear! + GrdaWarehouse::Utility.clear! + end + + it 'pauses the import when there are issues' do + expect(@loader.importer_log.status).to eq('paused') + end + + it 'enqueues a message' do + expect(ActiveJob::Base.queue_adapter.enqueued_jobs.size).to be >= 1 + expect(ActiveJob::Base.queue_adapter.enqueued_jobs.map { |j| j[:job] }).to include(ActionMailer::MailDeliveryJob) + end + end + + describe 'when import thresholds are not present' do + before(:all) do + HmisCsvImporter::Utility.clear! + GrdaWarehouse::Utility.clear! + + travel_to Time.local(2020, 1, 1) do + @loader = import_hmis_csv_fixture( + 'drivers/hmis_csv_importer/spec/fixtures/files/twenty_twenty_four/loader_errors', + data_source: @data_source, + version: 'AutoMigrate', + run_jobs: false, + ) + end + end + + after(:all) do + HmisCsvImporter::Utility.clear! + GrdaWarehouse::Utility.clear! + end + + it 'does not pause the import when there are issues' do + expect(@loader.importer_log.status).to eq('complete') + end + end + end +end diff --git a/drivers/hmis_csv_twenty_twenty/app/controllers/hmis_csv_twenty_twenty/importer_extensions_controller.rb b/drivers/hmis_csv_twenty_twenty/app/controllers/hmis_csv_twenty_twenty/importer_extensions_controller.rb index 489a74e9630..ed72445a024 100644 --- a/drivers/hmis_csv_twenty_twenty/app/controllers/hmis_csv_twenty_twenty/importer_extensions_controller.rb +++ b/drivers/hmis_csv_twenty_twenty/app/controllers/hmis_csv_twenty_twenty/importer_extensions_controller.rb @@ -16,7 +16,6 @@ def update config = { import_aggregators: {}, import_cleanups: {}, - refuse_imports_with_errors: params.dig(:extensions, :refuse_imports_with_errors), } allowed_extensions.each do |extension| next unless params[:extensions][extension.to_s] == '1' diff --git a/drivers/hmis_csv_twenty_twenty/app/models/hmis_csv_twenty_twenty/importer/importer.rb b/drivers/hmis_csv_twenty_twenty/app/models/hmis_csv_twenty_twenty/importer/importer.rb index 340373d9be1..c2665224652 100644 --- a/drivers/hmis_csv_twenty_twenty/app/models/hmis_csv_twenty_twenty/importer/importer.rb +++ b/drivers/hmis_csv_twenty_twenty/app/models/hmis_csv_twenty_twenty/importer/importer.rb @@ -77,26 +77,27 @@ def self.module_scope end # Needs to return an import_log instance - def import!(import_log = nil) + def import!(import_log = nil) # rubocop:disable Lint/UnusedMethodArgument + raise 'HmisCsvTwentyTwenty::Importer::Importer should no longer be used to import data, it has been superceded' # log that we're waiting, but then continue on. - already_running_for_data_source? - - GrdaWarehouse::DataSource.with_advisory_lock("hud_import_#{data_source.id}") do - start_import - @import_log = import_log - log_timing :pre_process! - log_timing :validate_data_set! - log_timing :aggregate! - log_timing :cleanup_data_set! - # refuse to proceed with the import if there are any errors and that setting is in effect - if should_pause? - pause_import - else - ingest! - log_timing :invalidate_aggregated_enrollments! - complete_import - end - end + # already_running_for_data_source? + + # GrdaWarehouse::DataSource.with_advisory_lock("hud_import_#{data_source.id}") do + # start_import + # @import_log = import_log + # log_timing :pre_process! + # log_timing :validate_data_set! + # log_timing :aggregate! + # log_timing :cleanup_data_set! + # # refuse to proceed with the import if there are any errors and that setting is in effect + # if should_pause? + # pause_import + # else + # ingest! + # log_timing :invalidate_aggregated_enrollments! + # complete_import + # end + # end end def resume! @@ -113,8 +114,6 @@ def resume! end def should_pause? - return false unless @data_source.refuse_imports_with_errors - db_errors = HmisCsvTwentyTwenty::Importer::ImportError.where( importer_log_id: importer_log.id, ) diff --git a/drivers/hmis_csv_twenty_twenty/app/views/hmis_csv_twenty_twenty/importer_extensions/edit.haml b/drivers/hmis_csv_twenty_twenty/app/views/hmis_csv_twenty_twenty/importer_extensions/edit.haml index 746f2286000..7c901b9f9dc 100644 --- a/drivers/hmis_csv_twenty_twenty/app/views/hmis_csv_twenty_twenty/importer_extensions/edit.haml +++ b/drivers/hmis_csv_twenty_twenty/app/views/hmis_csv_twenty_twenty/importer_extensions/edit.haml @@ -16,13 +16,6 @@ %th Extension %th Enabled? %tbody - %tr - %td All - %td Require review of imports with any errors. - - if editable - %td.text-center= f.input :refuse_imports_with_errors, as: :boolean, required: false, label: false, input_html: {checked: @data_source.refuse_imports_with_errors} - - else - %td.text-center= checkmark(@data_source.refuse_imports_with_errors) - allowed_extensions.each do |extension| %tr %td= extension.associated_model diff --git a/drivers/hmis_csv_twenty_twenty/spec/models/hmis_auto_detect_spec.rb b/drivers/hmis_csv_twenty_twenty/spec/models/hmis_auto_detect_spec.rb deleted file mode 100644 index c91db1518cc..00000000000 --- a/drivers/hmis_csv_twenty_twenty/spec/models/hmis_auto_detect_spec.rb +++ /dev/null @@ -1,56 +0,0 @@ -### -# Copyright 2016 - 2025 Green River Data Analysis, LLC -# -# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md -### - -require 'rails_helper' - -RSpec.describe HmisCsvTwentyTwenty, type: :model do - it 'supports .matches API' do - [ - 'drivers/hmis_csv_twenty_twenty/spec/fixtures/files/enrollment_test_files/source', - ].each do |path| - assert HmisCsvTwentyTwenty.matches(path) - end - end - - it 'supports .import! API' do - [ - 'drivers/hmis_csv_twenty_twenty/spec/fixtures/files/enrollment_test_files/source', - ].each do |path| - file = Tempfile.new('foo') - data_source = GrdaWarehouse::DataSource.create!( - name: 'Green River', - short_name: 'GR', - source_type: :sftp, - ) - tmp_path = path.gsub('source', data_source.id.to_s) - FileUtils.cp_r(path, tmp_path) - upload = GrdaWarehouse::Upload.new( - user_id: User.first, - data_source_id: data_source.id, - percent_complete: 0.0, - file: 'See S3', - ) - - upload.hmis_zip.attach( - io: file, - filename: file.path, - ) - upload.save! - log = HmisCsvTwentyTwenty.import!( - tmp_path, - data_source.id, - upload, - deidentified: false, - ) - assert log - ensure - file&.unlink - log&.destroy - upload&.destroy - data_source&.destroy - end - end -end diff --git a/drivers/hmis_csv_twenty_twenty/spec/models/prepend_organization_ids_spec.rb b/drivers/hmis_csv_twenty_twenty/spec/models/prepend_organization_ids_spec.rb deleted file mode 100644 index 77179d75501..00000000000 --- a/drivers/hmis_csv_twenty_twenty/spec/models/prepend_organization_ids_spec.rb +++ /dev/null @@ -1,76 +0,0 @@ -### -# Copyright 2016 - 2025 Green River Data Analysis, LLC -# -# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md -### - -require 'rails_helper' - -RSpec.describe 'Prepend Organization IDs', type: :model do - describe 'without cleanup' do - before(:all) do - setup(with_cleanup: false) - end - - after(:all) do - HmisCsvTwentyTwenty::Utility.clear! - GrdaWarehouse::Utility.clear! - - FileUtils.rm_rf(@import_path) - end - - it 'Has 1 organization' do - expect(GrdaWarehouse::Hud::Organization.count).to eq(1) - end - - it 'Organization Name is Test Organization' do - expect(GrdaWarehouse::Hud::Organization.first.OrganizationName).to eq('Test Org') - end - end - - describe 'with cleanup' do - before(:all) do - setup(with_cleanup: true) - end - - after(:all) do - HmisCsvTwentyTwenty::Utility.clear! - GrdaWarehouse::Utility.clear! - - FileUtils.rm_rf(@import_path) - end - - it 'Has 1 organization' do - expect(GrdaWarehouse::Hud::Organization.count).to eq(1) - end - - it 'has Organization ID prepended' do - expect(GrdaWarehouse::Hud::Organization.first.OrganizationName).to eq('(ORG-ID) Test Org') - end - end - - def setup(with_cleanup:) - GrdaWarehouse::Utility.clear! - HmisCsvTwentyTwenty::Utility.clear! - - file_path = 'drivers/hmis_csv_twenty_twenty/spec/fixtures/files/cleanup_move_ins' - - @data_source = if with_cleanup - create(:prepend_organization_ids) - else - create(:dont_cleanup_ds) - end - - source_file_path = File.join(file_path, 'source') - @import_path = File.join(file_path, @data_source.id.to_s) - FileUtils.cp_r(source_file_path, @import_path) - - @loader = HmisCsvTwentyTwenty::Loader::Loader.new( - file_path: @import_path, - data_source_id: @data_source.id, - remove_files: false, - ) - @loader.import! - Delayed::Worker.new.work_off(2) - end -end diff --git a/spec/factories/grda_warehouse/import_thresholds.rb b/spec/factories/grda_warehouse/import_thresholds.rb new file mode 100644 index 00000000000..b5bbcd4418d --- /dev/null +++ b/spec/factories/grda_warehouse/import_thresholds.rb @@ -0,0 +1,9 @@ +FactoryBot.define do + factory :import_threshold, class: 'GrdaWarehouse::ImportThreshold' do + association :data_source, factory: :source_data_source + record_count_change_min_threshold { 0 } + record_count_change_percent_threshold { 0 } + error_count_min_threshold { 0 } + error_percent_threshold { 0 } + end +end diff --git a/spec/factories/grda_warehouse/notification_configurations.rb b/spec/factories/grda_warehouse/notification_configurations.rb new file mode 100644 index 00000000000..fc5386d4b71 --- /dev/null +++ b/spec/factories/grda_warehouse/notification_configurations.rb @@ -0,0 +1,13 @@ +FactoryBot.define do + factory :notification_configuration_import_threshold, class: 'GrdaWarehouse::NotificationConfiguration' do + association :source, factory: :import_threshold + + trait :error_count_notification_event do + notification_slug { 'error_threshold_exceeded' } + end + + trait :record_count_change_notification_event do + notification_slug { 'count_threshold_exceeded' } + end + end +end diff --git a/spec/models/application_record_spec.rb b/spec/models/application_record_spec.rb new file mode 100644 index 00000000000..a540319f516 --- /dev/null +++ b/spec/models/application_record_spec.rb @@ -0,0 +1,33 @@ +require 'rails_helper' + +RSpec.describe ApplicationRecord, type: :model do + let!(:record) { create(:role) } + let!(:grda_warehouse_record) { create(:grda_warehouse_hud_client) } + + describe '.find_safely' do + context 'when given a valid integer ID' do + it 'returns the record' do + expect(Role.find_safely(record.id)).to eq(record) # integer lookup + expect(Role.find_safely(record.id.to_s)).to eq(record) # string lookup + expect(GrdaWarehouse::Hud::Client.find_safely(grda_warehouse_record.id)).to eq(grda_warehouse_record) # integer lookup + expect(GrdaWarehouse::Hud::Client.find_safely(grda_warehouse_record.id.to_s)).to eq(grda_warehouse_record) # string lookup + end + end + + context 'when given an invalid non-integer ID' do + it 'raises ActiveRecord::RecordNotFound' do + expect { Role.find_safely('invalid_id') }.to raise_error(ActiveRecord::RecordNotFound) + expect { Role.find_safely(nil) }.to raise_error(ActiveRecord::RecordNotFound) + expect { GrdaWarehouse::Hud::Client.find_safely('invalid_id') }.to raise_error(ActiveRecord::RecordNotFound) + expect { GrdaWarehouse::Hud::Client.find_safely(nil) }.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context 'when given an ID that does not exist' do + it 'raises ActiveRecord::RecordNotFound' do + expect { Role.find_safely(-999999) }.to raise_error(ActiveRecord::RecordNotFound) + expect { GrdaWarehouse::Hud::Client.find_safely(-999999) }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end +end diff --git a/spec/support/hmis_csv_fixtures.rb b/spec/support/hmis_csv_fixtures.rb index 5da75c8ff03..9dcc19ce3d8 100644 --- a/spec/support/hmis_csv_fixtures.rb +++ b/spec/support/hmis_csv_fixtures.rb @@ -23,35 +23,37 @@ def import_hmis_csv_fixture( @data_source = data_source source_file_path = File.join(file_path, 'source') + begin + # duplicate the fixture folder since some + # importers expect data in "#{file_path}/#{data_source_id}" + # and to be able to tamper with its contents + # TODO: fix importers to avoid mutating the source! + tmp_path = File.join(file_path, data_source.id.to_s) + FileUtils.cp_r(source_file_path, tmp_path) - # duplicate the fixture folder since some - # importers expect data in "#{file_path}/#{data_source_id}" - # and to be able to tamper with its contents - # TODO: fix importers to avoid mutating the source! - tmp_path = File.join(file_path, data_source.id.to_s) - FileUtils.cp_r(source_file_path, tmp_path) + importer = if version == '2020' + HmisCsvTwentyTwenty::Loader::Loader.new( + file_path: tmp_path, + data_source_id: data_source.id, + deidentified: deidentified, + ) + elsif version == 'AutoMigrate' + Importers::HmisAutoMigrate::Local.new( + file_path: tmp_path, + data_source_id: data_source.id, + deidentified: deidentified, + allowed_projects: allowed_projects, + project_cleanup: false, + ) + else + raise "Unsupported CSV version #{version}" + end - importer = if version == '2020' - HmisCsvTwentyTwenty::Loader::Loader.new( - file_path: tmp_path, - data_source_id: data_source.id, - deidentified: deidentified, - ) - elsif version == 'AutoMigrate' - Importers::HmisAutoMigrate::Local.new( - file_path: tmp_path, - data_source_id: data_source.id, - deidentified: deidentified, - allowed_projects: allowed_projects, - project_cleanup: false, - ) - else - raise "Unsupported CSV version #{version}" + # puts "Starting import: #{Time.now}" + importer.import! + ensure + FileUtils.rm_rf(tmp_path) if tmp_path end - - # puts "Starting import: #{Time.now}" - importer.import! - FileUtils.rm_rf(tmp_path) if tmp_path process_imported_fixtures(user: user, skip_location_cleanup: skip_location_cleanup) if run_jobs Rails.cache.delete([user, 'access_groups']) # These are cached in project.rb etc for one minute, which is too long for tests From 6c6ac5f6f6d35179938d75d6481fb4ca44a0dd34 Mon Sep 17 00:00:00 2001 From: Elliot Anders Date: Sat, 1 Feb 2025 08:45:51 -0500 Subject: [PATCH 02/18] Fix :true -> true --- .../migrate/20250116145506_create_notification_configuration.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/warehouse/migrate/20250116145506_create_notification_configuration.rb b/db/warehouse/migrate/20250116145506_create_notification_configuration.rb index 76df8669109..bbe158685e5 100644 --- a/db/warehouse/migrate/20250116145506_create_notification_configuration.rb +++ b/db/warehouse/migrate/20250116145506_create_notification_configuration.rb @@ -4,7 +4,7 @@ def change t.references :user, null: false t.references :source, polymorphic: true, null: false t.string :notification_slug, null: false, description: 'Class name for notification logic' - t.boolean :active, default: :true + t.boolean :active, default: true t.timestamps t.timestamp :deleted_at t.index [:user_id, :source_id, :source_type, :notification_slug], unique: true, where: 'deleted_at is NULL', name: 'nc_user_source_slug_uniq_idx' From 99a831ef7afb87debc50d09bcf501fe35b5f2daf Mon Sep 17 00:00:00 2001 From: Gig Date: Sun, 2 Feb 2025 08:09:08 -0500 Subject: [PATCH 03/18] Add bounds to year entered service (#5098) --- .../hmis/form/numeric_input_validator.rb | 2 +- .../fragments/client_demographics.json | 30 +++++++++++++++++-- 2 files changed, 29 insertions(+), 3 deletions(-) diff --git a/drivers/hmis/app/models/hmis/form/numeric_input_validator.rb b/drivers/hmis/app/models/hmis/form/numeric_input_validator.rb index f8aaaa45509..b5d9a2026c0 100644 --- a/drivers/hmis/app/models/hmis/form/numeric_input_validator.rb +++ b/drivers/hmis/app/models/hmis/form/numeric_input_validator.rb @@ -38,7 +38,7 @@ def validate_bounds(item, value) return [] if item.bounds.blank? item.bounds.each_with_object([]) do |bound, errors| - next unless bound.severity == 'error' + next if bound.severity == 'warning' # bound.value_number can be nil in the case where the bound is against a local constant or another question next unless bound.value_number diff --git a/drivers/hmis/lib/form_data/default/fragments/client_demographics.json b/drivers/hmis/lib/form_data/default/fragments/client_demographics.json index 8ae1d637cfc..3225d07f50c 100644 --- a/drivers/hmis/lib/form_data/default/fragments/client_demographics.json +++ b/drivers/hmis/lib/form_data/default/fragments/client_demographics.json @@ -193,7 +193,19 @@ "mapping": { "record_type": "CLIENT", "field_name": "yearEnteredService" - } + }, + "bounds": [ + { + "id": "min-year", + "type": "MIN", + "value_number": 1900 + }, + { + "id": "max-year", + "type": "MAX", + "value_number": 2080 + } + ] }, { "type": "INTEGER", @@ -203,7 +215,21 @@ "mapping": { "record_type": "CLIENT", "field_name": "yearSeparated" - } + }, + "bounds": [ + { + "id": "min-year", + "type": "MIN", + "severity": "error", + "value_number": 1900 + }, + { + "id": "max-year", + "type": "MAX", + "severity": "error", + "value_number": 2080 + } + ] }, { "type": "GROUP", From a65d9cddea0badf29b3fb1bc96ab025e4e18472c Mon Sep 17 00:00:00 2001 From: Gig Date: Sun, 2 Feb 2025 08:19:44 -0500 Subject: [PATCH 04/18] Remove duplicates from AC Export Pathways file (#5100) --- .../ac_hmis/exporters/pathways_export.rb | 30 +++++++---- .../ac_hmis/exporters/pathways_export_spec.rb | 51 ++++++++++++++++++- 2 files changed, 70 insertions(+), 11 deletions(-) diff --git a/drivers/hmis_external_apis/app/models/hmis_external_apis/ac_hmis/exporters/pathways_export.rb b/drivers/hmis_external_apis/app/models/hmis_external_apis/ac_hmis/exporters/pathways_export.rb index 1326117d9c7..c1e9fa478b1 100644 --- a/drivers/hmis_external_apis/app/models/hmis_external_apis/ac_hmis/exporters/pathways_export.rb +++ b/drivers/hmis_external_apis/app/models/hmis_external_apis/ac_hmis/exporters/pathways_export.rb @@ -7,6 +7,7 @@ module HmisExternalApis::AcHmis::Exporters class PathwaysExport include ::HmisExternalApis::AcHmis::Exporters::CsvExporter + include ::Hmis::Concerns::HmisArelHelper PATHWAY_KEYS = [ 'client_pathway_1', @@ -24,16 +25,20 @@ def run! Rails.logger.info 'Generating content of pathways export' write_row(columns) - total = clients_with_pathways.count + total = pathway_client_warehouse_id_to_client_ids.count Rails.logger.info "There are #{total} clients with pathways to export" - clients_with_pathways.find_each.with_index do |client, i| - Rails.logger.info "Processed #{i} of #{total}" if (i % 1000).zero? - next unless client.warehouse_id.present? + # Iterate through each Destination Client that has Source Client(s) with any Pathway values + pathway_client_warehouse_id_to_client_ids.each do |warehouse_id, client_ids| + # Find the pathway CDE that was most recently updated for this destination client + most_recent_pathway_cde = client_ids.map { |id| pathways_by_client_id[id] }.flatten.max_by(&:date_updated) + + # Collect all pathways for the source client that most recently had any pathway updated. + # This makes it so that we don't mix-and-match pathways values from different source clients. + pathways = pathways_by_client_id[most_recent_pathway_cde.owner_id] - pathways = pathways_by_client_id[client.id] values = [ - client.warehouse_id, # Matches PersonalID in HMIS CSV export + warehouse_id, # Matches PersonalID in HMIS CSV export find_pathway(pathways, 'client_pathway_1'), find_pathway(pathways, 'client_pathway_1_date'), find_pathway(pathways, 'client_pathway_1_narrative'), @@ -88,17 +93,24 @@ def columns end private def pathway_cded_key_to_id - @pathway_cded_key_to_id ||= Hmis::Hud::CustomDataElementDefinition.where(key: PATHWAY_KEYS).pluck(:key, :id).to_h + @pathway_cded_key_to_id ||= Hmis::Hud::CustomDataElementDefinition.where(key: PATHWAY_KEYS, owner_type: 'Hmis::Hud::Client').pluck(:key, :id).to_h end private def pathways_by_client_id @pathways_by_client_id ||= Hmis::Hud::CustomDataElement. where(data_element_definition_id: pathway_cded_key_to_id.values). # All Pathway-related definitions + where(owner_type: 'Hmis::Hud::Client'). group_by(&:owner_id) # By Client ID end - private def clients_with_pathways - @clients_with_pathways ||= Hmis::Hud::Client.where(id: pathways_by_client_id.keys).preload(:warehouse_client_source) + # { => [ ] } + private def pathway_client_warehouse_id_to_client_ids + @pathway_client_warehouse_id_to_client_ids ||= Hmis::WarehouseClient.joins(:source). + where(data_source_id: data_source.id). + where(source_id: pathways_by_client_id.keys). # Only include clients that have Pathways + group(:destination_id). + select('"destination_id", array_agg("source_id") as source_ids'). + map { |r| [r.destination_id, r.source_ids] }.to_h end end end diff --git a/drivers/hmis_external_apis/spec/models/hmis_external_apis/ac_hmis/exporters/pathways_export_spec.rb b/drivers/hmis_external_apis/spec/models/hmis_external_apis/ac_hmis/exporters/pathways_export_spec.rb index cce259628b6..8584fb8e0d3 100644 --- a/drivers/hmis_external_apis/spec/models/hmis_external_apis/ac_hmis/exporters/pathways_export_spec.rb +++ b/drivers/hmis_external_apis/spec/models/hmis_external_apis/ac_hmis/exporters/pathways_export_spec.rb @@ -43,12 +43,12 @@ it 'collects clients with pathways' do create(:hmis_custom_data_element, owner: client, data_element_definition: pathway_definitions['client_pathway_1']) subject.run! - expect(subject.send(:clients_with_pathways)).to contain_exactly(client) + expect(subject.send(:pathway_client_warehouse_id_to_client_ids)).to contain_exactly([client.warehouse_id, [client.id]]) end it 'doesnt fail if no clients have pathways' do subject.run! - expect(subject.send(:clients_with_pathways).length).to eq(0) + expect(subject.send(:pathway_client_warehouse_id_to_client_ids).length).to eq(0) end it 'makes a csv' do @@ -74,4 +74,51 @@ expect(result.first['Pathway3_Narrative']).to be_nil expect(result.first['Pathway3_DateUpdated']).to be_nil end + + context 'when source client does not have a destination client yet' do + let!(:client) { create(:hmis_hud_client, data_source: ds) } + let!(:pathway1) { create(:hmis_custom_data_element, value_string: 'value', owner: client, data_element_definition: pathway_definitions['client_pathway_1']) } + + it 'does not export the client' do + subject.run! + result = CSV.parse(output, headers: true) + expect(result.length).to eq(0) + end + end + + context 'when there are multiple pathway clients with the same destination id' do + # Two Pathway 1 CDEs, with different source clients but same dest client + let!(:pathway1) { create(:hmis_custom_data_element, value_string: 'retained', owner: client, data_element_definition: pathway_definitions['client_pathway_1']) } + let!(:pathway1_dup) { create(:hmis_custom_data_element, value_string: 'dropped', date_updated: 1.week.ago, owner: client2, data_element_definition: pathway_definitions['client_pathway_1']) } + # One Pathway 2, dropped because it will only export `client` + let!(:pathway2) { create(:hmis_custom_data_element, value_string: 'dropped', owner: client2, data_element_definition: pathway_definitions['client_pathway_2']) } + # Two Pathway 3 CDEs, with different source clients but same dest client + let!(:pathway3) { create(:hmis_custom_data_element, value_string: 'retained', owner: client, data_element_definition: pathway_definitions['client_pathway_3']) } + let!(:pathway3_dup) { create(:hmis_custom_data_element, value_string: 'dropped', owner: client3, date_updated: 1.week.ago, data_element_definition: pathway_definitions['client_pathway_3']) } + + before(:each) do + warehouse_id = client.warehouse_id + client2.warehouse_client_source.update!(destination_id: warehouse_id) + client3.warehouse_client_source.update!(destination_id: warehouse_id) + end + + it 'maps pathway_client_warehouse_id_to_client_ids correctly' do + expect(subject.send(:pathway_client_warehouse_id_to_client_ids)).to contain_exactly( + [client.warehouse_id, containing_exactly(client.id, client2.id, client3.id)], + ) + end + + it 'only exports one row for all destination client pathways, choosing recently updated value when there are multiples' do + subject.run! + result = CSV.parse(output, headers: true) + + expect(result.length).to eq(1) + expect(result.first['PersonalID']).to eq(client.warehouse_id.to_s) + expect(result.first['Pathway1']).to eq(pathway1.value_string) + expect(result.first['Pathway1_DateUpdated']).to eq(pathway1.date_updated.strftime('%Y-%m-%d %H:%M:%S')) + expect(result.first['Pathway2']).to eq(nil) # pathway2 not included because it's tied to client2 + expect(result.first['Pathway3']).to eq(pathway3.value_string) + expect(result.first['Pathway3_DateUpdated']).to eq(pathway3.date_updated.strftime('%Y-%m-%d %H:%M:%S')) + end + end end From 1ad5529368f797980747768a5272efc158086b2f Mon Sep 17 00:00:00 2001 From: Gig Date: Mon, 3 Feb 2025 13:17:15 -0500 Subject: [PATCH 05/18] Fix N+1 on household reminders query (#5102) --- .../graphql/types/hmis_schema/enrollment.rb | 8 ++++++++ .../app/graphql/types/hmis_schema/reminder.rb | 1 + .../hmis/spec/requests/hmis/reminder_spec.rb | 18 +++++++++++++----- 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/drivers/hmis/app/graphql/types/hmis_schema/enrollment.rb b/drivers/hmis/app/graphql/types/hmis_schema/enrollment.rb index 151954015a9..0f8285de852 100644 --- a/drivers/hmis/app/graphql/types/hmis_schema/enrollment.rb +++ b/drivers/hmis/app/graphql/types/hmis_schema/enrollment.rb @@ -441,5 +441,13 @@ def num_units_assigned_to_household def move_in_addresses load_ar_association(object, :move_in_addresses) end + + def intake_assessment + load_ar_association(object, :intake_assessment) + end + + def exit_assessment + load_ar_association(object, :exit_assessment) + end end end diff --git a/drivers/hmis/app/graphql/types/hmis_schema/reminder.rb b/drivers/hmis/app/graphql/types/hmis_schema/reminder.rb index 568a8ac598c..b0a6122b47a 100644 --- a/drivers/hmis/app/graphql/types/hmis_schema/reminder.rb +++ b/drivers/hmis/app/graphql/types/hmis_schema/reminder.rb @@ -18,6 +18,7 @@ class HmisSchema::Reminder < Types::BaseObject field :assessment_id, ID, null: true, description: 'Relevant existing assessment, if any' def client + # note: client and enrollment are preloaded by ReminderGenerator object.enrollment.client end diff --git a/drivers/hmis/spec/requests/hmis/reminder_spec.rb b/drivers/hmis/spec/requests/hmis/reminder_spec.rb index 0a20511f8e3..e3d01276fc6 100644 --- a/drivers/hmis/spec/requests/hmis/reminder_spec.rb +++ b/drivers/hmis/spec/requests/hmis/reminder_spec.rb @@ -19,12 +19,14 @@ hmis_login(user) end + let(:household_size) { 5 } let!(:enrollment) do client = create :hmis_hud_client_complete, data_source: ds1, user: u1 hoh = create :hmis_hud_enrollment, data_source: ds1, project: p1, client: client, user: u1, relationship_to_hoh: 1 - 3.times do + # Create N enrollments. Each will be missing an intake, so should have 1 reminder. + (household_size - 1).times do member = create :hmis_hud_client_complete, data_source: ds1, user: u1 - create :hmis_hud_enrollment, data_source: ds1, project: p1, client: member, user: u1, relationship_to_hoh: 99 + create(:hmis_hud_enrollment, household_id: hoh.household_id, data_source: ds1, project: p1, client: member, relationship_to_hoh: 6) end hoh end @@ -42,6 +44,12 @@ overdue enrollment { id + intakeAssessment { + id + } + exitAssessment { + id + } } client { id @@ -60,14 +68,14 @@ it 'minimizes n+1 queries' do expect do _, result = post_graphql(**variables) { query } - expect(result.dig('data', 'enrollment', 'reminders').size).to eq(1) - end.to make_database_queries(count: 10..30) + expect(result.dig('data', 'enrollment', 'reminders').size).to eq(household_size) + end.to make_database_queries(count: 10..35) end it 'is responsive' do expect do _, result = post_graphql(**variables) { query } - expect(result.dig('data', 'enrollment', 'reminders').size).to eq(1) + expect(result.dig('data', 'enrollment', 'reminders').size).to eq(household_size) end.to perform_under(300).ms end end From b3e1a4e4c0523c2e5ae7fa3377c0e07ee35cd26f Mon Sep 17 00:00:00 2001 From: Dave G <149399758+dtgreiner@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:22:23 -0500 Subject: [PATCH 06/18] max threshold for source clients (#5101) * add max number or source clients * comments & cleanup * additional test cleanup & comments * Update spec/models/grda_warehouse/tasks/identify_duplicates_spec.rb Co-authored-by: Elliot * clear out old artifacts * fix failing tests --------- Co-authored-by: Elliot --- .../tasks/identify_duplicates.rb | 18 +++++- .../tasks/identify_duplicates_spec.rb | 61 +++++++++++++++++++ 2 files changed, 76 insertions(+), 3 deletions(-) diff --git a/app/models/grda_warehouse/tasks/identify_duplicates.rb b/app/models/grda_warehouse/tasks/identify_duplicates.rb index efad9f58408..b4e888601b4 100644 --- a/app/models/grda_warehouse/tasks/identify_duplicates.rb +++ b/app/models/grda_warehouse/tasks/identify_duplicates.rb @@ -9,6 +9,8 @@ class IdentifyDuplicates include ArelHelper include NotifierConfig + MAX_SOURCE_CLIENTS = 50 + def initialize(run_post_processing: true) setup_notifier('IdentifyDuplicates') @run_post_processing = run_post_processing @@ -144,8 +146,10 @@ def match_existing! # This has already been fully merged next if source_id == destination_id - destination = client_destinations.find(destination_id) - source = client_destinations.find(source_id) + destination = client_destinations.find_by(id: destination_id) + source = client_destinations.find_by(id: source_id) + next unless destination.present? && source.present? + begin destination.merge_from(source, reviewed_by: user, reviewed_at: DateTime.current) rescue Exception => e @@ -241,9 +245,17 @@ def match_existing! ) end + # Destination clients are limited to MAX_SOURCE_CLIENTS number of source clients. This is to prevent runaway duplicate merges. + # This will return the ids of any destination clients that is at or beyond this threshold so they can be filtered out of future matching. + # We are using unmatchable destinations here to more easily include destination clients who do not have a warehouse client record. + # This should also be a smaller list than the full list of matchable destination ids. + private def unmatchable_destination_ids + GrdaWarehouse::WarehouseClient.group(:destination_id).having('count(*) >= ?', MAX_SOURCE_CLIENTS).select(:destination_id) + end + # fetch a list of existing clients from the DND Warehouse DataSource (current destinations) private def client_destinations - GrdaWarehouse::Hud::Client.destination + GrdaWarehouse::Hud::Client.destination.where.not(id: unmatchable_destination_ids) end # Look for really clear matches (2 of the following 3 should be good): diff --git a/spec/models/grda_warehouse/tasks/identify_duplicates_spec.rb b/spec/models/grda_warehouse/tasks/identify_duplicates_spec.rb index d93551d7cd7..f27ea1da58d 100644 --- a/spec/models/grda_warehouse/tasks/identify_duplicates_spec.rb +++ b/spec/models/grda_warehouse/tasks/identify_duplicates_spec.rb @@ -127,6 +127,67 @@ end end end + + describe 'source client threshold' do + before(:each) do + # We want to know the PPI data for this client, so set it specifically + client_in_source.update(first_name: 'A', last_name: 'Client', dob: '2000-01-01', ssn: 'XXXXX1234') + # Clear out the existing destination client, this will be generated for `client_in_source` later in the test + client_in_destination.destroy + end + it 'never creates more than the maximum number of source clients' do + number_sample_clients = 125 + + # Generating n unique clients that won't match the existing client or each other. + # We are generating `number_sample_clients` - 1 because we are using the existing client (`client_in_source`) + # as a baseline. This will bring our total number of clients to `number_sample_clients`. + (1..(number_sample_clients - 1)).each do |n| + client = create :grda_warehouse_hud_client, data_source: source_data_source + date = Date.new(2000, 1, 1) + n.days - n.months + str_n = n.to_s.rjust(4, '0') + ssn = "#{str_n.last(3)}#{str_n.last(2)}#{str_n.last(4)}" + client.update(first_name: "client_#{n}", last_name: 'Test', dob: date, ssn: ssn) + end + + GrdaWarehouse::Tasks::IdentifyDuplicates.new.run! + GrdaWarehouse::Tasks::IdentifyDuplicates.new.match_existing! + + # Test that each unique client received a destination client + expect(GrdaWarehouse::Hud::Client.destination.count).to eq(number_sample_clients) + expect(GrdaWarehouse::WarehouseClient.count).to eq(number_sample_clients) + + # Update the n unique clients so that they all match the baseline client. + (1..(number_sample_clients - 1)).each do |n| + str_n = n.to_s.rjust(4, '0') + ssn = "#{str_n.last(3)}#{str_n.last(2)}#{str_n.last(4)}" + client = GrdaWarehouse::Hud::Client.source.find_by(SSN: ssn) + client.update(first_name: 'A', last_name: 'Client', dob: '2000-01-01', ssn: 'XXXXX1234') + end + + GrdaWarehouse::Tasks::IdentifyDuplicates.new.run! + GrdaWarehouse::Tasks::IdentifyDuplicates.new.match_existing! + + # # The code below will calculate the number of expected destination clients in the case that MAX_SOURCE_CLIENTS changes enought to affect this number + # expected_number_destination_clients = ((number_sample_clients * 1.0) / GrdaWarehouse::Tasks::IdentifyDuplicates::MAX_SOURCE_CLIENTS).ceil + + # With MAX_SOURCE_CLIENTS set to 50, we are expecting 3 destiantion clients. 2 with 50 source clients and 1 with 25 source clients. + # We are setting this specifically instead of using the calculated number in case `MAX_SOURCE_CLIENTS` gets set to a number larger than + # `number_sample_clients`. If that happened, we wouldn't be reaching the threshold for the numebr of source clients that we are testing. + expected_number_destination_clients = 3 + + destination_clients = GrdaWarehouse::Hud::Client.destination.to_a + + expect(destination_clients.count).to eq(expected_number_destination_clients) + expect(GrdaWarehouse::WarehouseClient.count).to eq(number_sample_clients) + + # Pop the last client off of the array. This client will have less than the maximum number of source clients. + last_client = destination_clients.pop + destination_clients.each do |client| + expect(client.source_clients.count).to eq(GrdaWarehouse::Tasks::IdentifyDuplicates::MAX_SOURCE_CLIENTS) + end + expect(last_client.source_clients.count).to eq(number_sample_clients % GrdaWarehouse::Tasks::IdentifyDuplicates::MAX_SOURCE_CLIENTS) + end + end end describe 'When matching is disabled' do From 816e6352db24bd46c3e3ddc323104cd7c000623e Mon Sep 17 00:00:00 2001 From: Elliot Date: Mon, 3 Feb 2025 16:22:48 -0500 Subject: [PATCH 07/18] Redact names everywhere on the client dashboard and some other locations as well (#5099) --- .../clients/anomalies_controller.rb | 2 +- app/controllers/clients/audits_controller.rb | 2 +- .../clients/cas_readiness_controller.rb | 2 +- app/controllers/clients/chronic_controller.rb | 2 +- ...oordinated_entry_assessments_controller.rb | 2 +- .../clients/enrollment_history_controller.rb | 2 +- app/controllers/clients/files_controller.rb | 2 +- .../clients/hud_lots_controller.rb | 2 +- app/controllers/clients/notes_controller.rb | 2 +- .../clients/releases_controller.rb | 4 +-- app/controllers/clients/users_controller.rb | 2 +- .../clients/vispdats_controller.rb | 2 +- .../clients/youth/intakes_controller.rb | 2 +- .../cohorts/client_notes_controller.rb | 2 +- app/controllers/cohorts/clients_controller.rb | 2 +- app/controllers/cohorts/notes_controller.rb | 2 +- app/controllers/concerns/activity_logger.rb | 2 +- app/models/grda_warehouse/pii_provider.rb | 32 +++++++++++++---- app/views/clients/_assessment_form.html.haml | 10 ++---- app/views/clients/_enrollment_table.haml | 5 ++- app/views/clients/_match_results.html.haml | 27 ++++---------- app/views/clients/_new_client_form.haml | 9 +++-- .../clients/_potential_matches.html.haml | 9 +++-- app/views/clients/_unmerge_form.haml | 11 +++--- app/views/clients/anomalies/_breadcrumbs.haml | 4 +-- .../cas_readiness/_active_clients.haml | 2 +- .../cas_readiness/_ce_with_assessment.haml | 2 +- app/views/clients/cas_readiness/_chronic.haml | 2 +- .../clients/cas_readiness/_hud_chronic.haml | 2 +- .../clients/cas_readiness/_project_group.haml | 2 +- .../_release_and_project_group.haml | 2 +- .../cas_readiness/_release_present.haml | 2 +- .../coordinated_entry_assessments/edit.haml | 4 +-- .../coordinated_entry_assessments/new.haml | 4 +-- .../coordinated_entry_assessments/show.haml | 4 +-- app/views/clients/edit.haml | 2 +- .../clients/enrollment_history/index.haml | 2 +- app/views/clients/enrollments/show.haml | 2 +- .../clients/files/_permission_warning.haml | 4 +-- app/views/clients/notes/index.haml | 4 +-- app/views/clients/releases/_form.haml | 2 +- app/views/clients/rollup/_family.html.haml | 9 ++--- app/views/clients/simple.haml | 4 +-- app/views/clients/users/index.haml | 4 +-- app/views/clients/vispdats/_list.haml | 2 +- app/views/clients/vispdats/edit.haml | 2 +- app/views/clients/vispdats/new.haml | 2 +- app/views/clients/vispdats/show.haml | 4 +-- .../youth/housing_resolution_plans/_form.haml | 4 +-- .../clients/youth/intakes/_breadcrumbs.haml | 2 +- app/views/clients/youth/intakes/_intakes.haml | 4 +-- app/views/clients/youth/intakes/edit.haml | 4 +-- app/views/clients/youth/intakes/index.haml | 4 +-- app/views/clients/youth/intakes/new.haml | 4 +-- app/views/clients/youth/intakes/show.haml | 4 +-- app/views/he/clients/boston_covid_19.haml | 2 +- .../boston_covid_19/_isolation_form.haml | 4 +-- .../boston_covid_19/_quarantine_form.haml | 3 +- .../clients/boston_covid_19/_triage_form.haml | 6 ++-- .../clients/covid_19_vaccinations_only.haml | 2 +- app/views/projects/_clients.haml | 3 +- app/views/source_clients/edit.haml | 2 +- app/views/source_data/show.haml | 36 +++++++++++++++++-- .../warehouse_reports/anomalies/_section.haml | 2 +- .../cas/health_prioritization/index.haml | 2 +- .../health_prioritization/index.xlsx.axlsx | 2 +- .../ce_assessments/index.haml | 5 +-- .../conflicting_client_attributes/index.haml | 7 ++-- .../warehouse_reports/consent/index.haml | 2 +- .../dob_entry_same/index.haml | 5 +-- .../dv_victim_service/_table.haml | 2 +- .../expiring_consent/index.haml | 6 ++-- .../medical_restrictions/_table.haml | 7 ++-- .../inactive_youth_intakes/_table.haml | 2 +- .../warehouse_reports/incomes/index.haml | 2 +- .../warehouse_reports/outflow/details.haml | 2 +- .../warehouse_reports/recidivism/index.haml | 2 +- .../time_homeless_for_exits/_table.haml | 2 +- .../youth_activity/index.haml | 2 +- .../history_controller.rb | 2 +- .../clients/enrollment_details.haml | 2 +- .../client_access_control/clients/simple.haml | 2 +- .../warehouse_reports/reports/index.haml | 2 +- .../client_location_history/location.rb | 4 +-- .../client_location_history/clients/map.haml | 2 +- .../clients/rollup/_financial_clients.haml | 2 +- .../app/views/financial/clients/show.haml | 2 +- .../warehouse_reports/reports/_report.haml | 5 +-- .../warehouse_reports/reports/index.haml | 2 +- 89 files changed, 204 insertions(+), 169 deletions(-) diff --git a/app/controllers/clients/anomalies_controller.rb b/app/controllers/clients/anomalies_controller.rb index 5cc90b81823..0bd51f54329 100644 --- a/app/controllers/clients/anomalies_controller.rb +++ b/app/controllers/clients/anomalies_controller.rb @@ -66,7 +66,7 @@ def flash_interpolation_options end protected def title_for_show - "#{@client.name} - Anomalies" + "#{@client.pii_provider(user: current_user).full_name} - Anomalies" end private def anomaly_params diff --git a/app/controllers/clients/audits_controller.rb b/app/controllers/clients/audits_controller.rb index 0a6e8f92cd4..7ad3c48e3b4 100644 --- a/app/controllers/clients/audits_controller.rb +++ b/app/controllers/clients/audits_controller.rb @@ -28,6 +28,6 @@ def set_client end def title_for_show - "#{@client.name} - Audit" + "#{@client.pii_provider(user: current_user).full_name} - Audit" end end diff --git a/app/controllers/clients/cas_readiness_controller.rb b/app/controllers/clients/cas_readiness_controller.rb index b2f141bcd12..93e5cd59d09 100644 --- a/app/controllers/clients/cas_readiness_controller.rb +++ b/app/controllers/clients/cas_readiness_controller.rb @@ -65,7 +65,7 @@ def cas_readiness_params end def title_for_show - "#{@client.name} - CAS Readiness" + "#{@client.pii_provider(user: current_user).full_name} - CAS Readiness" end end end diff --git a/app/controllers/clients/chronic_controller.rb b/app/controllers/clients/chronic_controller.rb index 79961871d3e..62e2ee3366c 100644 --- a/app/controllers/clients/chronic_controller.rb +++ b/app/controllers/clients/chronic_controller.rb @@ -47,7 +47,7 @@ def cas_readiness_params end def title_for_show - "#{@client.name} - Chronic" + "#{@client.pii_provider(user: current_user).full_name} - Chronic" end end end diff --git a/app/controllers/clients/coordinated_entry_assessments_controller.rb b/app/controllers/clients/coordinated_entry_assessments_controller.rb index 62694b7aa72..4febe8878f4 100644 --- a/app/controllers/clients/coordinated_entry_assessments_controller.rb +++ b/app/controllers/clients/coordinated_entry_assessments_controller.rb @@ -96,7 +96,7 @@ def update end private def title_for_show - "#{@client.name} - #{Translation.translate('Coordinated Entry Assessment')}" + "#{@client.pii_provider(user: current_user).full_name} - #{Translation.translate('Coordinated Entry Assessment')}" end def flash_interpolation_options diff --git a/app/controllers/clients/enrollment_history_controller.rb b/app/controllers/clients/enrollment_history_controller.rb index 8eb8b2def6b..b4547be8e40 100644 --- a/app/controllers/clients/enrollment_history_controller.rb +++ b/app/controllers/clients/enrollment_history_controller.rb @@ -40,7 +40,7 @@ def history_scope end def title_for_show - "#{@client.name} - Historical Enrollments" + "#{@client.pii_provider(user: current_user).full_name} - Historical Enrollments" end end end diff --git a/app/controllers/clients/files_controller.rb b/app/controllers/clients/files_controller.rb index c27c25a79c8..228bed97cfe 100644 --- a/app/controllers/clients/files_controller.rb +++ b/app/controllers/clients/files_controller.rb @@ -268,7 +268,7 @@ def file_source end protected def title_for_show - "#{@client.name} - Files" + "#{@client.pii_provider(user: current_user).full_name} - Files" end def window_visible?(visibility) diff --git a/app/controllers/clients/hud_lots_controller.rb b/app/controllers/clients/hud_lots_controller.rb index d9ce2d66a2f..758d25f0995 100644 --- a/app/controllers/clients/hud_lots_controller.rb +++ b/app/controllers/clients/hud_lots_controller.rb @@ -29,7 +29,7 @@ def index end private def title_for_show - "#{@client.name} - Client-Level System Use & Length of Time Homeless Report" + "#{@client.pii_provider(user: current_user).full_name} - Client-Level System Use & Length of Time Homeless Report" end helper_method :title_for_show diff --git a/app/controllers/clients/notes_controller.rb b/app/controllers/clients/notes_controller.rb index 44e4fdf8ebd..268e10d71b6 100644 --- a/app/controllers/clients/notes_controller.rb +++ b/app/controllers/clients/notes_controller.rb @@ -112,7 +112,7 @@ def destroy end private def title_for_show - "#{@client.name} - Notes" + "#{@client.pii_provider(user: current_user).full_name} - Notes" end end end diff --git a/app/controllers/clients/releases_controller.rb b/app/controllers/clients/releases_controller.rb index 860936df37c..1a1ac6a68f6 100644 --- a/app/controllers/clients/releases_controller.rb +++ b/app/controllers/clients/releases_controller.rb @@ -139,7 +139,7 @@ def pre_populated private def render_pdf! @pdf = true - file_name = "Release of Information for #{@client.name}" + file_name = "Release of Information for #{@client.pii_provider(user: current_user).full_name}" send_data roi_pdf(file_name), filename: "#{file_name}.pdf", type: 'application/pdf' end @@ -175,7 +175,7 @@ def file_source end protected def title_for_show - "#{@client.name} - Release of Information" + "#{@client.pii_provider(user: current_user).full_name} - Release of Information" end def window_visible?(_visibility) diff --git a/app/controllers/clients/users_controller.rb b/app/controllers/clients/users_controller.rb index 2d5fa35ca29..ea0c6eabd1b 100644 --- a/app/controllers/clients/users_controller.rb +++ b/app/controllers/clients/users_controller.rb @@ -76,7 +76,7 @@ def set_user end protected def title_for_show - "#{@client.name} - Relationships" + "#{@client.pii_provider(user: current_user).full_name} - Relationships" end end end diff --git a/app/controllers/clients/vispdats_controller.rb b/app/controllers/clients/vispdats_controller.rb index f7a2da6202d..b5e5700ec79 100644 --- a/app/controllers/clients/vispdats_controller.rb +++ b/app/controllers/clients/vispdats_controller.rb @@ -164,7 +164,7 @@ def destroy_file end private def title_for_show - "#{@client.name} - VI-SPDATs" + "#{@client.pii_provider(user: current_user).full_name} - VI-SPDATs" end end end diff --git a/app/controllers/clients/youth/intakes_controller.rb b/app/controllers/clients/youth/intakes_controller.rb index 4fde5bad0d9..9c83cd5a173 100644 --- a/app/controllers/clients/youth/intakes_controller.rb +++ b/app/controllers/clients/youth/intakes_controller.rb @@ -88,7 +88,7 @@ def remove_all_youth_data @client.youth_follow_ups.destroy_all # TODO: This does not remove the client from the Youth DataSource - flash[:notice] = "All Youth information for #{@client.name} has been removed." + flash[:notice] = "All Youth information for #{@client.pii_provider(user: current_user).full_name} has been removed." redirect_to client_youth_intakes_path(@client) else not_authorized! diff --git a/app/controllers/cohorts/client_notes_controller.rb b/app/controllers/cohorts/client_notes_controller.rb index 399e7d4f71f..0256ffe589e 100644 --- a/app/controllers/cohorts/client_notes_controller.rb +++ b/app/controllers/cohorts/client_notes_controller.rb @@ -83,7 +83,7 @@ def cohort_id end def flash_interpolation_options - { resource_name: "Note for #{@note.client.name}" } + { resource_name: "Note for #{@note.client.pii_provider(user: current_user).full_name}" } end end end diff --git a/app/controllers/cohorts/clients_controller.rb b/app/controllers/cohorts/clients_controller.rb index 805a9a2c69a..6326340dabb 100644 --- a/app/controllers/cohorts/clients_controller.rb +++ b/app/controllers/cohorts/clients_controller.rb @@ -485,7 +485,7 @@ def destroy else log_removal(@client.cohort_id, @client.id, params.dig(:grda_warehouse_cohort_client, :reason)) if @client.destroy - flash[:notice] = "Removed #{@client.name}" + flash[:notice] = "Removed #{@client.pii_provider(user: current_user).full_name}" redirect_to cohort_path(@cohort) else render :pre_destroy diff --git a/app/controllers/cohorts/notes_controller.rb b/app/controllers/cohorts/notes_controller.rb index d6b7435830c..1cc2ce4568f 100644 --- a/app/controllers/cohorts/notes_controller.rb +++ b/app/controllers/cohorts/notes_controller.rb @@ -88,7 +88,7 @@ def cohort_id end def flash_interpolation_options - { resource_name: "Note for #{@note.client.name}" } + { resource_name: "Note for #{@note.client.pii_provider(user: current_user).full_name}" } end end end diff --git a/app/controllers/concerns/activity_logger.rb b/app/controllers/concerns/activity_logger.rb index c9d37060638..ee303088d50 100644 --- a/app/controllers/concerns/activity_logger.rb +++ b/app/controllers/concerns/activity_logger.rb @@ -58,7 +58,7 @@ def log_activity # override as necessary in the controller protected def title_for_show - return @client.name if @client.present? + return @client.pii_provider(user: current_user).full_name if @client.present? return @user.name if @user.present? end diff --git a/app/models/grda_warehouse/pii_provider.rb b/app/models/grda_warehouse/pii_provider.rb index adadaf420f2..b7e2e5b4c99 100644 --- a/app/models/grda_warehouse/pii_provider.rb +++ b/app/models/grda_warehouse/pii_provider.rb @@ -17,6 +17,12 @@ def self.viewable_name(value, policy:, replacement: REDACTED) value.presence end + def self.viewable_ssn(value, policy:, replacement: REDACTED) + return replacement unless policy.can_view_full_ssn? + + value.presence + end + def self.viewable_dob(value, policy:, replacement: REDACTED) return replacement unless policy.can_view_full_dob? @@ -46,32 +52,44 @@ def self.from_attributes(policy: nil, first_name: nil, last_name: nil, middle_na new(record, policy: policy) end + def redact_name? + ! policy.can_view_name? + end + + def redact_ssn? + ! policy.can_view_full_ssn? + end + + def redact_dob? + ! policy.can_view_full_dob? + end + def first_name - return name_redacted unless policy.can_view_name? + return name_redacted if redact_name? record.first_name.presence end def last_name - return name_redacted unless policy.can_view_name? + return name_redacted if redact_name? record.last_name.presence end def middle_name - return name_redacted unless policy.can_view_name? + return name_redacted if redact_name? record.middle_name.presence end def full_name - return name_redacted unless policy.can_view_name? + return name_redacted if redact_name? [record.first_name, record.middle_name, record.last_name].compact.join(' ').presence end def brief_name - return name_redacted unless policy.can_view_name? + return name_redacted if redact_name? [record.first_name, record.last_name].compact.join(' ').presence end @@ -84,7 +102,7 @@ def dob_and_age(force_year_only: false) return nil unless record.dob display_dob = record.dob - display_dob = display_dob&.year if force_year_only || !policy.can_view_full_dob? + display_dob = display_dob&.year if force_year_only || redact_dob? "#{display_dob} (#{age})" end @@ -101,7 +119,7 @@ def age def ssn(force_mask: false) value = record.ssn.presence - mask = force_mask || !policy.can_view_full_ssn? + mask = force_mask || redact_ssn? format_ssn(value, mask: mask) if value end diff --git a/app/views/clients/_assessment_form.html.haml b/app/views/clients/_assessment_form.html.haml index 7fd4b24d614..f07c207165d 100644 --- a/app/views/clients/_assessment_form.html.haml +++ b/app/views/clients/_assessment_form.html.haml @@ -1,13 +1,13 @@ - if @form[:answers].present? - form_name = (@form.assessment_type == @form.name) ? @form.name : "#{@form.assessment_type} < #{@form.name}" - content_for :modal_title, form_name - + .d-flex .w-100 .ssm__summary.d-flex.flex-column .mb-2 %dt.inline Name: - %dd.inline= @client.name + %dd.inline= @client.pii_provider(user: current_user).full_name .mb-2 %dt.inline Date Completed: %dd.inline= @form.collected_at&.to_date @@ -29,9 +29,3 @@ .client__assessment-answer= question[:answer] - else - content_for :modal_title, "Assessment Form Not Found" - - - - - - diff --git a/app/views/clients/_enrollment_table.haml b/app/views/clients/_enrollment_table.haml index 19fcf5d9bc9..755b0ace4d6 100644 --- a/app/views/clients/_enrollment_table.haml +++ b/app/views/clients/_enrollment_table.haml @@ -120,7 +120,10 @@ - tooltip += "Exit: #{c['last_date_in_program']}
" .mb-2.mt-2{ data: { toggle: :tooltip, title: tooltip, html: 'true', boundary: :window } } = link_to client_path(c['client_id']), class: 'd-block' do - #{c['FirstName']} #{c['LastName']} + - if @client.pii_provider(user: current_user).redact_name? + = @client.pii_provider(user: current_user).first_name + - else + #{c['FirstName']} #{c['LastName']} - if c['head_of_household'] %i.icon-user %br diff --git a/app/views/clients/_match_results.html.haml b/app/views/clients/_match_results.html.haml index ff17f23480a..fb00a25e002 100644 --- a/app/views/clients/_match_results.html.haml +++ b/app/views/clients/_match_results.html.haml @@ -5,12 +5,12 @@ %thead %tr %th{colspan: 2} Client - %th DOB + %th DOB/Age %th SSN %tbody - clients.each do |c| %tr.client__potential-match - - client_name = "#{c.full_name}" + - client_name = "#{c.pii_provider(user: current_user).full_name}" - sc_count = c.source_clients.count - colspan = if sc_count == 1 then 2 else 4 end - if sc_count == 1 @@ -23,16 +23,8 @@ %label{for:c.id, tabindex:'1' } = link_to(client_name, client_path(c), target: "_blank") - if sc_count == 1 - %td - - if can_view_full_dob? - = c.DOB - - else - = c.age - %td - - if can_view_full_ssn? - = ssn(c.SSN) - - else - = masked_ssn(c.SSN) + %td= c.pii_provider(user: current_user).dob_or_age + %td= c.pii_provider(user: current_user).ssn - else - c.source_clients.each do |sc| %tr @@ -42,11 +34,6 @@ .c-checkbox.c-checkbox.mr-4 = check_box_tag input_id, sc.id, nil, id: sc.id %label{for:sc.id, tabindex:'1' } - %span= "#{sc.full_name} in #{sc.data_source&.short_name}
#{sc.uuid}".html_safe - %td - = sc.DOB - %td - - if can_view_full_ssn? - = ssn(sc.SSN) - - else - = masked_ssn(sc.SSN) + %span= "#{sc.pii_provider(user: current_user).full_name} in #{sc.data_source&.short_name}
#{sc.uuid}".html_safe + %td= sc.pii_provider(user: current_user).dob_or_age + %td= sc.pii_provider(user: current_user).ssn diff --git a/app/views/clients/_new_client_form.haml b/app/views/clients/_new_client_form.haml index 170c53aee33..ea7a43ba8f1 100644 --- a/app/views/clients/_new_client_form.haml +++ b/app/views/clients/_new_client_form.haml @@ -18,6 +18,7 @@ %th SSN %tbody - @existing_matches.each do |client| + - pii = client.pii_provider(user: current_user) %tr %td - ds_id = client.data_source.id @@ -30,11 +31,9 @@ - else - link = client.destination_client.appropriate_path_for?(current_user) = link_to link do - = client.full_name - %td - = client.DOB - %td - = client.SSN + = pii.full_name + %td= pii.dob_or_age + %td= pii.ssn %h2 New Client %p If none of the above match the client you are attempting to add, click diff --git a/app/views/clients/_potential_matches.html.haml b/app/views/clients/_potential_matches.html.haml index fd6d2627887..5ffdfcb1817 100644 --- a/app/views/clients/_potential_matches.html.haml +++ b/app/views/clients/_potential_matches.html.haml @@ -1,7 +1,7 @@ %h3 Potential Matches -%p - This section allows you to merge a client into - = "#{@client.name}." +%p + This section allows you to merge a client into + = "#{@client.pii_provider(user: current_user).full_name}." If a potential client is the combination of merged clients, you can choose to merge with the client set, or with an individual client. - if @potential_matches.any? .row @@ -12,7 +12,6 @@ = k.to_s.humanize.titlecase = render 'match_results', f: f, clients: clients .form-actions - = f.button :submit, "Merge into #{@client.FirstName} #{@client.LastName}" + = f.button :submit, "Merge into #{@client.pii_provider(user: current_user).full_name}" - else %p No potential matches found - \ No newline at end of file diff --git a/app/views/clients/_unmerge_form.haml b/app/views/clients/_unmerge_form.haml index fa6f788a0b3..633a9d97662 100644 --- a/app/views/clients/_unmerge_form.haml +++ b/app/views/clients/_unmerge_form.haml @@ -30,13 +30,14 @@ %th Data Source %th Personal ID %th Name - %th DOB + %th DOB/Age %th SSN %th Merged %th Merged By %th %tbody - source_clients.each do |c| + - pii = c.pii_provider(user: current_user) %tr %td.jSplit= f.input_field :merge, as: :boolean, checked_value: c.id, unchecked_value: nil, name: input_id - if GrdaWarehouse::Config.get(:healthcare_available) @@ -46,9 +47,9 @@ = radio_button_tag 'grda_warehouse_hud_client[health_receiver]', c.id, false, disabled: true %td= c.data_source&.short_name %td= c.uuid - %td= c.full_name - %td= c.DOB - %td= ssn c.SSN + %td= pii.full_name + %td= pii.dob_or_age + %td= pii.ssn %td= c.warehouse_client_source.reviewed_at %td - if (user = reviewers[c.warehouse_client_source.reviewd_by.to_i]) @@ -59,7 +60,7 @@ Match Details .form-actions - = f.button :submit, "Split selected records from #{@client.FirstName} #{@client.LastName}", id: :splitButton, data: {confirm: "Are you sure you want to split #{@client.FirstName} #{@client.LastName}? Dependent data will be moved to the selected clients."} + = f.button :submit, "Split selected records from #{@client.pii_provider(user: current_user).full_name}", id: :splitButton, data: {confirm: "Are you sure you want to split #{@client.pii_provider(user: current_user).full_name}? Dependent data will be moved to the selected clients."} = content_for :page_js do :javascript diff --git a/app/views/clients/anomalies/_breadcrumbs.haml b/app/views/clients/anomalies/_breadcrumbs.haml index 9a8d124b593..515045c1a6e 100644 --- a/app/views/clients/anomalies/_breadcrumbs.haml +++ b/app/views/clients/anomalies/_breadcrumbs.haml @@ -1,4 +1,4 @@ = content_for :crumbs do = link_to client_path(@client) do - « Back to - = @client.name \ No newline at end of file + « Back to + = @client.pii_provider(user: current_user).full_name diff --git a/app/views/clients/cas_readiness/_active_clients.haml b/app/views/clients/cas_readiness/_active_clients.haml index 7225e09e9b9..240c354c5de 100644 --- a/app/views/clients/cas_readiness/_active_clients.haml +++ b/app/views/clients/cas_readiness/_active_clients.haml @@ -2,7 +2,7 @@ Any client with service within the past #{pluralize(GrdaWarehouse::Config.get(:cas_sync_months), 'month')}. %p Will - %strong= @client.name + %strong= @client.pii_provider(user: current_user).full_name sync with CAS? %br = yes_no @client.active_in_cas?(include_overridden: false) diff --git a/app/views/clients/cas_readiness/_ce_with_assessment.haml b/app/views/clients/cas_readiness/_ce_with_assessment.haml index 7fa1173e41c..f7e5febf350 100644 --- a/app/views/clients/cas_readiness/_ce_with_assessment.haml +++ b/app/views/clients/cas_readiness/_ce_with_assessment.haml @@ -2,7 +2,7 @@ Any client currently enrolled in a Coordinated Entry project and having at least one associated assessment. %p Will - %strong= @client.name + %strong= @client.pii_provider(user: current_user).full_name sync with CAS? %br = yes_no @client.active_in_cas?(include_overridden: false) diff --git a/app/views/clients/cas_readiness/_chronic.haml b/app/views/clients/cas_readiness/_chronic.haml index 3a743080355..1ecb5cba319 100644 --- a/app/views/clients/cas_readiness/_chronic.haml +++ b/app/views/clients/cas_readiness/_chronic.haml @@ -2,7 +2,7 @@ All clients on the most recent potentially chronic list will be included in each CAS sync. %p Will - %strong= @client.name + %strong= @client.pii_provider(user: current_user).full_name sync with CAS? %br = yes_no @client.active_in_cas?(include_overridden: false) diff --git a/app/views/clients/cas_readiness/_hud_chronic.haml b/app/views/clients/cas_readiness/_hud_chronic.haml index 4aa7113954c..f698cfc4c60 100644 --- a/app/views/clients/cas_readiness/_hud_chronic.haml +++ b/app/views/clients/cas_readiness/_hud_chronic.haml @@ -2,7 +2,7 @@ All clients on the most recent chronic list will be included in each CAS sync. %p Will - %strong= @client.name + %strong= @client.pii_provider(user: current_user).full_name sync with CAS? %br = yes_no @client.active_in_cas?(include_overridden: false) diff --git a/app/views/clients/cas_readiness/_project_group.haml b/app/views/clients/cas_readiness/_project_group.haml index d53aba4f82a..404673e684a 100644 --- a/app/views/clients/cas_readiness/_project_group.haml +++ b/app/views/clients/cas_readiness/_project_group.haml @@ -2,7 +2,7 @@ Any client in project group '#{GrdaWarehouse::Config.cas_sync_project_group.name}'. %p Will - %strong= @client.name + %strong= @client.pii_provider(user: current_user).full_name sync with CAS? %br = yes_no @client.active_in_cas?(include_overridden: false) diff --git a/app/views/clients/cas_readiness/_release_and_project_group.haml b/app/views/clients/cas_readiness/_release_and_project_group.haml index abfe4ba2a33..8dc68b5c6d1 100644 --- a/app/views/clients/cas_readiness/_release_and_project_group.haml +++ b/app/views/clients/cas_readiness/_release_and_project_group.haml @@ -10,7 +10,7 @@ will sync with CAS %p Will - %strong= @client.name + %strong= @client.pii_provider(user: current_user).full_name sync with CAS? %br = yes_no @client.active_in_cas?(include_overridden: false) diff --git a/app/views/clients/cas_readiness/_release_present.haml b/app/views/clients/cas_readiness/_release_present.haml index 4d133c72adc..b7377e211c0 100644 --- a/app/views/clients/cas_readiness/_release_present.haml +++ b/app/views/clients/cas_readiness/_release_present.haml @@ -5,7 +5,7 @@ %p Will - %strong= @client.name + %strong= @client.pii_provider(user: current_user).full_name sync with CAS? %br = yes_no @client.active_in_cas?(include_overridden: false) diff --git a/app/views/clients/coordinated_entry_assessments/edit.haml b/app/views/clients/coordinated_entry_assessments/edit.haml index 18e51e50c88..8dc673f27b9 100644 --- a/app/views/clients/coordinated_entry_assessments/edit.haml +++ b/app/views/clients/coordinated_entry_assessments/edit.haml @@ -1,4 +1,4 @@ -- title = "Editing #{_'Coordinated Entry Assessment'} for #{@client.name}" +- title = "Editing #{_'Coordinated Entry Assessment'} for #{@client.pii_provider(user: current_user).full_name}" - content_for :title, title .o-page = render 'clients/breadcrumbs' @@ -11,4 +11,4 @@ = render 'clients/tab_navigation', current: client_coordinated_entry_assessments_path(client_id: @client) = simple_form_for @assessment, url: client_coordinated_entry_assessment_path(@client, @assessment) do |f| - = render 'form', f: f, readonly: false \ No newline at end of file + = render 'form', f: f, readonly: false diff --git a/app/views/clients/coordinated_entry_assessments/new.haml b/app/views/clients/coordinated_entry_assessments/new.haml index ba4e987cfab..f49a275a5b5 100644 --- a/app/views/clients/coordinated_entry_assessments/new.haml +++ b/app/views/clients/coordinated_entry_assessments/new.haml @@ -1,4 +1,4 @@ -- title = "New #{_'Coordinated Entry Assessment'} for #{@client.name}" +- title = "New #{_'Coordinated Entry Assessment'} for #{@client.pii_provider(user: current_user).full_name}" - content_for :title, title .o-page = render 'clients/breadcrumbs' @@ -11,4 +11,4 @@ = render 'clients/tab_navigation', current: client_coordinated_entry_assessments_path(client_id: @client) = simple_form_for @assessment, url: client_coordinated_entry_assessments_path(@client) do |f| - = render 'form', f: f, readonly: false \ No newline at end of file + = render 'form', f: f, readonly: false diff --git a/app/views/clients/coordinated_entry_assessments/show.haml b/app/views/clients/coordinated_entry_assessments/show.haml index c264a6336a2..ec8d3e5ae69 100644 --- a/app/views/clients/coordinated_entry_assessments/show.haml +++ b/app/views/clients/coordinated_entry_assessments/show.haml @@ -1,4 +1,4 @@ -- title = "#{_'Coordinated Entry Assessment'} for #{@client.name}" +- title = "#{_'Coordinated Entry Assessment'} for #{@client.pii_provider(user: current_user).full_name}" - content_for :title, title .o-page = render 'clients/breadcrumbs' @@ -11,4 +11,4 @@ = render 'clients/tab_navigation', current: client_coordinated_entry_assessments_path(client_id: @client) = simple_form_for @assessment, wrapper: :readonly, url: client_coordinated_entry_assessment_path(@client, @assessment) do |f| - = render 'form', f: f \ No newline at end of file + = render 'form', f: f diff --git a/app/views/clients/edit.haml b/app/views/clients/edit.haml index 23aef9ff723..1bdbdf64ebf 100644 --- a/app/views/clients/edit.haml +++ b/app/views/clients/edit.haml @@ -1,4 +1,4 @@ -- title = "Potential Duplicates for #{@client.name}" +- title = "Potential Duplicates for #{@client.pii_provider(user: current_user).full_name}" - content_for :title, title = render 'breadcrumbs' diff --git a/app/views/clients/enrollment_history/index.haml b/app/views/clients/enrollment_history/index.haml index cea0bd859d1..953bfe67ded 100644 --- a/app/views/clients/enrollment_history/index.haml +++ b/app/views/clients/enrollment_history/index.haml @@ -1,4 +1,4 @@ -- title = "Historical Enrollment Data for #{@client.name} on #{@date}" +- title = "Historical Enrollment Data for #{@client.pii_provider(user: current_user).full_name} on #{@date}" - content_for :title, title %h1= title = render 'clients/anomalies/breadcrumbs' diff --git a/app/views/clients/enrollments/show.haml b/app/views/clients/enrollments/show.haml index e1b8d7f1af2..51255f7591a 100644 --- a/app/views/clients/enrollments/show.haml +++ b/app/views/clients/enrollments/show.haml @@ -1,4 +1,4 @@ -- title = "Enrollment at #{@enrollment.project&.name(current_user)} for #{@client.name} " +- title = "Enrollment at #{@enrollment.project&.name(current_user)} for #{@client.pii_provider(user: current_user).full_name} " - content_for :title, title = render 'clients/anomalies/breadcrumbs' %h1= title diff --git a/app/views/clients/files/_permission_warning.haml b/app/views/clients/files/_permission_warning.haml index 1356e31ce68..89784512f75 100644 --- a/app/views/clients/files/_permission_warning.haml +++ b/app/views/clients/files/_permission_warning.haml @@ -3,8 +3,8 @@ %i.alert__icon.icon-info %p - if can_manage_window_client_files? && @client.release_valid?(coc_codes: current_user.coc_codes) - = Translation.translate("#{@client.name} has a consent form on file, you can see any client files.") + = Translation.translate("#{@client.pii_provider(user: current_user).full_name} has a consent form on file, you can see any client files.") - elsif can_manage_window_client_files? - = Translation.translate("#{@client.name} does not have a consent form on file, you can only see files you upload.") + = Translation.translate("#{@client.pii_provider(user: current_user).full_name} does not have a consent form on file, you can only see files you upload.") - else = Translation.translate("You can only see files you upload.") diff --git a/app/views/clients/notes/index.haml b/app/views/clients/notes/index.haml index 4a4cf4373c6..7c35de849da 100644 --- a/app/views/clients/notes/index.haml +++ b/app/views/clients/notes/index.haml @@ -13,9 +13,9 @@ - unless can_edit_client_notes? || can_view_all_window_notes? %p.alert.alert-info - if can_edit_window_client_notes? && @client.release_valid? - = Translation.translate("#{@client.name} has a consent form on file, you can see any notes about this client.") + = Translation.translate("#{@client.pii_provider(user: current_user).full_name} has a consent form on file, you can see any notes about this client.") - elsif can_edit_window_client_notes? - = Translation.translate("#{@client.name} does not have a consent form on file, you can only see notes you add.") + = Translation.translate("#{@client.pii_provider(user: current_user).full_name} does not have a consent form on file, you can only see notes you add.") - else = Translation.translate("You can only see notes you add.") - if @notes.any? diff --git a/app/views/clients/releases/_form.haml b/app/views/clients/releases/_form.haml index 8d1af43786f..b31e4b51742 100644 --- a/app/views/clients/releases/_form.haml +++ b/app/views/clients/releases/_form.haml @@ -3,7 +3,7 @@ .col-4 %dl %dt Name: - %dd= @client.name + %dd= @client.pii_provider(user: current_user).full_name .col-4 %dl %dt Date of Birth: diff --git a/app/views/clients/rollup/_family.html.haml b/app/views/clients/rollup/_family.html.haml index ad8df00eb30..75668a179bf 100644 --- a/app/views/clients/rollup/_family.html.haml +++ b/app/views/clients/rollup/_family.html.haml @@ -9,14 +9,11 @@ %th Race %tbody - @client.family_members.each do |client| + - pii = client.pii_provider(user: current_user) %tr %td - = link_to client.name, client_path(client) - %td - - if can_view_full_ssn? - = ssn(client.SSN) - - else - = masked_ssn(client.SSN) + = link_to pii.full_name, client_path(client) + %td= pii.ssn %td= client.age %td= client.gender %td= client.race_fields.map{ |f| HudUtility2024.race(f) }.join ', ' diff --git a/app/views/clients/simple.haml b/app/views/clients/simple.haml index df60c6900fc..ec78b27dfdb 100644 --- a/app/views/clients/simple.haml +++ b/app/views/clients/simple.haml @@ -1,5 +1,5 @@ -- title = @client.full_name +- title = @client.pii_provider(user: current_user).full_name - content_for :title, title = render 'breadcrumbs' = render 'tab_navigation', current: simple_client_path(@client) -= render 'client_card', client: @client, disable_link: true \ No newline at end of file += render 'client_card', client: @client, disable_link: true diff --git a/app/views/clients/users/index.haml b/app/views/clients/users/index.haml index aeaae794b39..263b4ae07ed 100644 --- a/app/views/clients/users/index.haml +++ b/app/views/clients/users/index.haml @@ -1,8 +1,8 @@ -- title = "Assigned Relationships for #{@client.name}" +- title = "Assigned Relationships for #{@client.pii_provider(user: current_user).full_name}" - content_for :title, title = render 'clients/breadcrumbs' = render 'clients/aliases' = render 'clients/tab_navigation', current: client_users_path(@client) -= render 'relationship_list' \ No newline at end of file += render 'relationship_list' diff --git a/app/views/clients/vispdats/_list.haml b/app/views/clients/vispdats/_list.haml index 392660088f4..93e53278015 100644 --- a/app/views/clients/vispdats/_list.haml +++ b/app/views/clients/vispdats/_list.haml @@ -21,7 +21,7 @@ %i.icon-group Family -# else - %button.btn.btn-primary(title="#{@client.full_name} already has a VI-SPDAT in progress. Please complete that one or delete it before starting a new VI-SPDAT" data-toggle='tooltip') + %button.btn.btn-primary(title="#{@client.pii_provider(user: current_user).full_name} already has a VI-SPDAT in progress. Please complete that one or delete it before starting a new VI-SPDAT" data-toggle='tooltip') %span.icon-plus .card diff --git a/app/views/clients/vispdats/edit.haml b/app/views/clients/vispdats/edit.haml index 097e887f063..3bc62aecb44 100644 --- a/app/views/clients/vispdats/edit.haml +++ b/app/views/clients/vispdats/edit.haml @@ -16,7 +16,7 @@ %span.icon-eye View - if can_edit_vspdat? - = link_to polymorphic_path(vispdat_path_generator, client_id: @client.id, id: @vispdat.id), class: 'btn btn-danger', title: 'Delete', method: :delete, data: { toggle: 'tooltip', confirm: "Are you sure you want to delete the VI-SPDAT for #{@client.full_name}?" } do + = link_to polymorphic_path(vispdat_path_generator, client_id: @client.id, id: @vispdat.id), class: 'btn btn-danger', title: 'Delete', method: :delete, data: { toggle: 'tooltip', confirm: "Are you sure you want to delete the VI-SPDAT for #{@client.pii_provider(user: current_user).full_name}?" } do %span.icon-cross Delete = link_to polymorphic_path(vispdats_path_generator, client_id: @client.id), class: 'btn btn-default', title: 'All', data: { toggle: 'tooltip' } do diff --git a/app/views/clients/vispdats/new.haml b/app/views/clients/vispdats/new.haml index 0cbf163dea3..cc8cc0b06d8 100644 --- a/app/views/clients/vispdats/new.haml +++ b/app/views/clients/vispdats/new.haml @@ -1,4 +1,4 @@ -- title = "New VISPDAT for #{@client.name}" +- title = "New VISPDAT for #{@client.pii_provider(user: current_user).full_name}" - content_for :title, title .o-page = render 'clients/breadcrumbs' diff --git a/app/views/clients/vispdats/show.haml b/app/views/clients/vispdats/show.haml index 561227be0fe..1f69c8ce0c8 100644 --- a/app/views/clients/vispdats/show.haml +++ b/app/views/clients/vispdats/show.haml @@ -1,4 +1,4 @@ -- content_for(:title, @client.name) +- content_for(:title, @client.pii_provider(user: current_user).full_name) .o-page = render 'clients/breadcrumbs' @@ -20,7 +20,7 @@ %span.icon-pencil Edit - if can_edit_vspdat? - = link_to polymorphic_path(vispdat_path_generator, client_id: @client.id, id: @vispdat.id), class: 'btn btn-danger', title: 'Delete', method: :delete, data: { confirm: "Are you sure you want to delete the VI-SPDAT for #{@client.full_name}?" } do + = link_to polymorphic_path(vispdat_path_generator, client_id: @client.id, id: @vispdat.id), class: 'btn btn-danger', title: 'Delete', method: :delete, data: { confirm: "Are you sure you want to delete the VI-SPDAT for #{@client.pii_provider(user: current_user).full_name}?" } do %span.icon-cross Delete = link_to polymorphic_path(vispdats_path_generator, client_id: @client.id), class: 'btn btn-default', title: 'All', data: { toggle: 'tooltip' } do diff --git a/app/views/clients/youth/housing_resolution_plans/_form.haml b/app/views/clients/youth/housing_resolution_plans/_form.haml index 4a5034f8954..8b2059d50c2 100644 --- a/app/views/clients/youth/housing_resolution_plans/_form.haml +++ b/app/views/clients/youth/housing_resolution_plans/_form.haml @@ -1,11 +1,11 @@ .row .col-6 - = f.input 'name', as: :string, disabled: true, label: 'YYA Name (goes by)', input_html: {value: @client.name} + = f.input 'name', as: :string, disabled: true, label: 'YYA Name (goes by)', input_html: {value: @client.pii_provider(user: current_user).full_name} .col-6 = f.input :pronouns, label: 'YYA Pronouns' .row .col-6 - = f.input 'name', as: :string, disabled: true, label: 'YYA Legal Name', input_html: {value: @client.name} + = f.input 'name', as: :string, disabled: true, label: 'YYA Legal Name', input_html: {value: @client.pii_provider(user: current_user).full_name} .col-6 = f.input :planned_on, as: :date_picker, label: 'Date' .row diff --git a/app/views/clients/youth/intakes/_breadcrumbs.haml b/app/views/clients/youth/intakes/_breadcrumbs.haml index 21ef3bf25ce..0bbee9443b4 100644 --- a/app/views/clients/youth/intakes/_breadcrumbs.haml +++ b/app/views/clients/youth/intakes/_breadcrumbs.haml @@ -1,4 +1,4 @@ = content_for :crumbs do = link_to polymorphic_path(youth_intakes_path_generator) do « - = Translation.translate("Youth Intakes for #{@client.name}") + = Translation.translate("Youth Intakes for #{@client.pii_provider(user: current_user).full_name}") diff --git a/app/views/clients/youth/intakes/_intakes.haml b/app/views/clients/youth/intakes/_intakes.haml index 7400e6b2703..0a34f11066b 100644 --- a/app/views/clients/youth/intakes/_intakes.haml +++ b/app/views/clients/youth/intakes/_intakes.haml @@ -1,11 +1,11 @@ - if @client.youth_follow_up_due_soon? %div.alert.alert-warning %i.alert__icon.icon-warning - A 3-month follow up case note is due for #{@client.name} on #{@client.youth_follow_up_due_on} + A 3-month follow up case note is due for #{@client.pii_provider(user: current_user).full_name} on #{@client.youth_follow_up_due_on} - if @client.youth_follow_up_due? %div.alert.alert-danger %i.alert__icon.icon-warning - A 3-month follow up case note was due for #{@client.name} on #{@client.youth_follow_up_due_on} + A 3-month follow up case note was due for #{@client.pii_provider(user: current_user).full_name} on #{@client.youth_follow_up_due_on} = render 'intake_list' = render 'case_management_list' - if GrdaWarehouse::Config.get(:enable_youth_hrp) diff --git a/app/views/clients/youth/intakes/edit.haml b/app/views/clients/youth/intakes/edit.haml index 574910ba71e..4c8d38dd46d 100644 --- a/app/views/clients/youth/intakes/edit.haml +++ b/app/views/clients/youth/intakes/edit.haml @@ -1,4 +1,4 @@ -- content_for :title, "Edit Youth Intake for #{@client.full_name}" +- content_for :title, "Edit Youth Intake for #{@client.pii_provider(user: current_user).full_name}" %h1.page-title= content_for :title @@ -10,4 +10,4 @@ %i.icon-cross Cancel .ml-2 - = f.submit 'Save', class: 'btn btn-primary' \ No newline at end of file + = f.submit 'Save', class: 'btn btn-primary' diff --git a/app/views/clients/youth/intakes/index.haml b/app/views/clients/youth/intakes/index.haml index eadfe5a738e..c870baa8cab 100644 --- a/app/views/clients/youth/intakes/index.haml +++ b/app/views/clients/youth/intakes/index.haml @@ -8,6 +8,6 @@ - if can_delete_youth_intake? .text-right - = link_to remove_all_youth_data_client_youth_intakes_path(@client), method: :delete, class: 'btn btn-danger', data: {confirm: "Are you sure you want to delete all youth information for #{@client.name}? Deleting is non-reversible, and will remove all items on this page. Proceed?"} do + = link_to remove_all_youth_data_client_youth_intakes_path(@client), method: :delete, class: 'btn btn-danger', data: {confirm: "Are you sure you want to delete all youth information for #{@client.pii_provider(user: current_user).full_name}? Deleting is non-reversible, and will remove all items on this page. Proceed?"} do %i.icon-cross - Delete All Youth Data \ No newline at end of file + Delete All Youth Data diff --git a/app/views/clients/youth/intakes/new.haml b/app/views/clients/youth/intakes/new.haml index ba7fea7bee3..a3174847a2f 100644 --- a/app/views/clients/youth/intakes/new.haml +++ b/app/views/clients/youth/intakes/new.haml @@ -1,4 +1,4 @@ -- content_for :title, "New Youth Intake for #{@client.full_name}" +- content_for :title, "New Youth Intake for #{@client.pii_provider(user: current_user).full_name}" %h1.page-title= content_for :title @@ -10,4 +10,4 @@ %i.icon-cross Cancel .ml-2 - = f.submit 'Save', class: 'btn btn-primary' \ No newline at end of file + = f.submit 'Save', class: 'btn btn-primary' diff --git a/app/views/clients/youth/intakes/show.haml b/app/views/clients/youth/intakes/show.haml index a0a1b221785..0be6727876e 100644 --- a/app/views/clients/youth/intakes/show.haml +++ b/app/views/clients/youth/intakes/show.haml @@ -1,5 +1,5 @@ = render 'breadcrumbs' -- content_for :title, "Youth Intake for #{@client.full_name}" +- content_for :title, "Youth Intake for #{@client.pii_provider(user: current_user).full_name}" .d-flex %h1.page-title= content_for :title @@ -10,4 +10,4 @@ Edit = simple_form_for @intake, url: polymorphic_path(youth_intake_path_generator, id: @intake.id), wrapper: :readonly do |f| - = render 'intake_form', f: f, readonly: true \ No newline at end of file + = render 'intake_form', f: f, readonly: true diff --git a/app/views/he/clients/boston_covid_19.haml b/app/views/he/clients/boston_covid_19.haml index 78709e5315b..b9880a29b13 100644 --- a/app/views/he/clients/boston_covid_19.haml +++ b/app/views/he/clients/boston_covid_19.haml @@ -1,4 +1,4 @@ -- title = @client.full_name +- title = @client.pii_provider(user: current_user).full_name - content_for :title, title = render 'clients/breadcrumbs' diff --git a/app/views/he/clients/boston_covid_19/_isolation_form.haml b/app/views/he/clients/boston_covid_19/_isolation_form.haml index 05a5ada702b..78f6fca3c83 100644 --- a/app/views/he/clients/boston_covid_19/_isolation_form.haml +++ b/app/views/he/clients/boston_covid_19/_isolation_form.haml @@ -2,7 +2,7 @@ .row - if can_edit_health_emergency_clinical? .col-md-8 - = f.input :isolation_requested_at, as: :string, label: "When was #{@client.name} asked to isolate?", input_html: { class: :date_time_picker } + = f.input :isolation_requested_at, as: :string, label: "When was #{@client.pii_provider(user: current_user).full_name} asked to isolate?", input_html: { class: :date_time_picker } = f.input :location, label: 'Isolation location', collection: @isolation.location_options, input_html: { data: {tags: true} }, as: :select_two = f.input :started_on, as: :date_picker, label: 'Isolation start date' = f.input :scheduled_to_end_on, as: :date_picker, label: 'Scheduled isolation end date' @@ -14,4 +14,4 @@ = render "he/clients/#{health_emergency}/previous_isolation" - elsif can_see_health_emergency_clinical? .col - = render "he/clients/#{health_emergency}/previous_isolation" \ No newline at end of file + = render "he/clients/#{health_emergency}/previous_isolation" diff --git a/app/views/he/clients/boston_covid_19/_quarantine_form.haml b/app/views/he/clients/boston_covid_19/_quarantine_form.haml index d3ad642078f..c94f227130b 100644 --- a/app/views/he/clients/boston_covid_19/_quarantine_form.haml +++ b/app/views/he/clients/boston_covid_19/_quarantine_form.haml @@ -2,7 +2,7 @@ .row - if can_edit_health_emergency_clinical? .col-md-8 - = f.input :isolation_requested_at, as: :string, label: "When was #{@client.name} asked to quarantine?", input_html: { class: :date_time_picker } + = f.input :isolation_requested_at, as: :string, label: "When was #{@client.pii_provider(user: current_user).full_name} asked to quarantine?", input_html: { class: :date_time_picker } = f.input :location, label: 'Quarantine location', collection: @quarantine.location_options, input_html: { data: {tags: true} }, as: :select_two = f.input :started_on, as: :date_picker, label: 'Quarantine start date' = f.input :scheduled_to_end_on, as: :date_picker, label: 'Scheduled quarantine end date' @@ -15,4 +15,3 @@ - elsif can_see_health_emergency_clinical? .col = render "he/clients/#{health_emergency}/previous_quarantine" - diff --git a/app/views/he/clients/boston_covid_19/_triage_form.haml b/app/views/he/clients/boston_covid_19/_triage_form.haml index c9efac15838..ea87c524666 100644 --- a/app/views/he/clients/boston_covid_19/_triage_form.haml +++ b/app/views/he/clients/boston_covid_19/_triage_form.haml @@ -4,8 +4,8 @@ .col-md-8 = f.input :agency, label: 'Where do you work?', collection: Agency.all, as: :select_two, selected: @triage.agency&.id = f.input :location, label: 'Where is this being collected?' - = f.input :exposure, collection: @triage.exposure_options, include_blank: '', label: "Has #{@client.name} been exposed?", as: :boolean_button_group - = f.input :symptoms, collection: @triage.symptom_options, include_blank: '', label: "Does #{@client.name} have symptoms?", as: :boolean_button_group + = f.input :exposure, collection: @triage.exposure_options, include_blank: '', label: "Has #{@client.pii_provider(user: current_user).full_name} been exposed?", as: :boolean_button_group + = f.input :symptoms, collection: @triage.symptom_options, include_blank: '', label: "Does #{@client.pii_provider(user: current_user).full_name} have symptoms?", as: :boolean_button_group = f.input :first_symptoms_on, label: 'First symptom date', as: :date_picker = f.input :referred_on, as: :date_picker = f.input :referred_to @@ -16,4 +16,4 @@ = render "he/clients/#{health_emergency}/previous_triage" - elsif can_see_health_emergency_screening? .col - = render "he/clients/#{health_emergency}/previous_triage" \ No newline at end of file + = render "he/clients/#{health_emergency}/previous_triage" diff --git a/app/views/he/clients/covid_19_vaccinations_only.haml b/app/views/he/clients/covid_19_vaccinations_only.haml index d9a87b94f1f..ff3ef6fd477 100644 --- a/app/views/he/clients/covid_19_vaccinations_only.haml +++ b/app/views/he/clients/covid_19_vaccinations_only.haml @@ -1,4 +1,4 @@ -- title = @client.full_name +- title = @client.pii_provider(user: current_user).full_name - content_for :title, title = render 'clients/breadcrumbs' diff --git a/app/views/projects/_clients.haml b/app/views/projects/_clients.haml index f2ad3995425..2d0b69e9e97 100644 --- a/app/views/projects/_clients.haml +++ b/app/views/projects/_clients.haml @@ -14,7 +14,8 @@ %tr %td - if ! @project.confidential? || can_edit_projects? || ! project.confidential_for_user?(user) - = link_to_if can_view_clients?, service.client.name, client_path(service.client) + - name = service.client.pii_provider(user: current_user).full_name + = link_to_if can_view_clients?, name, client_path(service.client) - else Confidential Client %td= service.first_date_in_program diff --git a/app/views/source_clients/edit.haml b/app/views/source_clients/edit.haml index cb7bf7245a4..2781101db4f 100644 --- a/app/views/source_clients/edit.haml +++ b/app/views/source_clients/edit.haml @@ -1,4 +1,4 @@ -- title = "Edit #{@client.name} at #{@client.data_source.name}" +- title = "Edit #{@client.pii_provider(user: current_user).full_name} at #{@client.data_source.name}" - content_for :modal_title, title = simple_form_for(@client, as: :client, url: polymorphic_path(source_client_path_generator, id: @client.id)) do |f| .row diff --git a/app/views/source_data/show.haml b/app/views/source_data/show.haml index e61de59b954..62b6b0a22a3 100644 --- a/app/views/source_data/show.haml +++ b/app/views/source_data/show.haml @@ -1,4 +1,10 @@ - title = "HMIS #{@type} Source Data" + +- pii = nil +- redact = {} +- if @item.class.name == 'GrdaWarehouse::Hud::Client' + - pii = @item.pii_provider(user: current_user) + - content_for :title, title %h1= content_for :title - if @hmis @@ -82,10 +88,36 @@ - value = link_to @item[key], source_data_path(search: {id: @item[key], type: 'Organization'}) - elsif key == :ProjectID && @item.class.name != 'GrdaWarehouse::Hud::Project' - value = link_to @item[key], source_data_path(search: {id: @item[key], type: 'Project'}) + - elsif key == :FirstName && pii + - value = pii.first_name + - elsif key == :MiddleName && pii + - value = pii.middle_name + - elsif key == :LastName && pii + - value = pii.last_name + - elsif key == :SSN && pii + - value = pii.ssn + - elsif key == :DOB && pii + - value = pii.dob_and_age %tr %th= key %td= value - if @imported - %td= @imported[key] + %td + - if key.in?([:FirstName, :MiddleName, :LastName]) + = GrdaWarehouse::PiiProvider.viewable_name(@imported[key], policy: pii.policy) + - elsif key.in?([:SSN]) + = GrdaWarehouse::PiiProvider.viewable_ssn(@imported[key], policy: pii.policy) + - elsif key.in?([:DOB]) + = GrdaWarehouse::PiiProvider.viewable_dob(@imported[key], policy: pii.policy) + - else + = @imported[key] - if @csv - %td= @csv[key] + %td + - if key.in?([:FirstName, :MiddleName, :LastName]) + = GrdaWarehouse::PiiProvider.viewable_name(@csv[key], policy: pii.policy) + - elsif key.in?([:SSN]) + = GrdaWarehouse::PiiProvider.viewable_ssn(@csv[key], policy: pii.policy) + - elsif key.in?([:DOB]) + = GrdaWarehouse::PiiProvider.viewable_dob(@csv[key], policy: pii.policy) + - else + = @csv[key] diff --git a/app/views/warehouse_reports/anomalies/_section.haml b/app/views/warehouse_reports/anomalies/_section.haml index eef30ea8d66..0455b3ccd64 100644 --- a/app/views/warehouse_reports/anomalies/_section.haml +++ b/app/views/warehouse_reports/anomalies/_section.haml @@ -18,7 +18,7 @@ %tr %td - if anomaly.client.present? - = link_to_if can_view_clients?, anomaly.client.name, appropriate_client_path(anomaly.client) + = link_to_if can_view_clients?, anomaly.client.pii_provider(user: current_user).full_name, appropriate_client_path(anomaly.client) - else Client no longer available %td diff --git a/app/views/warehouse_reports/cas/health_prioritization/index.haml b/app/views/warehouse_reports/cas/health_prioritization/index.haml index 50d35d82a6c..207c8bb183f 100644 --- a/app/views/warehouse_reports/cas/health_prioritization/index.haml +++ b/app/views/warehouse_reports/cas/health_prioritization/index.haml @@ -30,7 +30,7 @@ %tbody - @clients.each do |client| %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) + %td= link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) %td= client.age %td.text-center - disability = @disabilities.key?(client.id) diff --git a/app/views/warehouse_reports/cas/health_prioritization/index.xlsx.axlsx b/app/views/warehouse_reports/cas/health_prioritization/index.xlsx.axlsx index 777cd92682d..ff67175fd1a 100644 --- a/app/views/warehouse_reports/cas/health_prioritization/index.xlsx.axlsx +++ b/app/views/warehouse_reports/cas/health_prioritization/index.xlsx.axlsx @@ -30,7 +30,7 @@ wb.add_worksheet(name: 'Health Prioritization') do |sheet| row = [ client.id, - ::GrdaWarehouse::Config.get(:include_pii_in_detail_downloads) ? client.name : 'Redacted', + ::GrdaWarehouse::Config.get(:include_pii_in_detail_downloads) ? client.pii_provider(user: current_user).full_name : 'Redacted', client.age, disability, vispdat_disability, diff --git a/app/views/warehouse_reports/ce_assessments/index.haml b/app/views/warehouse_reports/ce_assessments/index.haml index 04f931d22fd..4772843c0dd 100644 --- a/app/views/warehouse_reports/ce_assessments/index.haml +++ b/app/views/warehouse_reports/ce_assessments/index.haml @@ -21,9 +21,10 @@ %th Completed? %tbody - @clients.each do |client| + - pii = client.pii_provider(user: current_user) %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) - %td= client.DOB + %td= link_to_if can_view_clients?, pii.full_name, appropriate_client_path(client) + %td= pii.dob_or_age %td= client.ce_assessment.location %td= client.ce_assessment.score %td= client.ce_assessment.vulnerability_score diff --git a/app/views/warehouse_reports/conflicting_client_attributes/index.haml b/app/views/warehouse_reports/conflicting_client_attributes/index.haml index 93a9d9cc308..13b98959500 100644 --- a/app/views/warehouse_reports/conflicting_client_attributes/index.haml +++ b/app/views/warehouse_reports/conflicting_client_attributes/index.haml @@ -24,10 +24,11 @@ %th SSN %tbody - @clients.each do |client| + - pii = client.pii_provider(user: current_user) %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) - %td= dob_or_age client.DOB - %td= ssn client.SSN + %td= link_to_if can_view_clients?, pii.full_name, appropriate_client_path(client) + %td= pii.dob_or_age + %td= pii.ssn = render 'common/pagination_bottom', item_name: 'client' - else .none-found No clients found. diff --git a/app/views/warehouse_reports/consent/index.haml b/app/views/warehouse_reports/consent/index.haml index c1cd7bf1763..8bcbdc5682c 100644 --- a/app/views/warehouse_reports/consent/index.haml +++ b/app/views/warehouse_reports/consent/index.haml @@ -41,7 +41,7 @@ - columns_to_skip = @cohorts_for_unconfirmed.count %tr %td.pl-2 - = link_to_if can_view_clients?, client.full_name, appropriate_client_path(client) + = link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) %td.text-center - if current_user.can_confirm_housing_release? - if client.active_in_cas? diff --git a/app/views/warehouse_reports/dob_entry_same/index.haml b/app/views/warehouse_reports/dob_entry_same/index.haml index 25c3b27e6f3..9af6d7dfd79 100644 --- a/app/views/warehouse_reports/dob_entry_same/index.haml +++ b/app/views/warehouse_reports/dob_entry_same/index.haml @@ -19,10 +19,11 @@ %th Project %tbody - @clients.each do |client| + - pii = client.pii_provider(user: current_user) - enrollment = client.source_enrollments.select{ |m| m[:EntryDate] == client.DOB }.first %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) - %td= client.DOB + %td= link_to_if can_view_clients?, pii.full_name, appropriate_client_path(client) + %td= pii.dob_or_age %td= enrollment&.EntryDate %td= enrollment&.project&.name(current_user) = render 'common/pagination_bottom', item_name: 'client' diff --git a/app/views/warehouse_reports/dv_victim_service/_table.haml b/app/views/warehouse_reports/dv_victim_service/_table.haml index c31045d4515..ceb82b0c7b9 100644 --- a/app/views/warehouse_reports/dv_victim_service/_table.haml +++ b/app/views/warehouse_reports/dv_victim_service/_table.haml @@ -13,7 +13,7 @@ - client = source_client.destination_client %tr %td= client.id - %td= link_to_if can_view_clients?, source_client.name, appropriate_client_path(client.id) + %td= link_to_if can_view_clients?, source_client.pii_provider(user: current_user).full_name, appropriate_client_path(client.id) = render 'common/pagination_bottom', item_name: 'client' - else .none-found No clients found. diff --git a/app/views/warehouse_reports/expiring_consent/index.haml b/app/views/warehouse_reports/expiring_consent/index.haml index bd00e5616d3..7ce21f51d8e 100644 --- a/app/views/warehouse_reports/expiring_consent/index.haml +++ b/app/views/warehouse_reports/expiring_consent/index.haml @@ -18,7 +18,7 @@ - @unconfirmed.each do |client| %tr %td - = link_to_if can_view_clients?, client.full_name, appropriate_client_path(client) + = link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) %td = client.consent_form_signed_on + client.class.consent_validity_period %td @@ -40,7 +40,7 @@ - @expiring_clients.each do |client| %tr %td - = link_to_if can_view_clients?, client.full_name, appropriate_client_path(client) + = link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) %td = client.consent_form_signed_on + client.class.consent_validity_period %td @@ -62,7 +62,7 @@ - @expired_clients.each do |client| %tr %td - = link_to_if can_view_clients?, client.full_name, appropriate_client_path(client) + = link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) %td = client.consent_form_signed_on + client.class.consent_validity_period %td diff --git a/app/views/warehouse_reports/health_emergency/medical_restrictions/_table.haml b/app/views/warehouse_reports/health_emergency/medical_restrictions/_table.haml index 292c755ed31..0b606471d39 100644 --- a/app/views/warehouse_reports/health_emergency/medical_restrictions/_table.haml +++ b/app/views/warehouse_reports/health_emergency/medical_restrictions/_table.haml @@ -21,15 +21,16 @@ %tbody - @restrictions.each do |restriction| - client = restriction.client + - pii = client.pii_provider(user: current_user) - batch_class = if restriction.in_batch?(params[:batch_id]) then 'report-hightlight' else '' end %tr %td{class: batch_class} - if @html && client.image .client__image{ style: "background-image: url(#{ image_client_path(client) })" } %td= link_to_if @html, client.id, polymorphic_path(['client_he', health_emergency], client_id: client) - %td= link_to_if @html, client.name, polymorphic_path(['client_he', health_emergency], client_id: client) - %td= client.DOB - %td= client.age + %td= link_to_if @html, pii.full_name, polymorphic_path(['client_he', health_emergency], client_id: client) + %td= pii.dob + %td= pii.age %td= restriction.created_at.to_date %td= simple_format(restriction.notes || restriction.note) - if @html diff --git a/app/views/warehouse_reports/inactive_youth_intakes/_table.haml b/app/views/warehouse_reports/inactive_youth_intakes/_table.haml index b2a94619d96..7efd28d22dc 100644 --- a/app/views/warehouse_reports/inactive_youth_intakes/_table.haml +++ b/app/views/warehouse_reports/inactive_youth_intakes/_table.haml @@ -16,7 +16,7 @@ %tbody - @report.clients.each do |client| %tr - %td= link_to client.client.name, client_youth_intakes_path(client.client) + %td= link_to client.client.pii_provider(user: current_user).full_name, client_youth_intakes_path(client.client) %td= client.intake.engagement_date - highlight_class = if client.case_mangement.present? && client.case_mangement == client.max_date then 'table-warning' else '' end %td{class: highlight_class}= client.case_mangement diff --git a/app/views/warehouse_reports/incomes/index.haml b/app/views/warehouse_reports/incomes/index.haml index 380e259001d..617fc4db7f2 100644 --- a/app/views/warehouse_reports/incomes/index.haml +++ b/app/views/warehouse_reports/incomes/index.haml @@ -33,7 +33,7 @@ - @enrollments.each do |record| %tr %td - = link_to_if can_view_clients?, record.client.name, appropriate_client_path(record.client) + = link_to_if can_view_clients?, record.client.pii_provider(user: current_user).full_name, appropriate_client_path(record.client) %td %ul.list-unstyled - record.enrollment.income_benefits_at_entry&.sources_and_amounts&.each do |name, amount| diff --git a/app/views/warehouse_reports/outflow/details.haml b/app/views/warehouse_reports/outflow/details.haml index 019b215cbe0..ef22dde1a3e 100644 --- a/app/views/warehouse_reports/outflow/details.haml +++ b/app/views/warehouse_reports/outflow/details.haml @@ -31,7 +31,7 @@ %td{rowspan: enrollments.count+1} = link_to_if can_view_clients?, client.id, appropriate_client_path(client.id) %td{rowspan: enrollments.count+1} - = link_to_if can_view_clients?, client.name, appropriate_client_path(client.id) + = link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client.id) - enrollments.each do |enrollment| %tr %td= enrollment.project&.name(current_user) diff --git a/app/views/warehouse_reports/recidivism/index.haml b/app/views/warehouse_reports/recidivism/index.haml index b826d420595..e14a82f88c1 100644 --- a/app/views/warehouse_reports/recidivism/index.haml +++ b/app/views/warehouse_reports/recidivism/index.haml @@ -25,7 +25,7 @@ .warehouse-reports__client .warehouse-reports__client-name %h4 - = link_to_if can_view_clients?, client.full_name, appropriate_client_path(client) + = link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) .row .col-sm-6 %h4 PH Enrollments diff --git a/app/views/warehouse_reports/time_homeless_for_exits/_table.haml b/app/views/warehouse_reports/time_homeless_for_exits/_table.haml index 77675c6d78a..103bae6e8e4 100644 --- a/app/views/warehouse_reports/time_homeless_for_exits/_table.haml +++ b/app/views/warehouse_reports/time_homeless_for_exits/_table.haml @@ -15,7 +15,7 @@ %tbody - @report.data.each do |client| %tr - %td= link_to_if can_view_clients?, client.client.name, appropriate_client_path(client.client) + %td= link_to_if can_view_clients?, client.client.pii_provider(user: current_user).full_name, appropriate_client_path(client.client) %td= client.days %td= client.entry_date %td= client.exit_date diff --git a/app/views/warehouse_reports/youth_activity/index.haml b/app/views/warehouse_reports/youth_activity/index.haml index 3e257cdcb63..e139c98959b 100644 --- a/app/views/warehouse_reports/youth_activity/index.haml +++ b/app/views/warehouse_reports/youth_activity/index.haml @@ -12,7 +12,7 @@ - else %p No modifications were made to youth records between #{@filter.start} and #{@filter.end}. - @clients.find_each do |client| - %h3= client.name + %h3= client.pii_provider(user: current_user).full_name .c-card.mb-4 .c-card__content - # We've preloaded all, use ruby select to pick those in range diff --git a/drivers/client_access_control/app/controllers/client_access_control/history_controller.rb b/drivers/client_access_control/app/controllers/client_access_control/history_controller.rb index cfe3154dc12..23b859cd8a6 100644 --- a/drivers/client_access_control/app/controllers/client_access_control/history_controller.rb +++ b/drivers/client_access_control/app/controllers/client_access_control/history_controller.rb @@ -111,7 +111,7 @@ def client_needing_processing?(client: @client) end private def title_for_show - "#{@client.name} - Service History" + "#{@client.pii_provider(user: current_user).full_name} - Service History" end end end diff --git a/drivers/client_access_control/app/views/client_access_control/clients/enrollment_details.haml b/drivers/client_access_control/app/views/client_access_control/clients/enrollment_details.haml index 3d2f8239b52..77049d6cde6 100644 --- a/drivers/client_access_control/app/views/client_access_control/clients/enrollment_details.haml +++ b/drivers/client_access_control/app/views/client_access_control/clients/enrollment_details.haml @@ -1,4 +1,4 @@ -- title = @client.full_name +- title = @client.pii_provider(user: current_user).full_name - content_for :title, title = render 'clients/breadcrumbs' diff --git a/drivers/client_access_control/app/views/client_access_control/clients/simple.haml b/drivers/client_access_control/app/views/client_access_control/clients/simple.haml index a0c0c970fde..fbc99faa19e 100644 --- a/drivers/client_access_control/app/views/client_access_control/clients/simple.haml +++ b/drivers/client_access_control/app/views/client_access_control/clients/simple.haml @@ -1,4 +1,4 @@ -- title = @client.full_name +- title = @client.pii_provider(user: current_user).full_name - content_for :title, title = render 'clients/breadcrumbs' = render 'clients/tab_navigation', current: simple_client_path(@client) diff --git a/drivers/client_documents_report/app/views/client_documents_report/warehouse_reports/reports/index.haml b/drivers/client_documents_report/app/views/client_documents_report/warehouse_reports/reports/index.haml index da834cb937c..59d6a622cea 100644 --- a/drivers/client_documents_report/app/views/client_documents_report/warehouse_reports/reports/index.haml +++ b/drivers/client_documents_report/app/views/client_documents_report/warehouse_reports/reports/index.haml @@ -46,7 +46,7 @@ %tbody - @clients.each do |client| %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) + %td= link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) %td= @report.required_documents(client).count %td= @report.optional_documents(client).count %td= @report.overall_documents(client).count diff --git a/drivers/client_location_history/app/models/client_location_history/location.rb b/drivers/client_location_history/app/models/client_location_history/location.rb index 30474f46d46..56c7bcc07de 100644 --- a/drivers/client_location_history/app/models/client_location_history/location.rb +++ b/drivers/client_location_history/app/models/client_location_history/location.rb @@ -47,9 +47,9 @@ def as_marker(user = nil, label_attributes = [:seen_on, :collected_by]) private def name_for_label(user) if user.can_view_clients? - link_for(client_path(client), client.name) + link_for(client_path(client), client.pii_provider(user: current_user).full_name) else - client.name + client.pii_provider(user: current_user).full_name end end diff --git a/drivers/client_location_history/app/views/client_location_history/clients/map.haml b/drivers/client_location_history/app/views/client_location_history/clients/map.haml index 0fe4ca0e339..000679b5638 100644 --- a/drivers/client_location_history/app/views/client_location_history/clients/map.haml +++ b/drivers/client_location_history/app/views/client_location_history/clients/map.haml @@ -1,4 +1,4 @@ -- title = "Location Map for #{@client.name}" +- title = "Location Map for #{@client.pii_provider(user: current_user).full_name}" - content_for :title, title = render 'clients/breadcrumbs' diff --git a/drivers/financial/app/views/financial/clients/rollup/_financial_clients.haml b/drivers/financial/app/views/financial/clients/rollup/_financial_clients.haml index b9a8102820c..b16bb7d4d5b 100644 --- a/drivers/financial/app/views/financial/clients/rollup/_financial_clients.haml +++ b/drivers/financial/app/views/financial/clients/rollup/_financial_clients.haml @@ -17,7 +17,7 @@ %tbody - clients.each do |client| %tr - %td= client.name + %td= client.pii_provider(user: current_user).full_name %td= yes_no(client.head_of_household == 1) %td.date-cell= client.date_of_referral_to_agency&.to_date %td.date-cell= client.date_of_referral_to_wit&.to_date diff --git a/drivers/financial/app/views/financial/clients/show.haml b/drivers/financial/app/views/financial/clients/show.haml index d3e9b3a03af..e091d07f325 100644 --- a/drivers/financial/app/views/financial/clients/show.haml +++ b/drivers/financial/app/views/financial/clients/show.haml @@ -1,4 +1,4 @@ -- title = @client.full_name +- title = @client.pii_provider(user: current_user).full_name - content_for :title, title = render 'clients/breadcrumbs' diff --git a/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml b/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml index 9dbad18b3da..c2996fe0105 100644 --- a/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml +++ b/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml @@ -19,10 +19,11 @@ %th Most-Recent CE Assessor %tbody - @clients.each do |client| + - pii = client.pii_provider(user: current_user) - projects = client.last_intentional_contacts( current_user, include_confidential_names: false, include_dates: true).select(&:present?) %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) - %td= dob_or_age(client.dob) + %td= link_to_if can_view_clients?, pii.full_name, appropriate_client_path(client) + %td= pii.dob_or_age %td - projects.each do |p| = p diff --git a/drivers/start_date_dq/app/views/start_date_dq/warehouse_reports/reports/index.haml b/drivers/start_date_dq/app/views/start_date_dq/warehouse_reports/reports/index.haml index f4458eefd36..91bffd43496 100644 --- a/drivers/start_date_dq/app/views/start_date_dq/warehouse_reports/reports/index.haml +++ b/drivers/start_date_dq/app/views/start_date_dq/warehouse_reports/reports/index.haml @@ -22,7 +22,7 @@ - @enrollments.each do |row| - client = row.client %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) + %td= link_to_if can_view_clients?, client.pii_provider(user: current_user).full_name, appropriate_client_path(client) - @report.column_values(row, current_user).each do |key, value| - next if key == :project_type - if key.in?([:days_between, :days_between_start_and_exit]) From fdcc6431677d258bfb80d5e0b3cd59840be7b4a5 Mon Sep 17 00:00:00 2001 From: Elliot Date: Mon, 3 Feb 2025 19:47:41 -0500 Subject: [PATCH 08/18] Silence Ruby EOL (#5105) --- config/brakeman.ignore | 101 ++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 41 deletions(-) diff --git a/config/brakeman.ignore b/config/brakeman.ignore index dbd822df41b..d618ca11a8a 100644 --- a/config/brakeman.ignore +++ b/config/brakeman.ignore @@ -156,7 +156,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/grda_warehouse/hud/project.rb", - "line": 322, + "line": 314, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "where(\"#{access_to_project_through_viewable_entities(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)} OR #{access_to_project_through_organization(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)} OR #{access_to_project_through_data_source(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)} OR #{access_to_project_through_coc_codes(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)} OR #{access_to_project_through_project_access_groups(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)}\")", "render_path": null, @@ -249,7 +249,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/bi/view_maintainer.rb", - "line": 396, + "line": 399, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "GrdaWarehouseBase.connection.execute(\"DROP VIEW IF EXISTS #{name}\")", "render_path": null, @@ -364,7 +364,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/bi/view_maintainer.rb", - "line": 416, + "line": 423, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "GrdaWarehouseBase.connection.execute(\"CREATE OR REPLACE VIEW #{name} AS #{sql_definition}\")", "render_path": null, @@ -410,7 +410,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 89, + "line": 93, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_inactive\" => true)", "render_path": null, @@ -433,7 +433,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 77, + "line": 81, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_first_time\" => true)", "render_path": null, @@ -502,7 +502,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 18, + "line": 22, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "spm_leaver(period).where(\"#{period}_period_days_to_return\" => (1..731))", "render_path": null, @@ -571,7 +571,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/hmis_external_apis/app/models/hmis_external_apis/tc_hmis/importers/importer.rb", - "line": 105, + "line": 111, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "Hmis::Hud::Base.connection.exec_query(\"ANALYZE #{table_names.uniq.map do\n Hmis::Hud::Base.connection.quote_table_name(n)\n end.join(\",\")};\")", "render_path": null, @@ -594,7 +594,7 @@ "check_name": "Execute", "message": "Possible command injection", "file": "drivers/hmis/app/models/hmis/form/definition.rb", - "line": 365, + "line": 366, "link": "https://brakemanscanner.org/docs/warning_types/command_injection/", "code": "`No Definition found for System form #{role}`", "render_path": null, @@ -697,7 +697,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 37, + "line": 41, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_entering_housing\" => true)", "render_path": null, @@ -720,7 +720,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 29, + "line": 33, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_caper_leaver\" => true)", "render_path": null, @@ -820,7 +820,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 18, + "line": 22, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "spm_leaver(period).where(\"#{period}_period_days_to_return\" => (1..731))", "render_path": null, @@ -866,7 +866,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 77, + "line": 81, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_first_time\" => true)", "render_path": null, @@ -979,7 +979,7 @@ "check_name": "Execute", "message": "Possible command injection", "file": "app/jobs/worker_status.rb", - "line": 114, + "line": 109, "link": "https://brakemanscanner.org/docs/warning_types/command_injection/", "code": "`curl #{ENV[\"ECS_CONTAINER_METADATA_URI_V4\"]}/task`", "render_path": null, @@ -1025,7 +1025,7 @@ "check_name": "UnsafeReflection", "message": "Unsafe reflection method `constantize` called on model attribute", "file": "drivers/hmis/app/models/hmis/form/definition.rb", - "line": 420, + "line": 431, "link": "https://brakemanscanner.org/docs/warning_types/remote_code_execution/", "code": "{ :SERVICE => ({ :owner_class => \"Hmis::Hud::HmisService\", :permission => :can_edit_enrollments }), :PROJECT => ({ :owner_class => \"Hmis::Hud::Project\", :permission => :can_edit_project_details }), :ORGANIZATION => ({ :owner_class => \"Hmis::Hud::Organization\", :permission => :can_edit_organization }), :CLIENT => ({ :owner_class => \"Hmis::Hud::Client\", :permission => :can_edit_clients }), :FUNDER => ({ :owner_class => \"Hmis::Hud::Funder\", :permission => :can_edit_project_details }), :INVENTORY => ({ :owner_class => \"Hmis::Hud::Inventory\", :permission => :can_edit_project_details }), :PROJECT_COC => ({ :owner_class => \"Hmis::Hud::ProjectCoc\", :permission => :can_edit_project_details }), :HMIS_PARTICIPATION => ({ :owner_class => \"Hmis::Hud::HmisParticipation\", :permission => :can_edit_project_details }), :CE_PARTICIPATION => ({ :owner_class => \"Hmis::Hud::CeParticipation\", :permission => :can_edit_project_details }), :CE_ASSESSMENT => ({ :owner_class => \"Hmis::Hud::Assessment\", :permission => :can_edit_enrollments }), :CE_EVENT => ({ :owner_class => \"Hmis::Hud::Event\", :permission => :can_edit_enrollments }), :CASE_NOTE => ({ :owner_class => \"Hmis::Hud::CustomCaseNote\", :permission => :can_edit_enrollments }), :FILE => ({ :owner_class => \"Hmis::File\", :permission => ([:can_manage_any_client_files, :can_manage_own_client_files]), :authorize => (lambda do\n Hmis::File.authorize_proc.call(entity_base, user)\n end) }), :REFERRAL_REQUEST => ({ :owner_class => \"HmisExternalApis::AcHmis::ReferralRequest\", :permission => :can_manage_incoming_referrals }), :REFERRAL => ({ :owner_class => \"HmisExternalApis::AcHmis::ReferralPosting\", :permission => :can_manage_outgoing_referrals }), :CURRENT_LIVING_SITUATION => ({ :owner_class => \"Hmis::Hud::CurrentLivingSituation\", :permission => :can_edit_enrollments }), :OCCURRENCE_POINT => ({ :owner_class => \"Hmis::Hud::Enrollment\", :permission => :can_edit_enrollments }), :ENROLLMENT => ({ :owner_class => \"Hmis::Hud::Enrollment\", :permission => :can_edit_enrollments }), :NEW_CLIENT_ENROLLMENT => ({ :permission => :can_edit_enrollments, :owner_class => \"Hmis::Hud::Enrollment\" }), :CLIENT_DETAIL => ({ :owner_class => \"Hmis::Hud::Client\", :permission => :can_edit_clients }), :EXTERNAL_FORM => ({ :owner_class => \"HmisExternalApis::ExternalForms::FormSubmission\", :permission => :can_manage_external_form_submissions }) }[role.to_sym][:owner_class].constantize", "render_path": null, @@ -1140,7 +1140,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "lib/rds_sql_server/rds.rb", - "line": 244, + "line": 260, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "SqlServerBootstrapModel.connection.execute(\"if not exists(select * from sys.databases where name = '#{database}')\\n select 0;\\nelse\\n select 1;\\n\")", "render_path": null, @@ -1186,7 +1186,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 14, + "line": 18, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "where(\"include_in_#{period}_period\" => true)", "render_path": null, @@ -1209,7 +1209,7 @@ "check_name": "Redirect", "message": "Possible unprotected redirect", "file": "app/controllers/user_training_controller.rb", - "line": 56, + "line": 75, "link": "https://brakemanscanner.org/docs/warning_types/redirect/", "code": "redirect_to(Talentlms::Facade.new(current_user).course_url(course.config, course.courseid, (clients_url or root_url), logout_talentlms_url), :allow_other_host => true)", "render_path": null, @@ -1282,6 +1282,25 @@ ], "note": "Injection from internal sources" }, + { + "warning_type": "Unmaintained Dependency", + "warning_code": 123, + "fingerprint": "715ee6d743a8af33c7b930d728708ce19c765fb40e2ad9d2b974db04d92dc7d1", + "check_name": "EOLRuby", + "message": "Support for Ruby 3.1.6 ends on 2025-03-31", + "file": ".ruby-version", + "line": 1, + "link": "https://brakemanscanner.org/docs/warning_types/unmaintained_dependency/", + "code": null, + "render_path": null, + "location": null, + "user_input": null, + "confidence": "Weak", + "cwe_id": [ + 1104 + ], + "note": "" + }, { "warning_type": "SQL Injection", "warning_code": 0, @@ -1289,7 +1308,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/collection.rb", - "line": 92, + "line": 98, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "where(\"#{quoted_table_name}.coc_codes ?| #{SqlHelper.quote_sql_array(coc_codes, :type => :varchar)}\")", "render_path": null, @@ -1458,7 +1477,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 37, + "line": 41, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_entering_housing\" => true)", "render_path": null, @@ -1481,7 +1500,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 85, + "line": 89, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_entering_housing\" => true)", "render_path": null, @@ -1527,7 +1546,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 81, + "line": 85, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_reentering\" => true)", "render_path": null, @@ -1550,7 +1569,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 57, + "line": 61, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_in_outflow\" => true)", "render_path": null, @@ -1767,7 +1786,7 @@ "check_name": "ValidationRegex", "message": "Insufficient validation for `postal_code` using `/\\A\\d{5}/`. Use `\\A` and `\\z` as anchors", "file": "drivers/hmis/app/models/hmis/hud/custom_client_address.rb", - "line": 108, + "line": 117, "link": "https://brakemanscanner.org/docs/warning_types/format_validation/", "code": null, "render_path": null, @@ -1835,7 +1854,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/grda_warehouse/data_source.rb", - "line": 88, + "line": 94, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "where(\"#{has_access_to_data_source_through_viewable_entities(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)} OR #{has_access_to_data_source_through_organizations(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)} OR #{has_access_to_data_source_through_projects(user, lambda do\n connection.quote(s)\n end, lambda do\n connection.quote_column_name(s)\n end)}\")", "render_path": null, @@ -1858,7 +1877,7 @@ "check_name": "UnsafeReflection", "message": "Unsafe reflection method `constantize` called on model attribute", "file": "app/controllers/cohorts/clients_controller.rb", - "line": 471, + "line": 473, "link": "https://brakemanscanner.org/docs/warning_types/remote_code_execution/", "code": "GrdaWarehouse::Cohort.available_columns.map(&:class).map(&:name).select do\n (m == params.require(:field))\n end.first.constantize", "render_path": null, @@ -1881,7 +1900,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 33, + "line": 37, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "where(\"#{period}_period_spm_leaver\" => true)", "render_path": null, @@ -1927,7 +1946,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/bi/view_maintainer.rb", - "line": 410, + "line": 414, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "GrdaWarehouseBase.connection.execute(\"DO $$\\nBEGIN\\n CREATE ROLE #{role} WITH NOLOGIN;\\n EXCEPTION WHEN DUPLICATE_OBJECT THEN\\n RAISE NOTICE 'not creating role #{role} -- it already exists';\\nEND\\n$$;\\n\")", "render_path": null, @@ -2007,7 +2026,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_measurement/app/models/performance_measurement/equity_analysis/data.rb", - "line": 211, + "line": 217, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "scope.where(\"#{period}_age\" => (age_params.map do\n Filters::FilterBase.age_range(d)\n end))", "render_path": null, @@ -2030,7 +2049,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/homeless_summary_report/app/models/homeless_summary_report/report.rb", - "line": 881, + "line": 899, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "clients.send(variant).send(\"spm_#{field}\").average(\"spm_#{field}\")", "render_path": null, @@ -2076,7 +2095,7 @@ "check_name": "Execute", "message": "Possible command injection", "file": "app/jobs/worker_status.rb", - "line": 72, + "line": 67, "link": "https://brakemanscanner.org/docs/warning_types/command_injection/", "code": "`curl -k https://#{ENV[\"FQDN\"]}/system_status/details`", "render_path": null, @@ -2132,7 +2151,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 89, + "line": 93, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_inactive\" => true)", "render_path": null, @@ -2155,7 +2174,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 14, + "line": 18, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "where(\"include_in_#{period}_period\" => true)", "render_path": null, @@ -2257,7 +2276,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 57, + "line": 61, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_in_outflow\" => true)", "render_path": null, @@ -2303,7 +2322,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 29, + "line": 33, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_caper_leaver\" => true)", "render_path": null, @@ -2326,7 +2345,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 33, + "line": 37, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "where(\"#{period}_period_spm_leaver\" => true)", "render_path": null, @@ -2372,7 +2391,7 @@ "check_name": "MassAssignment", "message": "Specify exact keys allowed for mass assignment instead of using `permit!` which allows any keys", "file": "app/controllers/application_controller.rb", - "line": 71, + "line": 91, "link": "https://brakemanscanner.org/docs/warning_types/mass_assignment/", "code": "params.permit!", "render_path": null, @@ -2521,7 +2540,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "lib/rds_sql_server/rds.rb", - "line": 258, + "line": 274, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "SqlServerBootstrapModel.connection.execute(\"if not exists(select * from sys.databases where name = '#{database}')\\n create database #{database}\\n\")", "render_path": null, @@ -2590,7 +2609,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 81, + "line": 85, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_reentering\" => true)", "render_path": null, @@ -2613,7 +2632,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "app/models/bi/view_maintainer.rb", - "line": 419, + "line": 427, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "GrdaWarehouseBase.connection.execute(\"GRANT SELECT ON #{name} TO #{\"bi\"}\")", "render_path": null, @@ -2774,7 +2793,7 @@ "check_name": "SQL", "message": "Possible SQL injection", "file": "drivers/performance_metrics/app/models/performance_metrics/client.rb", - "line": 85, + "line": 89, "link": "https://brakemanscanner.org/docs/warning_types/sql_injection/", "code": "served(period).where(\"#{period}_period_entering_housing\" => true)", "render_path": null, @@ -2860,6 +2879,6 @@ "note": "" } ], - "updated": "2024-10-29 12:24:27 +0000", + "updated": "2025-02-03 21:06:52 +0000", "brakeman_version": "6.2.1" } From a9b62bc4c98e5459b4a9c462d18788c46897ab0c Mon Sep 17 00:00:00 2001 From: Dave G <149399758+dtgreiner@users.noreply.github.com> Date: Tue, 4 Feb 2025 13:36:33 -0500 Subject: [PATCH 09/18] current_user issues in client activity report backgrounding (#5106) --- .../warehouse_reports/reports/_report.haml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml b/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml index 9dbad18b3da..c369e0a93d4 100644 --- a/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml +++ b/drivers/inactive_client_report/app/views/inactive_client_report/warehouse_reports/reports/_report.haml @@ -5,7 +5,7 @@ %table.table.table-striped.mb-0 %thead %tr - - age_label = if can_view_full_dob? then 'DOB' else 'Age' end + - age_label = if current_user.can_view_full_dob? then 'DOB' else 'Age' end %th Client %th= age_label %th Last Seen @@ -21,8 +21,8 @@ - @clients.each do |client| - projects = client.last_intentional_contacts( current_user, include_confidential_names: false, include_dates: true).select(&:present?) %tr - %td= link_to_if can_view_clients?, client.name, appropriate_client_path(client) - %td= dob_or_age(client.dob) + %td= link_to_if current_user.can_view_clients?, client.name, appropriate_client_path(client) + %td= client.pii_provider(user: current_user).dob_or_age %td - projects.each do |p| = p From 1a7b606173918e4abaaeec7b78d22665dfa5ca5b Mon Sep 17 00:00:00 2001 From: Elliot Date: Tue, 4 Feb 2025 15:01:19 -0500 Subject: [PATCH 10/18] Fix for using new find_safely (#5109) --- app/controllers/notification_configurations_controller.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/app/controllers/notification_configurations_controller.rb b/app/controllers/notification_configurations_controller.rb index 97104180c9c..245b7d577ee 100644 --- a/app/controllers/notification_configurations_controller.rb +++ b/app/controllers/notification_configurations_controller.rb @@ -53,11 +53,14 @@ def destroy helper_method :import_threshold def notification_configuration - @notification_configuration ||= GrdaWarehouse::NotificationConfiguration.find_safely(params[:id]) || + @notification_configuration ||= if params[:id].present? + GrdaWarehouse::NotificationConfiguration.find_safely(params[:id]) + else GrdaWarehouse::NotificationConfiguration.new( source: import_threshold, notification_slug: import_threshold.valid_notification_slug(params[:notification_slug]), ) + end end helper_method :notification_configuration end From a378d4757468571859be4edef4db51e1b33ccc64 Mon Sep 17 00:00:00 2001 From: Elliot Date: Tue, 4 Feb 2025 15:04:22 -0500 Subject: [PATCH 11/18] Access control fixes (#5108) * Don't fail on project group new; assign viewer permission when creating a cohort * access groups for new project group --------- Co-authored-by: Dave G --- app/controllers/cohorts_controller.rb | 3 ++- app/controllers/project_groups_controller.rb | 6 +++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/controllers/cohorts_controller.rb b/app/controllers/cohorts_controller.rb index 7e08462f680..2674c08ee21 100644 --- a/app/controllers/cohorts_controller.rb +++ b/app/controllers/cohorts_controller.rb @@ -142,6 +142,7 @@ def create # If the user doesn't have All Cohorts access, grant them access to the cohort @cohort.replace_access(current_user, scope: :editor) + @cohort.replace_access(current_user, scope: :viewer) # Always add the cohort to the system group AccessGroup.maintain_system_groups(group: :cohorts) # Add default tabs @@ -150,7 +151,7 @@ def create end end # Search the list so you can see the newly created cohort - redirect_to cohorts_path('q[name_cont]' => @cohort.name) + redirect_to cohorts_path('search_form[q]' => @cohort.name) rescue Exception => e flash[:error] = e.message redirect_to cohorts_path diff --git a/app/controllers/project_groups_controller.rb b/app/controllers/project_groups_controller.rb index f841976ab09..4224e40fddb 100644 --- a/app/controllers/project_groups_controller.rb +++ b/app/controllers/project_groups_controller.rb @@ -28,7 +28,7 @@ def index def new @project_group = project_group_source.new - set_access + set_group_access end def create @@ -166,6 +166,10 @@ def set_project_group def set_access @editor_ids = @project_group.editable_access_control.user_ids + set_group_access + end + + def set_group_access # TODO: START_ACL remove when ACL transition complete @groups = @project_group.access_groups @group_ids = @project_group.access_group_ids From f74fa4619800091c3c476aa7d562e3a69f5b6edf Mon Sep 17 00:00:00 2001 From: Dave G <149399758+dtgreiner@users.noreply.github.com> Date: Tue, 4 Feb 2025 15:32:02 -0500 Subject: [PATCH 12/18] hotfix for checkgin if spm enrollment has valid project type (#5103) --- .../app/models/homeless_summary_report/report.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/homeless_summary_report/app/models/homeless_summary_report/report.rb b/drivers/homeless_summary_report/app/models/homeless_summary_report/report.rb index 16e41c62c04..8d3d90696ef 100644 --- a/drivers/homeless_summary_report/app/models/homeless_summary_report/report.rb +++ b/drivers/homeless_summary_report/app/models/homeless_summary_report/report.rb @@ -508,7 +508,7 @@ def title_for(household_category, demographic_category) # this must be for a PH project, exluding RRH valid_project_types = HudUtility2024.permanent_housing_project_types - [HudUtility2024.project_type_number('PH - RRH')] - valid_project = valid_project_types.include?(spm_member.enrollment.project.project_type) + valid_project = valid_project_types.include?(spm_member.enrollment&.project&.project_type) valid_project && valid_move_in && valid_exit end From e284a34113575449c1b772b99a930dc6a09c5d71 Mon Sep 17 00:00:00 2001 From: Gig Date: Wed, 5 Feb 2025 07:54:44 -0500 Subject: [PATCH 13/18] Change handling of legacy occurrence point data (#5022) --- .../hmis_schema/occurrence_point_form.rb | 2 +- .../form/occurrence_point_form_collection.rb | 111 ++++++++++++++++++ .../hmis/app/models/hmis/hud/enrollment.rb | 45 +------ drivers/hmis/app/models/hmis/hud/project.rb | 12 +- .../spec/factories/hmis/form/definitions.rb | 2 +- .../spec/models/hmis/hud/enrollment_spec.rb | 86 ++++++++------ .../hmis/spec/models/hmis/hud/project_spec.rb | 13 +- 7 files changed, 178 insertions(+), 93 deletions(-) create mode 100644 drivers/hmis/app/models/hmis/form/occurrence_point_form_collection.rb diff --git a/drivers/hmis/app/graphql/types/hmis_schema/occurrence_point_form.rb b/drivers/hmis/app/graphql/types/hmis_schema/occurrence_point_form.rb index 6adad4a84a5..a937ff9ece3 100644 --- a/drivers/hmis/app/graphql/types/hmis_schema/occurrence_point_form.rb +++ b/drivers/hmis/app/graphql/types/hmis_schema/occurrence_point_form.rb @@ -14,7 +14,7 @@ class HmisSchema::OccurrencePointForm < Types::BaseObject # Form used for Viewing/Creating/Editing records field :definition, Types::Forms::FormDefinition, null: false, extras: [:parent] - # object is an OpenStruct, see Hmis::Hud::Enrollment occurrence_point_forms + # object is an OpenStruct, see Hmis::Form::OccurrencePointFormCollection def id(parent:) # Include project id (if present) so that instance is not cached for use across projects. diff --git a/drivers/hmis/app/models/hmis/form/occurrence_point_form_collection.rb b/drivers/hmis/app/models/hmis/form/occurrence_point_form_collection.rb new file mode 100644 index 00000000000..0a897425559 --- /dev/null +++ b/drivers/hmis/app/models/hmis/form/occurrence_point_form_collection.rb @@ -0,0 +1,111 @@ +### +# Copyright 2016 - 2025 Green River Data Analysis, LLC +# +# License detail: https://github.com/greenriver/hmis-warehouse/blob/production/LICENSE.md +### + +### +# Hmis::Form::OccurrencePointFormCollection +# +# This class is responsible for determining which Occurrence Point forms to display on a given Enrollment in HMIS. +# The Occurrence Point forms appear in the "Enrollment Details" card on the HMIS Enrollment dashboard. +# +# These forms collect data elements onto an Enrollment "at occurrence" (a.k.a. when they occur), +# as opposed to data elements that are collected at a specific point in time (e.g. at intake, exit). +### +class Hmis::Form::OccurrencePointFormCollection + # Struct that backs Types::HmisSchema::OccurrencePointForm + OccurrencePointForm = Struct.new(:definition, :legacy, :data_collected_about, keyword_init: true) + private_constant :OccurrencePointForm + + # Occurrence Point forms to display on the Enrollment, including legacy forms to show existing data + def for_enrollment(enrollment) + structs = active_for_enrollment(enrollment) + structs += legacy_for_enrollment(enrollment, active_forms: structs) + structs + end + + # Occurrence Point forms that are enabled in the Project. This is only used for purposes of displaying Project configuration. + def for_project(project) + occurrence_point_definition_scope.map do |definition| + # Choose the most specific Instance that enables this FormDefinition for this Project + best_instance = definition.instances.active.order(updated_at: :desc).detect_best_instance_for_project(project: project) + next unless best_instance + + create_form_struct( + definition: definition, + data_collected_about: best_instance.data_collected_about, + legacy: false, # not legacy, because there is an active Form Instance enabling it + ) + end.compact + end + + private + + # Occurrence Point forms that are enabled for this Enrollment via an active form instance + def active_for_enrollment(enrollment) + occurrence_point_definition_scope.map do |definition| + # Choose the most specific Instance that enables this FormDefinition for this Enrollment + best_instance = definition.instances.active.order(updated_at: :desc).detect_best_instance_for_enrollment(enrollment: enrollment) + # If there was no active instance, that means this Occurrence Point form is not enabled. Skip it. + next unless best_instance + + create_form_struct( + definition: definition, + data_collected_about: best_instance.data_collected_about, + legacy: false, # not legacy, because there is an active Form Instance enabling it + ) + end.compact + end + + # Default Occurrence Point forms that collect HUD fields. The system should already enforce that + # these forms are enabled for the appropriate projects (e.g. Move-in Date collected on HoH in PH). + # This code ensures that for contexts when the form ISN'T enabled (e.g. Move-in Date on a Child), + # AND the Enrollment has a value for the primary field it collects (e.g. 'MoveInDate'), we still show the value and the form. + # This allows users to see the full set of HUD occurrence point data elements, and do data correction. + HUD_DEFAULT_FORMS = [ + # Note: form_identifier matches the filename of the form, e.g. ../default/occurrence_point_forms/move_in_date.json + { form_identifier: :move_in_date, field_name: :move_in_date }, + { form_identifier: :date_of_engagement, field_name: :date_of_engagement }, + { form_identifier: :path_status, field_name: :date_of_path_status }, + ].freeze + + def legacy_for_enrollment(enrollment, active_forms:) + # Add legacy forms to ensure that HUD Data Elements are not hidden. + # In the event that an Enrollment has a MoveInDate, for example, but there is no active form that collects it, + # we still need to show it so that user can see the data and perform data correction. + HUD_DEFAULT_FORMS.map do |config| + form_identifier, field_name = config.values_at(:form_identifier, :field_name) + # this enrollment does not have this field (e.g. MoveInDate), skip + next unless enrollment.send(field_name).present? + # this field is already collected by an active enable form, skip + next if active_forms.find { |s| collects_enrollment_field?(s.definition, field_name) } + + definition = occurrence_point_definition_scope.find { |fd| fd.identifier == form_identifier.to_s && fd.managed_in_version_control? } + raise "Unexpected: #{field_name} present, but default form '#{form_identifier}' not found" unless definition + + create_form_struct(definition: definition, legacy: true) + end.compact + end + + def occurrence_point_definition_scope + @occurrence_point_definition_scope ||= Hmis::Form::Definition.with_role(:OCCURRENCE_POINT).published + end + + def create_form_struct(definition:, legacy:, data_collected_about: nil) + OccurrencePointForm.new( + definition: definition, + legacy: legacy, + data_collected_about: data_collected_about || 'ALL_CLIENTS', + ) + end + + # Check if the given FormDefinition collects the given field from the Enrollment. + # This is a bit hacky (transforming fieldname to graphql casing) but it works for the known fields (Move-in date, DOE, PATH). + def collects_enrollment_field?(definition, field_name) + normalized_field_name = field_name.to_s.camelize(:lower) + definition.link_id_item_hash.values.any? do |item| + item.mapping&.record_type == 'ENROLLMENT' && item.mapping&.field_name == normalized_field_name + end + end +end diff --git a/drivers/hmis/app/models/hmis/hud/enrollment.rb b/drivers/hmis/app/models/hmis/hud/enrollment.rb index bdd718e74e1..6083bd3f4f7 100644 --- a/drivers/hmis/app/models/hmis/hud/enrollment.rb +++ b/drivers/hmis/app/models/hmis/hud/enrollment.rb @@ -339,49 +339,10 @@ def data_collection_features end.compact end + # Occurrence Point Forms that are enabled for this Enrollment. + # Returns array of OpenStructs, which are resolved by the HmisSchema::OccurrencePointForm GQL type. def occurrence_point_forms - # Get definitions for Occurrence Point forms, including inactive/retired (but excluding drafts) - definitions = Hmis::Form::Definition.with_role(:OCCURRENCE_POINT).published_or_retired.latest_versions - # Get cdeds that this enrollment has CDE record(s) for. Do this in advance so we don't make extra trips to db - cdeds_this_enrollment_has = custom_data_element_definitions.pluck(:key).to_set - - definitions.map do |definition| - # Choose the most specific instance for this enrollment - best_instance = definition.instances.active.detect_best_instance_for_enrollment(enrollment: self) - - # Check for legacy data. Skip the calculation if there is a current instance - has_legacy_data = best_instance ? false : legacy_occurrence_point_data?(definition, cdeds_this_enrollment_has) - - next unless best_instance || has_legacy_data - - OpenStruct.new( - legacy: has_legacy_data && !best_instance, - definition: definition, - data_collected_about: best_instance&.data_collected_about || 'ALL_CLIENTS', - ) - end.compact - end - - private def legacy_occurrence_point_data?(definition, cdeds_this_enrollment_has) - definition.walk_definition_nodes(as_open_struct: true) do |item| - next unless item.mapping.present? - - record_type = item.mapping&.record_type - field_name = item.mapping&.field_name&.underscore - custom_field_key = item.mapping&.custom_field_key - - next unless record_type == 'ENROLLMENT' || custom_field_key - - if record_type && field_name - # Example: if this item collects `move_in_date` and the Enrollment has a Move-in Date value, then we want to show this form on the Enrollment Dashboard (even though it isn't "enabled" via an instance) - return true if respond_to?(field_name) && send(field_name).present? - elsif custom_field_key - # For simplicity, for now, just look for CDEDs where the owner is this Enrollment - return true if cdeds_this_enrollment_has.include?(custom_field_key) - end - end - - false + Hmis::Form::OccurrencePointFormCollection.new.for_enrollment(self) end def save_new_enrollment! diff --git a/drivers/hmis/app/models/hmis/hud/project.rb b/drivers/hmis/app/models/hmis/hud/project.rb index 8b9f730e305..8427cd22944 100644 --- a/drivers/hmis/app/models/hmis/hud/project.rb +++ b/drivers/hmis/app/models/hmis/hud/project.rb @@ -295,17 +295,7 @@ def available_service_types # Occurrence Point Form Instances that are enabled for this project (e.g. Move In Date form) def occurrence_point_form_instances - # All instances for Occurrence Point forms - base_scope = Hmis::Form::Instance.with_role(:OCCURRENCE_POINT).active.published - - # All possible form identifiers used for Occurrence Point collection - occurrence_point_identifiers = base_scope.pluck(:definition_identifier).uniq - - # Choose the most specific instance for each definition identifier - occurrence_point_identifiers.map do |identifier| - scope = base_scope.where(definition_identifier: identifier).order(updated_at: :desc) - scope.detect_best_instance_for_project(project: self) - end.compact + Hmis::Form::OccurrencePointFormCollection.new.for_project(self) end def uniq_coc_codes diff --git a/drivers/hmis/spec/factories/hmis/form/definitions.rb b/drivers/hmis/spec/factories/hmis/form/definitions.rb index 5204f962609..69c9b68d6a7 100644 --- a/drivers/hmis/spec/factories/hmis/form/definitions.rb +++ b/drivers/hmis/spec/factories/hmis/form/definitions.rb @@ -852,7 +852,7 @@ end factory :occurrence_point_form, parent: :hmis_form_definition do - identifier { 'move_in_date' } + identifier { 'move_in_date_form' } role { :OCCURRENCE_POINT } definition do JSON.parse(<<~JSON) diff --git a/drivers/hmis/spec/models/hmis/hud/enrollment_spec.rb b/drivers/hmis/spec/models/hmis/hud/enrollment_spec.rb index 2e25290811d..43eb7a92d0a 100644 --- a/drivers/hmis/spec/models/hmis/hud/enrollment_spec.rb +++ b/drivers/hmis/spec/models/hmis/hud/enrollment_spec.rb @@ -273,14 +273,14 @@ let(:legacy_expected_struct) do have_attributes( legacy: true, - definition: definition, + definition: have_attributes(identifier: 'move_in_date'), # default form seeded by JsonForms.seed_all data_collected_about: 'ALL_CLIENTS', ) end before(:all) do - Hmis::Form::Definition.delete_all - Hmis::Form::Instance.delete_all + # seed default FormDefinitions so that the default move_in_date form is present + ::HmisUtil::JsonForms.seed_all end it 'does not return the form when no instance exists' do @@ -297,7 +297,7 @@ expect(hoh_enrollment.occurrence_point_forms).to be_empty end - context 'when there is no instance, but there is legacy data' do + context 'when there is no instance, but Enrollment has a MoveInDate value' do let!(:spouse_enrollment) do create( :hmis_hud_enrollment, @@ -309,7 +309,7 @@ ) end - it 'does return the form' do + it 'does return the default move_in_date form' do expect(spouse_enrollment.occurrence_point_forms).to contain_exactly(legacy_expected_struct) end @@ -319,7 +319,7 @@ let!(:instance3) { create(:hmis_form_instance, role: role, project_type: 4, active: true, definition: definition) } let!(:inactive_instance) { create(:hmis_form_instance, role: role, project_type: 6, active: false, definition: definition) } - it 'returns the form, with no duplicates' do + it 'returns the default move_in_date form, with no duplicates' do expect(spouse_enrollment.occurrence_point_forms).to contain_exactly(legacy_expected_struct) end end @@ -327,7 +327,7 @@ context 'when a draft version of the form does not collect the same data' do let!(:draft_definition) { create(:occurrence_point_form, version: 2, status: :draft) } - it 'returns the form' do + it 'returns the default move_in_date form' do expect(spouse_enrollment.occurrence_point_forms).to contain_exactly(legacy_expected_struct) end end @@ -351,6 +351,7 @@ definition: definition, data_collected_about: 'ALL_CLIENTS', ) + # binding.pry expect(hoh_enrollment.occurrence_point_forms).to contain_exactly(expected) expect(spouse_enrollment.occurrence_point_forms).to contain_exactly(expected) end @@ -382,32 +383,7 @@ ) end - it 'returns the form for non-HoH' do - expect(spouse_enrollment.occurrence_point_forms).to contain_exactly(legacy_expected_struct) - end - end - - context 'when legacy data exists on a CDED' do - let!(:definition_json) do - { - 'item': [ - { - 'text': 'Foo data element', - 'type': 'STRING', - 'link_id': 'foo', - 'mapping': { - 'custom_field_key': 'foo', - 'record_type': 'ENROLLMENT', - }, - }, - ], - } - end - let!(:definition) { create :occurrence_point_form, definition: definition_json } - let!(:cded) { create :hmis_custom_data_element_definition, key: 'foo', data_source: ds1, owner_type: 'Hmis::Hud::Enrollment', repeats: false } - let!(:cde) { create :hmis_custom_data_element, data_element_definition: cded, owner: spouse_enrollment, data_source: ds1, value_string: 'bar' } - - it 'returns the form for non-HoH' do + it 'returns the default form for non-HoH' do expect(spouse_enrollment.occurrence_point_forms).to contain_exactly(legacy_expected_struct) end end @@ -428,5 +404,49 @@ end end end + + context 'PATH Status form' do + let(:legacy_expected_struct) do + have_attributes( + legacy: true, + definition: have_attributes(identifier: 'path_status'), # default form seeded by JsonForms.seed_all + data_collected_about: 'ALL_CLIENTS', + ) + end + + context 'when DateOfPATHStatus does not exist' do + it 'does not return PATH status form' do + expect(hoh_enrollment.occurrence_point_forms).to be_empty + end + end + context 'when DateOfPATHStatus exists' do + before(:each) { hoh_enrollment.update!(date_of_path_status: 3.weeks.ago) } + it 'returns the default PATH status form' do + expect(hoh_enrollment.occurrence_point_forms).to contain_exactly(legacy_expected_struct) + end + end + end + + context 'Date of Engagement form' do + let(:legacy_expected_struct) do + have_attributes( + legacy: true, + definition: have_attributes(identifier: 'date_of_engagement'), # default form seeded by JsonForms.seed_all + data_collected_about: 'ALL_CLIENTS', + ) + end + + context 'when DateOfEngagement does not exist' do + it 'does not return Date of engagement form' do + expect(hoh_enrollment.occurrence_point_forms).to be_empty + end + end + context 'when DateOfEngagement exists' do + before(:each) { hoh_enrollment.update!(date_of_engagement: 3.weeks.ago) } + it 'returns the default Date of engagement form' do + expect(hoh_enrollment.occurrence_point_forms).to contain_exactly(legacy_expected_struct) + end + end + end end end diff --git a/drivers/hmis/spec/models/hmis/hud/project_spec.rb b/drivers/hmis/spec/models/hmis/hud/project_spec.rb index 0dbfe7b02cd..80138635e75 100644 --- a/drivers/hmis/spec/models/hmis/hud/project_spec.rb +++ b/drivers/hmis/spec/models/hmis/hud/project_spec.rb @@ -209,13 +209,16 @@ def selected_instances end it 'returns most specific instance per definition identifier' do - mid_ptype = create(:hmis_form_instance, role: role, entity: nil, project_type: 13, definition_identifier: 'move_in_date') - mid_project = create(:hmis_form_instance, role: role, entity: project, definition_identifier: mid_ptype.definition_identifier) + mid_ptype = create(:hmis_form_instance, role: role, entity: nil, project_type: 13, definition_identifier: 'move_in_date', data_collected_about: 'HOH') + mid_project = create(:hmis_form_instance, role: role, entity: project, definition_identifier: mid_ptype.definition_identifier, data_collected_about: 'HOH_AND_ADULTS') - doe_default = create(:hmis_form_instance, role: role, entity: nil, definition_identifier: 'date_of_engagement') - doe_org = create(:hmis_form_instance, role: role, entity: project.organization, definition_identifier: doe_default.definition_identifier) + doe_default = create(:hmis_form_instance, role: role, entity: nil, definition_identifier: 'date_of_engagement', data_collected_about: 'ALL_CLIENTS') + doe_org = create(:hmis_form_instance, role: role, entity: project.organization, definition_identifier: doe_default.definition_identifier, data_collected_about: 'HOH') - expect(selected_instances).to contain_exactly(mid_project, doe_org) + expect(selected_instances).to contain_exactly( + have_attributes(definition: mid_project.definition, data_collected_about: mid_project.data_collected_about), + have_attributes(definition: doe_default.definition, data_collected_about: doe_org.data_collected_about), + ) end it 'does not return draft forms, even for active instances' do From b0eaa4a85ce80e720f6aa22692ec8ae8b1ff458f Mon Sep 17 00:00:00 2001 From: Gig Date: Wed, 5 Feb 2025 08:11:37 -0500 Subject: [PATCH 14/18] Add picklist `ADMIN_AVAILABLE_UNITS_FOR_ENROLLMENT` for AC unit changes (#5111) --- drivers/hmis/app/graphql/schema.graphql | 9 +++- .../types/forms/enums/pick_list_type.rb | 3 +- .../graphql/types/forms/pick_list_option.rb | 26 ++++++++-- .../hmis/app/models/hmis/hud/enrollment.rb | 1 + .../hmis/spec/requests/hmis/pick_list_spec.rb | 50 +++++++++++++------ 5 files changed, 68 insertions(+), 21 deletions(-) diff --git a/drivers/hmis/app/graphql/schema.graphql b/drivers/hmis/app/graphql/schema.graphql index 74cd01eb2b8..2e367a03449 100644 --- a/drivers/hmis/app/graphql/schema.graphql +++ b/drivers/hmis/app/graphql/schema.graphql @@ -6656,6 +6656,12 @@ type PickListOption { } enum PickListType { + """ + Units available for the given Enrollment at the given project. Includes all + available units at project even if they have a different type from what the + household is currently occupying. + """ + ADMIN_AVAILABLE_UNITS_FOR_ENROLLMENT ALL_SERVICE_CATEGORIES ALL_SERVICE_TYPES @@ -6680,7 +6686,8 @@ enum PickListType { AVAILABLE_SERVICE_TYPES """ - Units available for the given household at the given project + Units available for the given Enrollment at the given project. List is limited + to units with the same unit type currently occupied by the household, if any. """ AVAILABLE_UNITS_FOR_ENROLLMENT diff --git a/drivers/hmis/app/graphql/types/forms/enums/pick_list_type.rb b/drivers/hmis/app/graphql/types/forms/enums/pick_list_type.rb index bc3dda05be0..88c4d699a5b 100644 --- a/drivers/hmis/app/graphql/types/forms/enums/pick_list_type.rb +++ b/drivers/hmis/app/graphql/types/forms/enums/pick_list_type.rb @@ -27,7 +27,8 @@ class Forms::Enums::PickListType < Types::BaseEnum value 'ALL_UNIT_TYPES', 'All unit types.' value 'POSSIBLE_UNIT_TYPES_FOR_PROJECT', 'Unit types that are eligible to be added to project' value 'AVAILABLE_UNIT_TYPES', 'Unit types that have unoccupied units in the specified project' - value 'AVAILABLE_UNITS_FOR_ENROLLMENT', 'Units available for the given household at the given project' + value 'AVAILABLE_UNITS_FOR_ENROLLMENT', 'Units available for the given Enrollment at the given project. List is limited to units with the same unit type currently occupied by the household, if any.' + value 'ADMIN_AVAILABLE_UNITS_FOR_ENROLLMENT', 'Units available for the given Enrollment at the given project. Includes all available units at project even if they have a different type from what the household is currently occupying.' value 'ALL_SERVICE_TYPES' value 'ALL_SERVICE_CATEGORIES' value 'CUSTOM_SERVICE_CATEGORIES' diff --git a/drivers/hmis/app/graphql/types/forms/pick_list_option.rb b/drivers/hmis/app/graphql/types/forms/pick_list_option.rb index 44fbd670c2a..9cff15d2bd3 100644 --- a/drivers/hmis/app/graphql/types/forms/pick_list_option.rb +++ b/drivers/hmis/app/graphql/types/forms/pick_list_option.rb @@ -70,6 +70,8 @@ def self.options_for_type(pick_list_type, user:, project_id: nil, client_id: nil available_unit_types_for_project(project) when 'AVAILABLE_UNITS_FOR_ENROLLMENT' available_units_for_enrollment(project, household_id: household_id) + when 'ADMIN_AVAILABLE_UNITS_FOR_ENROLLMENT' + admin_available_units_for_enrollment(project, household_id: household_id) when 'OPEN_HOH_ENROLLMENTS_FOR_PROJECT' open_hoh_enrollments_for_project(project, user: user) when 'ENROLLMENTS_FOR_CLIENT' @@ -474,7 +476,7 @@ def self.enrollments_for_client(client, user:) end end - def self.available_units_for_enrollment(project, household_id: nil) + def self.admin_available_units_for_enrollment(project, household_id: nil) return [] unless project # Eligible units are unoccupied units, PLUS units occupied by household members @@ -487,10 +489,8 @@ def self.available_units_for_enrollment(project, household_id: nil) [] end - unit_types_assigned_to_household = Hmis::Unit.where(id: hh_units).pluck(:unit_type_id).compact.uniq eligible_units = Hmis::Unit.where(id: unoccupied_units + hh_units) - # If some household members are assigned to units with unit types, then list should be limited to units of the same type. - eligible_units = eligible_units.where(unit_type_id: unit_types_assigned_to_household) if unit_types_assigned_to_household.any? + eligible_units.preload(:unit_type). order(:unit_type_id, :id). map do |unit| @@ -504,6 +504,24 @@ def self.available_units_for_enrollment(project, household_id: nil) end end + def self.available_units_for_enrollment(project, household_id: nil) + return [] unless project + + # use picklist that includes all available units including units of other types + picklist = admin_available_units_for_enrollment(project, household_id: household_id) + return picklist unless household_id # no household, so no need to filter unit types + + # drop units that have different types + hh_unit_type_ids = project.enrollments.where(household_id: household_id).map(&:current_unit_type).compact.map(&:id).uniq + return picklist if hh_unit_type_ids.empty? # household doesn't have a unit type, so no need for further filtering + + # if the household has a unit type, exclude units that don't match + allowed_unit_type_unit_ids = project.units.where(unit_type_id: hh_unit_type_ids).pluck(:id).to_set + picklist.filter do |option| + option[:code].in?(allowed_unit_type_unit_ids) + end + end + def self.assessment_names_for_project(project) # It's a little odd to combine the "roles" (eg INTAKE) with the identifiers (eg housing_needs_assessment), but # we need to do that in order to get the desired behavior. The "Intake" option should show all Intakes, diff --git a/drivers/hmis/app/models/hmis/hud/enrollment.rb b/drivers/hmis/app/models/hmis/hud/enrollment.rb index 6083bd3f4f7..211efd07a32 100644 --- a/drivers/hmis/app/models/hmis/hud/enrollment.rb +++ b/drivers/hmis/app/models/hmis/hud/enrollment.rb @@ -82,6 +82,7 @@ class Hmis::Hud::Enrollment < Hmis::Hud::Base has_many :unit_occupancies, class_name: 'Hmis::UnitOccupancy', inverse_of: :enrollment, dependent: :destroy has_one :active_unit_occupancy, -> { active }, class_name: 'Hmis::UnitOccupancy', inverse_of: :enrollment has_one :current_unit, through: :active_unit_occupancy, class_name: 'Hmis::Unit', source: :unit + has_one :current_unit_type, through: :current_unit, class_name: 'Hmis::UnitType', source: :unit_type # Cached chronically homeless at entry has_one :ch_enrollment, class_name: 'Hmis::ChEnrollment', dependent: :destroy diff --git a/drivers/hmis/spec/requests/hmis/pick_list_spec.rb b/drivers/hmis/spec/requests/hmis/pick_list_spec.rb index 3c327787d04..9e5a55ba2c6 100644 --- a/drivers/hmis/spec/requests/hmis/pick_list_spec.rb +++ b/drivers/hmis/spec/requests/hmis/pick_list_spec.rb @@ -308,36 +308,56 @@ def picklist_option_codes(project) end end - describe 'AVAILABLE_UNITS_FOR_ENROLLMENT' do + describe 'unit picklists' do let!(:e1) { create :hmis_hud_enrollment, data_source: ds1, project: p1, client: c1 } - let!(:un1) { create :hmis_unit, project: p1 } - let!(:un2) { create :hmis_unit, project: p1 } - let!(:un3) { create :hmis_unit } # in another project + let!(:br1) { create :hmis_unit_type, description: '1 BR' } + let!(:br2) { create :hmis_unit_type, description: '2 BR' } + + let!(:un1) { create :hmis_unit, project: p1, unit_type: br1 } + let!(:un2) { create :hmis_unit, project: p1, unit_type: br1 } + let!(:un3) { create :hmis_unit, project: p1, unit_type: br2 } + + # cruft: units in other projects + let!(:un4) { create :hmis_unit, unit_type: br1 } + let!(:un5) { create :hmis_unit, unit_type: br2 } + let!(:un6) { create :hmis_unit } # assign e1 to un1 let!(:uo1) { create :hmis_unit_occupancy, unit: un1, enrollment: e1, start_date: 1.week.ago } - def picklist_option_codes(project, household_id = nil) + def picklist_option_codes(project, picklist: 'AVAILABLE_UNITS_FOR_ENROLLMENT', household_id: nil) Types::Forms::PickListOption.options_for_type( - 'AVAILABLE_UNITS_FOR_ENROLLMENT', + picklist, user: hmis_user, project_id: project.id, household_id: household_id, ).map { |opt| opt[:code] } end - it 'resolves available units for project' do - expect(picklist_option_codes(p1)).to contain_exactly(un2.id) - end + context 'AVAILABLE_UNITS_FOR_ENROLLMENT' do + it 'resolves available units for project' do + expect(picklist_option_codes(p1)).to contain_exactly(un2.id, un3.id) + end + + it 'includes units that are currently occupied by the household, plus other units of the same type' do + result = picklist_option_codes(p1, household_id: e1.household_id) + expect(result).to contain_exactly(un1.id, un2.id) + end - it 'includes units that are currently occupied by the household' do - expect(picklist_option_codes(p1, e1.household_id)).to contain_exactly(un1.id, un2.id) + it 'if household unit doesn\'t have a type, includes all available units' do + un1.update!(unit_type: nil) + expect(picklist_option_codes(p1, household_id: e1.household_id)).to contain_exactly(un1.id, un2.id, un3.id) + end end - it 'if household is occupied by a unit that has a type, excludes other unit typoes from list' do - un1.unit_type = create(:hmis_unit_type) - un1.save! - expect(picklist_option_codes(p1, e1.household_id)).to contain_exactly(un1.id) + context 'ADMIN_AVAILABLE_UNITS_FOR_ENROLLMENT' do + it 'resolves available units for project' do + expect(picklist_option_codes(p1)).to contain_exactly(un2.id, un3.id) + end + + it 'includes units with differing unit types' do + expect(picklist_option_codes(p1, picklist: 'ADMIN_AVAILABLE_UNITS_FOR_ENROLLMENT', household_id: e1.household_id)).to contain_exactly(un1.id, un2.id, un3.id) + end end end From 16acdc275d18e3aac7fe841c1c1610be2b3c1b5c Mon Sep 17 00:00:00 2001 From: Elliot Date: Wed, 5 Feb 2025 08:39:55 -0500 Subject: [PATCH 15/18] Shape updates (#5107) * Fix for 2024 MA shape file; brief documentation about updating CoC Shape files * Cleanup shapes inserts * Revert unplanned change --- shape_files/CoC/README.md | 10 ++++++++++ shape_files/CoC/make.inserts | 1 + 2 files changed, 11 insertions(+) diff --git a/shape_files/CoC/README.md b/shape_files/CoC/README.md index 3b702d7886d..421625817fc 100644 --- a/shape_files/CoC/README.md +++ b/shape_files/CoC/README.md @@ -13,3 +13,13 @@ https://www.hudexchange.info/programs/coc/gis-tools/?&filter_tooltype=ShapeFile& * copied HTML on two pages of links and hacked it into wget lines. * result is in get.zips * make.inserts unzips and converts to postgres inserts + +# Upgrading + +If a new file is released with updated CoCs, you can: + +1. Remove `shape_files/.did-shape-sync` (maybe only needed in development) +2. Add the new file to S3 (ensure roughly the same naming convention) +3. Remove the old file from S3 (and your local `shape_files/CoC` if you have any) +4. Delete the CoC shapes form the database `GrdaWarehouse::Shape::Coc.delete_all` +5. Re-run the installer (with AWS credentials) `GrdaWarehouse::Shape::Installer.new.run!` diff --git a/shape_files/CoC/make.inserts b/shape_files/CoC/make.inserts index fdfcb0bb9c1..fb421143071 100755 --- a/shape_files/CoC/make.inserts +++ b/shape_files/CoC/make.inserts @@ -35,6 +35,7 @@ Dir.glob('**/*shp').each do |shape_file| system(<<~EOS) shp2pgsql -s 4269:4326 -c -I #{shape_file} shape_cocs \ | sed -e 's/gid/id/' \ + | sed -e 's/"st_1"/"st"/' \ | grep 'INSERT' \ >> inserts.sql EOS From 047a5fd32532c423c729a95fe2c340dafd500ed7 Mon Sep 17 00:00:00 2001 From: Elliot Anders Date: Wed, 5 Feb 2025 12:56:28 -0500 Subject: [PATCH 16/18] Language adjustment to indicate no threshold met if no existing data exists --- app/views/import_thresholds/show.haml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/import_thresholds/show.haml b/app/views/import_thresholds/show.haml index f00235ae3a9..0576999c8d5 100644 --- a/app/views/import_thresholds/show.haml +++ b/app/views/import_thresholds/show.haml @@ -7,7 +7,7 @@ = simple_form_for import_threshold, url: data_source_import_threshold_path, method: :patch do |f| .well %h2 Changes in Record Counts - %p The following settings apply when an import significantly changes the number of rows in the data that already exists in the warehouse for the data source. This will look at the aggregate change, so if 100 rows are removed and 100 added, no change is indicated. + %p The following settings apply when an import significantly changes the number of rows in the data that already exist in the warehouse for the data source that meet the criteria of the import. This will not trigger if no data exists in the warehouse for the included projects and date range. This will look at the aggregate change, so if 100 rows are removed and 100 added, no change is indicated. .row .col = f.input :record_count_change_percent_threshold, label: 'Record count change threshold (%)', hint: 'Imports will pause and alert if more than the chosen percentage have changed', input_html: { style: 'width: 5em' } From 761b8fbadd2b0d743d2113d90e0c28db9007be68 Mon Sep 17 00:00:00 2001 From: Elliot Date: Wed, 5 Feb 2025 16:10:56 -0500 Subject: [PATCH 17/18] Expose RRH Sub Types and limit HMIS DQ Tool (#5112) * Differentiate RRH with housing from those without * Expose RRH Sub Type in various locations; Limit HMIS DQ Tool checks for overlapping data to RRH projects that provide housing --- app/controllers/api/projects_controller.rb | 5 ++-- app/models/grda_warehouse/hud/project.rb | 12 +++++++++ app/views/clients/_enrollment_table.haml | 4 +-- app/views/data_sources/_project.haml | 4 +-- app/views/projects/show.haml | 2 +- .../grda_warehouse/hud/client_extension.rb | 1 + .../models/hmis_data_quality_tool/client.rb | 4 +-- lib/util/hud_utility_2024.rb | 25 +++++++++++++++++++ 8 files changed, 48 insertions(+), 9 deletions(-) diff --git a/app/controllers/api/projects_controller.rb b/app/controllers/api/projects_controller.rb index b4b1ab18ba3..42a3af06a84 100644 --- a/app/controllers/api/projects_controller.rb +++ b/app/controllers/api/projects_controller.rb @@ -24,14 +24,15 @@ def index :id, :ProjectName, # OK to use non-confidentialized name because list is filtered by confidentiality in project_scope :ProjectType, + :RRHSubType, o_t[:OrganizationName], o_t[:id], ds_t[:short_name], - ).each do |id, p_name, type, o_name, o_id, ds_name| + ).each do |id, p_name, type, rrh_sub_type, o_name, o_id, ds_name| o_name_at_ds = "#{o_name} at #{ds_name}" @data[[o_id, o_name_at_ds]] ||= [] - p_name += " (#{HudUtility2024.project_type_brief(type)})" if HudUtility2024.project_type_brief(type).present? + p_name += " (#{HudUtility2024.brief_project_type_with_sub_type(type, rrh_sub_type)})" if HudUtility2024.brief_project_type_with_sub_type(type).present? @data[[o_id, o_name_at_ds]] << [ p_name, id, diff --git a/app/models/grda_warehouse/hud/project.rb b/app/models/grda_warehouse/hud/project.rb index 9e745400ba6..51a6c262008 100644 --- a/app/models/grda_warehouse/hud/project.rb +++ b/app/models/grda_warehouse/hud/project.rb @@ -590,6 +590,18 @@ def rrh? project_type_to_use.in?(HudUtility2024.performance_reporting[:rrh]) end + def rrh_sso_only? + rrh? && self.RRHSubType == rrh_sso_sub_type_code + end + + def rrh_with_housing? + rrh? && self.RRHSubType != rrh_sso_sub_type_code + end + + def rrh_sso_sub_type_code + HudUtility2024.rrh_sub_type('RRH: Services Only', true) + end + def psh? project_type_to_use.in?(HudUtility2024.performance_reporting[:psh]) end diff --git a/app/views/clients/_enrollment_table.haml b/app/views/clients/_enrollment_table.haml index 755b0ace4d6..e16e1635f37 100644 --- a/app/views/clients/_enrollment_table.haml +++ b/app/views/clients/_enrollment_table.haml @@ -50,8 +50,8 @@ %td.nobr .ds.jClickToCopy{class: "ds-color-#{e[:data_source_id]}", data: { toggle: :tooltip, html: 'true', title: ds_tooltip_content(e[:client_source_id], e[:data_source_id]).html_safe }}= ds_short_name_for(e[:client_source_id]) .enrollment__project_type{class: e[:class]} - %span.service-type__program-type{data: {toggle: :tooltip, title: HudUtility2024.project_type(e[:project_type_id])}} - = e[:project_type] + %span.service-type__program-type{data: {toggle: :tooltip, title: HudUtility2024.project_type_with_sub_type(e[:project_type_id], e[:rrh_sub_type])}} + = HudUtility2024.brief_project_type_with_sub_type(e[:project_type_id], e[:rrh_sub_type]) - if e[:confidential_project] && can_view_confidential_project_names? .confidential_project{data: {toggle: :tooltip, title: GrdaWarehouse::Hud::Project.confidential_project_name}} CP diff --git a/app/views/data_sources/_project.haml b/app/views/data_sources/_project.haml index 9b6da6616cc..741059e123b 100644 --- a/app/views/data_sources/_project.haml +++ b/app/views/data_sources/_project.haml @@ -4,8 +4,8 @@ = link_to_if can_view_projects?, project.name(current_user, ignore_confidential_status: can_edit_projects?), project_path(project) %td.d-flex .enrollment__project_type.mr-2{class: "client__service_type_#{project.ProjectType}"} - .service-type__program-type{data: {toggle: :tooltip, title: HudUtility2024.project_type(project.ProjectType)}} - = HudUtility2024.project_type_brief project.ProjectType + .service-type__program-type{data: { toggle: :tooltip, title: HudUtility2024.project_type_with_sub_type(project.project_type, project.rrh_sub_type) }} + = HudUtility2024.brief_project_type_with_sub_type(project.project_type, project.rrh_sub_type) %td.text-center %span{data: {toggle: :tooltip, title: project.confidential_hint}} = checkmark project.confidential diff --git a/app/views/projects/show.haml b/app/views/projects/show.haml index 082e1a11e05..c837d9f1726 100644 --- a/app/views/projects/show.haml +++ b/app/views/projects/show.haml @@ -15,7 +15,7 @@ %tr %th Project Type %td - = HudUtility2024.project_type(@project.ProjectType) + = HudUtility2024.project_type_with_sub_type(@project.project_type, @project.rrh_sub_type) - if @project.active_homeless_status_override.present? %br %em Enrolled clients are actively homeless for CAS and Cohorts diff --git a/drivers/client_access_control/extensions/grda_warehouse/hud/client_extension.rb b/drivers/client_access_control/extensions/grda_warehouse/hud/client_extension.rb index 5eb77aed5e6..a8305a03b7d 100644 --- a/drivers/client_access_control/extensions/grda_warehouse/hud/client_extension.rb +++ b/drivers/client_access_control/extensions/grda_warehouse/hud/client_extension.rb @@ -228,6 +228,7 @@ def enrollments_for_rollup(user:, en_scope: scope, include_confidential_names: f household: household(entry.household_id, entry.enrollment.data_source_id), project_type: ::HudUtility2024.project_type_brief(entry.project_type), project_type_id: entry.project_type, + rrh_sub_type: project.rrh_sub_type, class: "client__service_type_#{entry.project_type}", most_recent_service: most_recent_service, new_episode: new_episode, diff --git a/drivers/hmis_data_quality_tool/app/models/hmis_data_quality_tool/client.rb b/drivers/hmis_data_quality_tool/app/models/hmis_data_quality_tool/client.rb index 503135403fc..2e07009fd11 100644 --- a/drivers/hmis_data_quality_tool/app/models/hmis_data_quality_tool/client.rb +++ b/drivers/hmis_data_quality_tool/app/models/hmis_data_quality_tool/client.rb @@ -241,7 +241,7 @@ def self.overlapping_entry_exit(enrollments:, report:) # check for overlapping PH post-move-in def self.overlapping_post_move_in(enrollments:, report:) involved_enrollments = enrollments.select do |en| - en.project&.ph? + en.project&.ph? && ! en.project&.rrh_sso_only? end return [] if involved_enrollments.blank? || involved_enrollments.count == 1 @@ -288,7 +288,7 @@ def self.overlapping_homeless_post_move_in(enrollments:, report:) # moved-in PH enrollments involved_enrollments = enrollments.select do |en| - en.project&.ph? && en.MoveInDate.present? + en.project&.ph? && ! en.project&.rrh_sso_only? && en.MoveInDate.present? end return [] if involved_enrollments.blank? diff --git a/lib/util/hud_utility_2024.rb b/lib/util/hud_utility_2024.rb index 07af208c69a..ae01a6f5986 100644 --- a/lib/util/hud_utility_2024.rb +++ b/lib/util/hud_utility_2024.rb @@ -86,6 +86,31 @@ def veteran_status(*args) no_yes_reasons_for_missing_data(*args) end + def project_type_with_sub_type(type, sub_type = nil) + [ + project_type(type), + rrh_sub_type(sub_type), + ].compact_blank.join(' — ') + end + + def brief_project_type_with_sub_type(type, sub_type = nil) + [ + project_type_brief(type), + rrh_sub_type_brief(sub_type), + ].compact_blank.join(' — ') + end + + def rrh_sub_types_brief + { + 1 => 'SSO', + 2 => 'Housing', + }.freeze + end + + def rrh_sub_type_brief(id, reverse = false, raise_on_missing: false) + _translate(rrh_sub_types_brief, id, reverse, raise_on_missing: raise_on_missing) + end + def project_type_number(type) # attempt to lookup full name number = project_type(type, true) # reversed From cf61a7b45ae0c610586b26a37d9a55a7d6281923 Mon Sep 17 00:00:00 2001 From: martha Date: Thu, 6 Feb 2025 07:20:48 -0500 Subject: [PATCH 18/18] Update HMIS system tests for new Table Action Column pattern (#5039) --- .../spec/system/hmis/assessment_definitions_spec.rb | 2 +- drivers/hmis/spec/system/hmis/bulk_services_spec.rb | 10 +++++----- .../hmis/spec/system/hmis/intake_assessment_spec.rb | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/hmis/spec/system/hmis/assessment_definitions_spec.rb b/drivers/hmis/spec/system/hmis/assessment_definitions_spec.rb index ded0012c1d7..6a6a2cb7f6a 100644 --- a/drivers/hmis/spec/system/hmis/assessment_definitions_spec.rb +++ b/drivers/hmis/spec/system/hmis/assessment_definitions_spec.rb @@ -176,7 +176,7 @@ sign_in(hmis_user) visit "/client/#{c1.id}/enrollments/#{e1.id}/assessments" - click_link 'Intake' + click_link 'Finish Intake' end def select_member(client) diff --git a/drivers/hmis/spec/system/hmis/bulk_services_spec.rb b/drivers/hmis/spec/system/hmis/bulk_services_spec.rb index 82fc0082a1c..5d04f1234e6 100644 --- a/drivers/hmis/spec/system/hmis/bulk_services_spec.rb +++ b/drivers/hmis/spec/system/hmis/bulk_services_spec.rb @@ -8,7 +8,7 @@ require_relative '../../requests/hmis/login_and_permissions' require_relative '../../support/hmis_base_setup' -RSpec.feature 'Hmis Form behavior', type: :system do +RSpec.feature 'Bulk Services behavior', type: :system do include_context 'hmis base setup' include_context 'hmis service setup' @@ -54,9 +54,9 @@ # Find the indices of the two columns we want to check header_cells = all('thead th') last_bed_night_date_index = header_cells.find_index { |cell| cell.text == 'Last Bed Night Date' } - assign_bed_night_index = header_cells.find_index { |cell| cell.text == "Assign Bed Night for #{bed_night_date.strftime('%m/%d/%Y')}" } + button_column_index = header_cells.find_index { |cell| cell.text == 'Actions' } expect(last_bed_night_date_index).not_to be_nil - expect(assign_bed_night_index).not_to be_nil + expect(button_column_index).not_to be_nil # Verify that all 3 rows have the expected attributes all('tbody tr').each do |row| @@ -64,8 +64,8 @@ last_bed_night_date = row.all('td')[last_bed_night_date_index].text expect(last_bed_night_date).to match(/#{bed_night_date.strftime('%m/%d/%Y')}/) - # Check the "Assign Bed Night for mm/dd/yyyy" column - assign_button = row.all('td')[assign_bed_night_index].find('button') + # Check the "Actions" column + assign_button = row.all('td')[button_column_index].first('button') expect(assign_button.text).to eq('Assigned') end diff --git a/drivers/hmis/spec/system/hmis/intake_assessment_spec.rb b/drivers/hmis/spec/system/hmis/intake_assessment_spec.rb index 2a6df7ae4cc..dbe9296de7c 100644 --- a/drivers/hmis/spec/system/hmis/intake_assessment_spec.rb +++ b/drivers/hmis/spec/system/hmis/intake_assessment_spec.rb @@ -8,7 +8,7 @@ require_relative '../../requests/hmis/login_and_permissions' require_relative '../../support/hmis_base_setup' -RSpec.feature 'Enrollment/household management', type: :system do +RSpec.feature 'Intake assessment', type: :system do include_context 'hmis base setup' # could parse CAPYBARA_APP_HOST let!(:ds1) { create(:hmis_data_source, hmis: 'localhost') }