Fix Outcome Result Groups IDs

app/models/outcome.rb

@@ -2,12 +2,12 @@ class Outcome < StudyRelationship
   has_many :outcome_counts, inverse_of: :outcome 
   has_many :outcome_analyses, inverse_of: :outcome
   has_many :outcome_measurements, inverse_of: :outcome
+  has_many :result_groups, inverse_of: :outcome
 
   add_mapping do
     {
       table: :outcomes,
       root: [:resultsSection, :outcomeMeasuresModule, :outcomeMeasures],
-      requires: :result_groups,
       columns: [
         { name: :outcome_type, value: :type },
         { name: :title, value: :title },
@@ -22,13 +22,24 @@ class Outcome < StudyRelationship
         { name: :param_type, value: :paramType }
       ],
       children: [
+        {
+          table: :result_groups,
+          root: nil,
+          flatten: [:groups],
+          index: [:ctgov_group_code, :result_type, :outcome_id],
+          columns: [
+            { name: :ctgov_group_code, value: :id },
+            { name: :result_type, value: 'Outcome' },
+            { name: :title, value: :title },
+            { name: :description, value: :description }
+          ]
+        },
         {
           table: :outcome_measurements,
           root: nil,
           flatten: [:classes, :categories, :measurements],
           columns: [
-            { name: :outcome_id, value: nil },
-            { name: :result_group_id, value: reference(:result_groups)[:groupId, 'Outcome'] },
+            # result_group_id is set by ResultGroup.set_outcome_results_group_ids
             { name: :ctgov_group_code, value: :groupId },
             { name: :classification, value: [:$parent, :$parent, :title] },
             { name: :category, value: [:$parent, :title] },
@@ -53,11 +64,9 @@ class Outcome < StudyRelationship
         { 
           table: :outcome_counts,
           root: nil,
-          requires: :result_groups, # do I need this since the parent has it?
           flatten: [:denoms, :counts],
           columns: [
-            { name: :outcome_id, value: nil },
-            { name: :result_group_id, value: reference(:result_groups)[:groupId, 'Outcome'] },
+            # result_group_id is set by ResultGroup.set_outcome_results_group_ids
             { name: :ctgov_group_code, value: [:groupId] },
             { name: :scope, value: 'Measure' },
             { name: :units, value: [:$parent, :units] },

app/models/result_group.rb

@@ -5,34 +5,26 @@ class ResultGroup < StudyRelationship
   has_many :drop_withdrawals
   has_many :baseline_counts
   has_many :baseline_measures
+
   has_many :outcome_counts
   has_many :outcome_measurements
+  # TODO: review analysis_groups
   has_many :outcome_analysis_groups, inverse_of: :result_group
   has_many :outcome_analyses, :through => :outcome_analysis_groups
 
+  belongs_to :outcome, optional: true
+
   add_mapping do
     [
       {
         table: :result_groups,
         root: [:resultsSection, :baselineCharacteristicsModule, :groups],
-        index: [:ctgov_group_code, :result_type], 
-        unique: true,
-        columns: [
-          { name: :ctgov_group_code, value: :id },
-          { name: :result_type, value: 'Baseline' },
-          { name: :title, value: :title },
-          { name: :description, value: :description },
-        ]
-      },
-      {
-        table: :result_groups,
-        root: [:resultsSection, :outcomeMeasuresModule, :outcomeMeasures],
-        flatten: [:groups],
         index: [:ctgov_group_code, :result_type],
+        # requires: :outcomes, # review this
         unique: true,
         columns: [
           { name: :ctgov_group_code, value: :id },
-          { name: :result_type, value: 'Outcome' },
+          { name: :result_type, value: 'Baseline' },
           { name: :title, value: :title },
           { name: :description, value: :description },
         ]
@@ -41,6 +33,7 @@ class ResultGroup < StudyRelationship
         table: :result_groups,
         root: [:resultsSection, :participantFlowModule, :groups],
         index: [:ctgov_group_code, :result_type], 
+        # requires: :outcomes,  # review this
         unique: true,
         columns: [
           { name: :ctgov_group_code, value: :id },
@@ -53,6 +46,8 @@ class ResultGroup < StudyRelationship
         table: :result_groups,
         root: [:resultsSection, :adverseEventsModule, :eventGroups],
         index: [:ctgov_group_code, :result_type],
+        # outcomes should be loaded first to avoid double loading of reported events
+        requires: :outcomes,
         unique: true,
         columns: [
           { name: :ctgov_group_code, value: :id },
@@ -63,4 +58,43 @@ class ResultGroup < StudyRelationship
       }
     ]
   end
+
+
+  def self.set_outcome_results_group_ids(nct_ids)
+    Rails.logger.info "Setting Result Group IDs for Outcome Counts and Measurements"
+
+    groups = fetch_outcome_groups_for(nct_ids)
+
+    outcome_counts_updates = prepare_updates_for(OutcomeCount, nct_ids, groups)
+    outcome_measurements_updates = prepare_updates_for(OutcomeMeasurement, nct_ids, groups)
+
+    bulk_update(OutcomeCount, outcome_counts_updates)
+    bulk_update(OutcomeMeasurement, outcome_measurements_updates)
+  end
+
+  private
+
+  # TODO: consider adding index for nct_id and result_type
+  def self.fetch_outcome_groups_for(nct_ids)
+    where(nct_id: nct_ids).index_by do |group|
+      [group.nct_id, group.ctgov_group_code, group.outcome_id]
+    end
+  end
+
+  def self.prepare_updates_for(model, nct_ids, groups)
+    updates = []
+
+    records = model.where(nct_id: nct_ids)
+    records.each do |record|
+      index = [record.nct_id, record.ctgov_group_code, record.outcome_id]
+      group = groups[index]
+      updates << { id: record.id, result_group_id: group.id } if group
+    end
+    updates
+  end
+
+
+  def self.bulk_update(model, updates)
+    model.import updates, on_duplicate_key_update: { conflict_target: [:id], columns: [:result_group_id] }
+  end
 end

app/models/study_json_record/worker.rb

@@ -52,13 +52,21 @@ def save_children(parents, indent="  ")
     return if klass == Study
 
     klass.reflect_on_all_associations(:has_many).each do |association|
+      # byebug if association.name == :result_groups and klass == Outcome
       # update the nct_id and parent_id of the children
       collection = []
       collections[association.name].each do |child|
         inverse_name = association.inverse_of&.name
         next unless inverse_name
 
         parent = child.send(inverse_name)
+        # byebug if parent.nct_id.nil? or child.nct_id.nil?
+        if parent.nil?
+          puts "Skipping child because parent is nil"
+          puts "Parrent: #{inverse_name}, child: #{association.name}"
+          # byebug
+          next
+        end
         child.nct_id = parent.nct_id
         child[association.foreign_key] = parent.id
         collection << child
@@ -106,6 +114,7 @@ def process(count = 1, records = nil)
 
     # recalculate values for processed records
     CalculatedValue.populate_for(records)
+    ResultGroup.set_outcome_results_group_ids(records.pluck(:nct_id))
     # mark study records as saved
     StudyJsonRecord.version_2.where(nct_id: records.map(&:nct_id)).update_all(saved_study_at: Time.zone.now) # rubocop:disable Rails/SkipsModelValidations
   end
@@ -237,7 +246,7 @@ def process_mapping(mapping, records)
         values = mapping[:columns].map do |column|
           [column[:name], get_value(column, entry, index, nct_id)]
         end
-        row = model.new(values.to_h)
+        row = model.new(values.to_h) # Create Model Instance
         row.nct_id = nct_id
         prepare_children(row, entry, mapping[:children]) if mapping[:children]
         collection << row
@@ -261,6 +270,7 @@ def process_mapping(mapping, records)
         @index[mapping[:table]][row_index] = row.id
       end
     end
+
     save_children(collection)
   rescue
     raise "Error processing #{mapping[:table]}"

db/migrate/20240730222441_add_outome_id_to_result_groups.rb

@@ -0,0 +1,5 @@
+class AddOutomeIdToResultGroups < ActiveRecord::Migration[6.0]
+  def change
+    add_column :result_groups, :outcome_id, :integer
+  end
+end

lib/tasks/load.rake

@@ -10,9 +10,12 @@ namespace :db do
 
   desc 'process study json records'
   task :import_study, [:nct_id] => :environment do |t, args|
+    dbmgr = Util::DbManager.new
+    dbmgr.remove_constraints
     worker = StudyJsonRecord::Worker.new
     records = StudyJsonRecord.where(nct_id: args[:nct_id], version: '2')
     worker.process(1, records)
+    dbmgr.add_constraints
   end
 
   desc 'process study json records'