Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pull in programs from HSES nightly download #464

Merged
merged 2 commits into from
Oct 11, 2021
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion src/lib/updateGrantsGrantees.js
Original file line number Diff line number Diff line change
@@ -2,8 +2,10 @@ import AdmZip from 'adm-zip';
import { toJson } from 'xml2json';
import {} from 'dotenv/config';
import axios from 'axios';
import { keyBy, mapValues } from 'lodash';

import {
Grantee, Grant,
Grantee, Grant, Program,
} from '../models';
import { logger, auditLogger } from '../logger';

@@ -68,6 +70,9 @@ export async function processFiles() {
const grantData = await fs.readFile('./temp/grant_award.xml');
const grant = JSON.parse(toJson(grantData));

const programData = await fs.readFile('./temp/grant_program.xml');
const programs = JSON.parse(toJson(programData));

const grantsForDb = grant.grant_awards.grant_award.map((g) => {
let { grant_start_date: startDate, grant_end_date: endDate } = g;
if (typeof startDate === 'object') { startDate = null; }
@@ -100,6 +105,27 @@ export async function processFiles() {
};
});

const grantIds = grantsForDb.map((g) => g.id);
const grantPrograms = grantGrantees.filter(
(ga) => grantIds.includes(parseInt(ga.grant_award_id, 10)),
);

const grantAgencyMap = mapValues(keyBy(grantPrograms, 'grant_agency_id'), 'grant_award_id');
const programsWithGrants = programs.grant_programs.grant_program.filter(
(p) => parseInt(p.grant_agency_id, 10) in grantAgencyMap,
);

const programsForDb = programsWithGrants.map((program) => ({
id: parseInt(program.grant_program_id, 10),
grantId: parseInt(grantAgencyMap[program.grant_agency_id], 10),
programType: valueFromXML(program.program_type),
startYear: valueFromXML(program.program_start_year),
startDate: valueFromXML(program.grant_program_start_date),
endDate: valueFromXML(program.grant_program_end_date),
status: valueFromXML(program.program_status),
name: valueFromXML(program.program_name),
}));

const cdiGrants = grantsForDb.filter((g) => g.regionId === 13);
const nonCdiGrants = grantsForDb.filter((g) => g.regionId !== 13);

@@ -113,6 +139,10 @@ export async function processFiles() {
{
updateOnDuplicate: ['number', 'status', 'startDate', 'endDate', 'updatedAt', 'programSpecialistName', 'programSpecialistEmail', 'grantSpecialistName', 'grantSpecialistEmail'],
});
await Program.bulkCreate(programsForDb,
{
updateOnDuplicate: ['programType', 'startYear', 'startDate', 'endDate', 'status', 'name'],
});
} catch (error) {
auditLogger.error(`Error reading or updating database on HSES data import: ${error.message}`);
throw error;
12 changes: 11 additions & 1 deletion src/lib/updateGrantsGrantees.test.js
Original file line number Diff line number Diff line change
@@ -3,7 +3,7 @@ import axios from 'axios';
import fs from 'mz/fs';
import updateGrantsGrantees, { processFiles } from './updateGrantsGrantees';
import db, {
Grantee, Grant,
Grantee, Grant, Program,
} from '../models';

jest.mock('axios');
@@ -63,10 +63,12 @@ describe('Update HSES data', () => {

describe('Update grants and grantees', () => {
beforeAll(async () => {
await Program.destroy({ where: { id: [1, 2, 3] } });
await Grant.destroy({ where: { id: { [Op.gt]: SMALLEST_GRANT_ID } } });
await Grantee.destroy({ where: { id: { [Op.gt]: SMALLEST_GRANT_ID } } });
});
afterEach(async () => {
await Program.destroy({ where: { id: [1, 2, 3] } });
await Grant.destroy({ where: { id: { [Op.gt]: SMALLEST_GRANT_ID } } });
await Grantee.destroy({ where: { id: { [Op.gt]: SMALLEST_GRANT_ID } } });
});
@@ -167,4 +169,12 @@ describe('Update grants and grantees', () => {
expect(grant.regionId).toBe(5);
expect(grant.granteeId).toBe(500);
});

it('should import programs', async () => {
await processFiles();
const program = await Program.findOne({ where: { id: 1 }, include: { model: Grant, as: 'grant' } });
expect(program.status).toBe('Inactive');
expect(program.grant.id).toBe(10567);
expect(program.programType).toBe('HS');
});
});
53 changes: 53 additions & 0 deletions src/migrations/20211008175906-add-program-table.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,53 @@
module.exports = {
up: async (queryInterface, Sequelize) => {
await queryInterface.createTable('Programs', {
id: {
allowNull: false,
autoIncrement: false,
primaryKey: true,
type: Sequelize.INTEGER,
},
grantId: {
type: Sequelize.INTEGER,
allowNull: false,
references: {
model: {
tableName: 'Grants',
},
key: 'id',
},
},
programType: {
type: Sequelize.STRING,
},
startYear: {
type: Sequelize.STRING,
},
startDate: {
type: Sequelize.STRING,
},
endDate: {
type: Sequelize.STRING,
},
status: {
type: Sequelize.STRING,
},
name: {
type: Sequelize.STRING,
},
createdAt: {
allowNull: false,
type: Sequelize.DATE,
defaultValue: Sequelize.fn('NOW'),
},
updatedAt: {
allowNull: false,
type: Sequelize.DATE,
defaultValue: Sequelize.fn('NOW'),
},
});
},
down: async (queryInterface) => {
await queryInterface.dropTable('Grants');
},
};
34 changes: 34 additions & 0 deletions src/models/program.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
const {
Model,
} = require('sequelize');

module.exports = (sequelize, DataTypes) => {
class Program extends Model {
static associate(models) {
Program.belongsTo(models.Grant, { foreignKey: 'grantId', as: 'grant' });
}
}
Program.init({
id: {
type: DataTypes.INTEGER,
allowNull: false,
defaultValue: null,
primaryKey: true,
autoIncrement: false,
},
grantId: {
type: DataTypes.INTEGER,
allowNull: false,
},
programType: DataTypes.STRING,
startYear: DataTypes.STRING,
startDate: DataTypes.STRING,
endDate: DataTypes.STRING,
status: DataTypes.STRING,
name: DataTypes.STRING,
}, {
sequelize,
modelName: 'Program',
});
return Program;
};
157 changes: 157 additions & 0 deletions temp/grant_program.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
<grant_programs xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="gov.hhs.acf.ohs.hses">
<grant_program>
<grant_program_id>1</grant_program_id>
<ohs_program_id>1</ohs_program_id>
<grant_agency_id>11636</grant_agency_id>
<program_number>000</program_number>
<program_start_year>2014</program_start_year>
<grant_program_start_date>2014-05-01</grant_program_start_date>
<grant_program_end_date>2019-04-30</grant_program_end_date>
<program_type>HS</program_type>
<program_status>Inactive</program_status>
<program_name>Children's Services</program_name>
<program_web_url xsi:nil="true"/>
<program_email>[email protected]</program_email>
<street_address_line1>111 Broadway Street</street_address_line1>
<street_address_line2 xsi:nil="true"/>
<street_city>City</street_city>
<street_state>CA</street_state>
<street_zip>55555</street_zip>
<postal_address_line1>111 Broadway Street</postal_address_line1>
<postal_address_line2 xsi:nil="true"/>
<postal_city>City</postal_city>
<postal_state>CA</postal_state>
<postal_zip>55555</postal_zip>
<administrative_address_line1>111 Broadway Street</administrative_address_line1>
<administrative_address_line2 xsi:nil="true"/>
<administrative_city>City</administrative_city>
<administrative_state>CA</administrative_state>
<administrative_zip>55555</administrative_zip>
<auth_official_contact_prefix>Mr.</auth_official_contact_prefix>
<auth_official_contact_first_name>F47125</auth_official_contact_first_name>
<auth_official_contact_last_name>L47125</auth_official_contact_last_name>
<auth_official_contact_suffix xsi:nil="true"/>
<auth_official_contact_title>Board President</auth_official_contact_title>
<auth_official_contact_email>[email protected]</auth_official_contact_email>
<ceo_prefix>Ms.</ceo_prefix>
<ceo_first_name>F47126</ceo_first_name>
<ceo_last_name>L47126</ceo_last_name>
<ceo_suffix xsi:nil="true"/>
<ceo_title>CEO</ceo_title>
<ceo_email>[email protected]</ceo_email>
<policy_council_prefix>Ms.</policy_council_prefix>
<policy_council_first_name>F47128</policy_council_first_name>
<policy_council_last_name>L47128</policy_council_last_name>
<policy_council_suffix xsi:nil="true"/>
<policy_council_email>[email protected]</policy_council_email>
<director_prefix>Ms.</director_prefix>
<director_first_name>F47124</director_first_name>
<director_last_name>L47124</director_last_name>
<director_suffix xsi:nil="true"/>
<director_email>[email protected]</director_email>
<director_phone>555-555-7124</director_phone>
</grant_program>
<grant_program>
<grant_program_id>2</grant_program_id>
<ohs_program_id>2</ohs_program_id>
<grant_agency_id>11636</grant_agency_id>
<program_number>001</program_number>
<program_start_year xsi:nil="true"/>
<grant_program_start_date xsi:nil="true"/>
<grant_program_end_date xsi:nil="true"/>
<program_type>HS</program_type>
<program_status>Active</program_status>
<program_name>Children first</program_name>
<program_web_url>www.example.org</program_web_url>
<program_email>[email protected]</program_email>
<street_address_line1>555 Vincent Rd</street_address_line1>
<street_address_line2>Post Office Box 5555</street_address_line2>
<street_city>City</street_city>
<street_state>LA</street_state>
<street_zip>55555-5555</street_zip>
<postal_address_line1>555 Vincent Rd</postal_address_line1>
<postal_address_line2>Post Office Box 5555</postal_address_line2>
<postal_city>City</postal_city>
<postal_state>LA</postal_state>
<postal_zip>55555-5555</postal_zip>
<administrative_address_line1>123 Vincent Rd</administrative_address_line1>
<administrative_address_line2>Post Office Box 5555</administrative_address_line2>
<administrative_city>City</administrative_city>
<administrative_state>LA</administrative_state>
<administrative_zip>71108-2542</administrative_zip>
<auth_official_contact_prefix>Rev.</auth_official_contact_prefix>
<auth_official_contact_first_name>F24962</auth_official_contact_first_name>
<auth_official_contact_last_name>L24962</auth_official_contact_last_name>
<auth_official_contact_suffix xsi:nil="true"/>
<auth_official_contact_title>Board Chairperson</auth_official_contact_title>
<auth_official_contact_email>[email protected]</auth_official_contact_email>
<ceo_prefix>Mr.</ceo_prefix>
<ceo_first_name>F24961</ceo_first_name>
<ceo_last_name>L24961</ceo_last_name>
<ceo_suffix xsi:nil="true"/>
<ceo_title xsi:nil="true"/>
<ceo_email>[email protected]</ceo_email>
<policy_council_prefix>Ms.</policy_council_prefix>
<policy_council_first_name>F24963</policy_council_first_name>
<policy_council_last_name>L24963</policy_council_last_name>
<policy_council_suffix xsi:nil="true"/>
<policy_council_email>[email protected]</policy_council_email>
<director_prefix>Mr.</director_prefix>
<director_first_name>F9209</director_first_name>
<director_last_name>L9209</director_last_name>
<director_suffix xsi:nil="true"/>
<director_email>[email protected]</director_email>
<director_phone>555-555-9209</director_phone>
</grant_program>
<grant_program>
<grant_program_id>3</grant_program_id>
<ohs_program_id>3</ohs_program_id>
<grant_agency_id>9393</grant_agency_id>
<program_number>000</program_number>
<program_start_year>1992</program_start_year>
<grant_program_start_date>2020-03-01</grant_program_start_date>
<grant_program_end_date xsi:nil="true"/>
<program_type>HS</program_type>
<program_status>Active</program_status>
<program_name>COMMUNITY ACTION COUNCIL, INC</program_name>
<program_web_url>http://www.example.com</program_web_url>
<program_email>[email protected]</program_email>
<street_address_line1>123 main</street_address_line1>
<street_address_line2 xsi:nil="true"/>
<street_city>city</street_city>
<street_state>OK</street_state>
<street_zip>55555-5555</street_zip>
<postal_address_line1>PO Box 111</postal_address_line1>
<postal_address_line2 xsi:nil="true"/>
<postal_city>city</postal_city>
<postal_state>OK</postal_state>
<postal_zip>55555-5555</postal_zip>
<administrative_address_line1>123 main</administrative_address_line1>
<administrative_address_line2 xsi:nil="true"/>
<administrative_city>City</administrative_city>
<administrative_state>OK</administrative_state>
<administrative_zip>55555-5555</administrative_zip>
<auth_official_contact_prefix>Mr.</auth_official_contact_prefix>
<auth_official_contact_first_name>F123</auth_official_contact_first_name>
<auth_official_contact_last_name>L123</auth_official_contact_last_name>
<auth_official_contact_suffix xsi:nil="true"/>
<auth_official_contact_title>Governing Board Chairperson</auth_official_contact_title>
<auth_official_contact_email>[email protected]</auth_official_contact_email>
<ceo_prefix>Ms.</ceo_prefix>
<ceo_first_name>F321</ceo_first_name>
<ceo_last_name>L321</ceo_last_name>
<ceo_suffix xsi:nil="true"/>
<ceo_title>Executive Director-Head Start/EHS Director</ceo_title>
<ceo_email>[email protected]</ceo_email>
<policy_council_prefix>Ms.</policy_council_prefix>
<policy_council_first_name>F2222</policy_council_first_name>
<policy_council_last_name>L2222</policy_council_last_name>
<policy_council_suffix xsi:nil="true"/>
<policy_council_email>[email protected]</policy_council_email>
<director_prefix>Ms.</director_prefix>
<director_first_name>F3333</director_first_name>
<director_last_name>L3333</director_last_name>
<director_suffix xsi:nil="true"/>
<director_email>[email protected]</director_email>
<director_phone>555-555-5797 x 5555</director_phone>
</grant_program>