Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix(Cronjob): don't restart CasparCG using the nightly cronjob if there's a Rundown active in the Studio (SOFIE-3618) #1365

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
119 changes: 115 additions & 4 deletions meteor/server/__tests__/cronjobs.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,13 @@ import { MeteorMock } from '../../__mocks__/meteor'
import { logger } from '../logging'
import { getRandomId, getRandomString, literal, protectString } from '../lib/tempLib'
import { SnapshotType } from '@sofie-automation/meteor-lib/dist/collections/Snapshots'
import { IBlueprintPieceType, PieceLifespan, StatusCode, TSR } from '@sofie-automation/blueprints-integration'
import {
IBlueprintPieceType,
PieceLifespan,
PlaylistTimingType,
StatusCode,
TSR,
} from '@sofie-automation/blueprints-integration'
import {
PeripheralDeviceType,
PeripheralDeviceCategory,
Expand All @@ -27,6 +33,8 @@ import {
SegmentId,
SnapshotId,
UserActionsLogItemId,
StudioId,
RundownPlaylistId,
} from '@sofie-automation/corelib/dist/dataModel/Ids'

// Set up mocks for tests in this suite
Expand All @@ -53,6 +61,8 @@ import {
UserActionsLog,
Segments,
SofieIngestDataCache,
Studios,
RundownPlaylists,
} from '../collections'
import { NrcsIngestCacheType } from '@sofie-automation/corelib/dist/dataModel/NrcsIngestDataCache'
import { JSONBlobStringify } from '@sofie-automation/shared-lib/dist/lib/JSONBlob'
Expand All @@ -64,7 +74,7 @@ import {
import { DBSegment } from '@sofie-automation/corelib/dist/dataModel/Segment'
import { Settings } from '../Settings'
import { SofieIngestCacheType } from '@sofie-automation/corelib/dist/dataModel/SofieIngestDataCache'
import { ObjectOverrideSetOp } from '@sofie-automation/corelib/dist/settings/objectWithOverrides'
import { ObjectOverrideSetOp, ObjectWithOverrides } from '@sofie-automation/corelib/dist/settings/objectWithOverrides'

describe('cronjobs', () => {
let env: DefaultEnvironment
Expand Down Expand Up @@ -476,7 +486,10 @@ describe('cronjobs', () => {
expect(await Snapshots.findOneAsync(snapshot1)).toBeUndefined()
})
async function insertPlayoutDevice(
props: Pick<PeripheralDevice, 'subType' | 'deviceName' | 'lastSeen' | 'parentDeviceId'> &
props: Pick<
PeripheralDevice,
'subType' | 'deviceName' | 'lastSeen' | 'parentDeviceId' | 'studioAndConfigId'
> &
Partial<Pick<PeripheralDevice, 'token'>>
): Promise<PeripheralDeviceId> {
const deviceId = protectString<PeripheralDeviceId>(getRandomString())
Expand Down Expand Up @@ -504,7 +517,10 @@ describe('cronjobs', () => {
return deviceId
}

async function createMockPlayoutGatewayAndDevices(lastSeen: number): Promise<{
async function createMockPlayoutGatewayAndDevices(
lastSeen: number,
studioId?: StudioId
): Promise<{
deviceToken: string
mockPlayoutGw: PeripheralDeviceId
mockCasparCg: PeripheralDeviceId
Expand All @@ -516,6 +532,12 @@ describe('cronjobs', () => {
lastSeen: lastSeen,
subType: PERIPHERAL_SUBTYPE_PROCESS,
token: deviceToken,
studioAndConfigId: studioId
? {
configId: '',
studioId,
}
: undefined,
})
const mockCasparCg = await insertPlayoutDevice({
deviceName: 'CasparCG',
Expand All @@ -540,6 +562,73 @@ describe('cronjobs', () => {
}
}

async function createMockStudioAndRundown(): Promise<{
studioId: StudioId
rundownPlaylistId: RundownPlaylistId
}> {
function newObjectWithOverrides<T extends {}>(defaults: T): ObjectWithOverrides<T> {
return {
defaults,
overrides: [],
}
}
const studioId = protectString<StudioId>(getRandomString())
await Studios.insertAsync({
_id: studioId,
organizationId: null,
name: 'Studio',
blueprintConfigWithOverrides: newObjectWithOverrides({}),
_rundownVersionHash: '',
lastBlueprintConfig: undefined,
lastBlueprintFixUpHash: undefined,
mappingsWithOverrides: newObjectWithOverrides({}),
supportedShowStyleBase: [],
settingsWithOverrides: newObjectWithOverrides({
allowHold: true,
allowPieceDirectPlay: true,
enableBuckets: true,
enableEvaluationForm: true,
frameRate: 25,
mediaPreviewsUrl: '',
minimumTakeSpan: 1000,
}),
routeSetsWithOverrides: newObjectWithOverrides({}),
routeSetExclusivityGroupsWithOverrides: newObjectWithOverrides({}),
packageContainersWithOverrides: newObjectWithOverrides({}),
previewContainerIds: [],
thumbnailContainerIds: [],
peripheralDeviceSettings: {
deviceSettings: newObjectWithOverrides({}),
ingestDevices: newObjectWithOverrides({}),
inputDevices: newObjectWithOverrides({}),
playoutDevices: newObjectWithOverrides({}),
},
})

const rundownPlaylistId = protectString<RundownPlaylistId>(getRandomString())
await RundownPlaylists.mutableCollection.insertAsync({
_id: rundownPlaylistId,
created: Date.now(),
currentPartInfo: null,
nextPartInfo: null,
externalId: '',
modified: Date.now(),
name: 'Rundown',
previousPartInfo: null,
rundownIdsInOrder: [],
studioId,
timing: {
type: PlaylistTimingType.None,
},
activationId: protectString(''),
})

return {
studioId,
rundownPlaylistId,
}
}

test('Attempts to restart CasparCG when job is enabled', async () => {
const { mockCasparCg, deviceToken } = await createMockPlayoutGatewayAndDevices(Date.now()) // Some time after the threshold

Expand Down Expand Up @@ -605,6 +694,28 @@ describe('cronjobs', () => {
expect(logger.info).toHaveBeenLastCalledWith('Nightly cronjob: done')
}, MAX_WAIT_TIME)
})
test('Skips CasparCG in Studios with active Playlists when job is enabled', async () => {
const { studioId } = await createMockStudioAndRundown()
await createMockPlayoutGatewayAndDevices(Date.now(), studioId) // Some time after the threshold
;(logger.info as jest.Mock).mockClear()
// set time to 2020/07/{date} 04:05 Local Time, should be more than 24 hours after 2020/07/19 00:00 UTC
mockCurrentTime = new Date(2020, 6, date++, 4, 5, 0).getTime()
// cronjob is checked every 5 minutes, so advance 6 minutes
await jest.advanceTimersByTimeAsync(6 * 60 * 1000)

await waitUntil(async () => {
// Run timers, so that all promises in the cronjob has a chance to resolve:
const pendingCommands = await PeripheralDeviceCommands.findFetchAsync({})
expect(pendingCommands).toHaveLength(0)
}, MAX_WAIT_TIME)

// make sure that the cronjob ends
await waitUntil(async () => {
// Run timers, so that all promises in the cronjob has a chance to resolve:
await runAllTimers()
expect(logger.info).toHaveBeenLastCalledWith('Nightly cronjob: done')
}, MAX_WAIT_TIME)
})
test('Does not attempt to restart CasparCG when job is disabled', async () => {
await createMockPlayoutGatewayAndDevices(Date.now()) // Some time after the threshold
await setCasparCGCronEnabled(false)
Expand Down
41 changes: 40 additions & 1 deletion meteor/server/cronjobs.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ import {
translateMessage,
} from '@sofie-automation/corelib/dist/TranslatableMessage'
import { applyAndValidateOverrides } from '@sofie-automation/corelib/dist/settings/objectWithOverrides'
import { DBRundownPlaylist } from '@sofie-automation/corelib/dist/dataModel/RundownPlaylist'
import { StudioId } from '@sofie-automation/corelib/dist/dataModel/Ids'

const lowPrioFcn = (fcn: () => any) => {
// Do it at a random time in the future:
Expand Down Expand Up @@ -101,9 +103,35 @@ async function restartCasparCG(systemSettings: ICoreSystemSettings | undefined,
subType: 1,
parentDeviceId: 1,
lastSeen: 1,
studioAndConfigId: 1,
},
}
)) as Array<Pick<PeripheralDevice, '_id' | 'subType' | 'parentDeviceId' | 'lastSeen'>>
)) as Array<Pick<PeripheralDevice, '_id' | 'subType' | 'parentDeviceId' | 'lastSeen' | 'studioAndConfigId'>>

const relevantStudioIds = Array.from(
new Set(
casparcgAndParentDevices
.map((device) => device.studioAndConfigId?.studioId)
.filter((id) => id !== undefined)
)
) as StudioId[]

const activePlaylists = (await RundownPlaylists.findFetchAsync(
{
activationId: {
$exists: true,
},
studioId: {
$in: relevantStudioIds,
},
},
{
projection: {
_id: 1,
studioId: 1,
},
}
)) as Array<Pick<DBRundownPlaylist, '_id' | 'studioId'>>

const deviceMap = normalizeArrayToMap(casparcgAndParentDevices, '_id')

Expand All @@ -128,6 +156,17 @@ async function restartCasparCG(systemSettings: ICoreSystemSettings | undefined,
continue
}

const activePlaylistUsingDevice = activePlaylists.find(
(playlist) => playlist.studioId === parentDevice.studioAndConfigId?.studioId
)
if (activePlaylistUsingDevice) {
logger.info(
`Cronjob: Skipping CasparCG device "${device._id}" with a parent device belonging to a Studio ("${activePlaylistUsingDevice.studioId}") with an active RundownPlaylist: "${activePlaylistUsingDevice._id}"`
)
// If a Rundown is active during "low season", it's proably best to just let it go until next "low season" the following day, don't retry
continue
}

if (parentDevice.lastSeen < getCurrentTime() - CASPARCG_LAST_SEEN_PERIOD_MS) {
logger.info(`Cronjob: Skipping CasparCG device "${device._id}" with offline parent device`)
shouldRetryAttempt = true
Expand Down
Loading