diff --git a/include/dav1d/headers.rs b/include/dav1d/headers.rs
index 26ef4af79..6bf403526 100644
--- a/include/dav1d/headers.rs
+++ b/include/dav1d/headers.rs
@@ -2,6 +2,7 @@ use crate::src::enum_map::EnumKey;
 use std::ffi::c_int;
 use std::ffi::c_uint;
 use std::ops::BitAnd;
+use std::ops::Deref;
 use std::sync::atomic::AtomicU64;
 use std::sync::atomic::Ordering;
 use strum::EnumCount;
@@ -15,15 +16,6 @@ pub struct DRav1d<R, D> {
     pub dav1d: D,
 }
 
-impl<R, D> DRav1d<R, D>
-where
-    D: Clone + Into<R>,
-{
-    pub fn update_rav1d(&mut self) {
-        self.rav1d = self.dav1d.clone().into();
-    }
-}
-
 impl<R, D> DRav1d<R, D>
 where
     R: Clone + Into<D>,
@@ -34,6 +26,17 @@ where
     }
 }
 
+/// Since the `D`/`Dav1d*` type is only used externally by C,
+/// it's reasonable to `.deref()`
+/// to the `R`/`Rav1d*` type used everywhere internally.
+impl<R, D> Deref for DRav1d<R, D> {
+    type Target = R;
+
+    fn deref(&self) -> &Self::Target {
+        &self.rav1d
+    }
+}
+
 // Constants from Section 3. "Symbols and abbreviated terms"
 pub const DAV1D_MAX_CDEF_STRENGTHS: usize = 8;
 pub const DAV1D_MAX_OPERATING_POINTS: usize = 32;
@@ -346,7 +349,7 @@ impl From<Rav1dPixelLayoutSubSampled> for Rav1dPixelLayout {
 }
 
 #[derive(Clone, Copy, PartialEq, Eq, FromRepr)]
-pub(crate) enum Rav1dFrameType {
+pub enum Rav1dFrameType {
     Key = 0,
     Inter = 1,
     Intra = 2,
@@ -569,7 +572,7 @@ pub struct Dav1dSequenceHeaderOperatingPoint {
 
 #[derive(Clone, Copy, Default, PartialEq, Eq)]
 #[repr(C)]
-pub(crate) struct Rav1dSequenceHeaderOperatingPoint {
+pub struct Rav1dSequenceHeaderOperatingPoint {
     pub major_level: c_int,
     pub minor_level: c_int,
     pub initial_display_delay: c_int,
@@ -635,7 +638,7 @@ pub struct Dav1dSequenceHeaderOperatingParameterInfo {
 
 #[derive(Clone, Copy, Default, PartialEq, Eq)]
 #[repr(C)]
-pub(crate) struct Rav1dSequenceHeaderOperatingParameterInfo {
+pub struct Rav1dSequenceHeaderOperatingParameterInfo {
     pub decoder_buffer_delay: c_int,
     pub encoder_buffer_delay: c_int,
     pub low_delay_mode: c_int,
@@ -732,7 +735,7 @@ pub struct Dav1dSequenceHeader {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dSequenceHeader {
+pub struct Rav1dSequenceHeader {
     pub profile: c_int,
     pub max_width: c_int,
     pub max_height: c_int,
@@ -1151,7 +1154,7 @@ pub struct Dav1dSegmentationData {
 
 #[derive(Clone, Default)]
 #[repr(C)]
-pub(crate) struct Rav1dSegmentationData {
+pub struct Rav1dSegmentationData {
     pub delta_q: c_int,
     pub delta_lf_y_v: c_int,
     pub delta_lf_y_h: c_int,
@@ -1222,7 +1225,7 @@ pub struct Dav1dSegmentationDataSet {
 
 #[derive(Clone, Default)]
 #[repr(C)]
-pub(crate) struct Rav1dSegmentationDataSet {
+pub struct Rav1dSegmentationDataSet {
     pub d: [Rav1dSegmentationData; RAV1D_MAX_SEGMENTS as usize],
     pub preskip: c_int,
     pub last_active_segid: c_int,
@@ -1267,7 +1270,7 @@ pub struct Dav1dLoopfilterModeRefDeltas {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dLoopfilterModeRefDeltas {
+pub struct Rav1dLoopfilterModeRefDeltas {
     pub mode_delta: [c_int; 2],
     pub ref_delta: [c_int; RAV1D_TOTAL_REFS_PER_FRAME],
 }
@@ -1475,7 +1478,7 @@ pub struct Dav1dFrameHeader_film_grain {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_film_grain {
+pub struct Rav1dFrameHeader_film_grain {
     pub data: Rav1dFilmGrainData,
     pub present: c_int,
     pub update: c_int,
@@ -1519,7 +1522,7 @@ pub struct Dav1dFrameHeaderOperatingPoint {
 
 #[derive(Clone, Copy, Default)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeaderOperatingPoint {
+pub struct Rav1dFrameHeaderOperatingPoint {
     pub buffer_removal_time: c_int,
 }
 
@@ -1554,7 +1557,7 @@ pub struct Dav1dFrameHeader_super_res {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_super_res {
+pub struct Rav1dFrameHeader_super_res {
     pub width_scale_denominator: c_int,
     pub enabled: c_int,
 }
@@ -1605,7 +1608,7 @@ pub struct Dav1dFrameHeader_tiling {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_tiling {
+pub struct Rav1dFrameHeader_tiling {
     pub uniform: c_int,
     pub n_bytes: c_uint,
     pub min_log2_cols: c_int,
@@ -1708,7 +1711,7 @@ pub struct Dav1dFrameHeader_quant {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_quant {
+pub struct Rav1dFrameHeader_quant {
     pub yac: c_int,
     pub ydc_delta: c_int,
     pub udc_delta: c_int,
@@ -1793,7 +1796,7 @@ pub struct Dav1dFrameHeader_segmentation {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_segmentation {
+pub struct Rav1dFrameHeader_segmentation {
     pub enabled: c_int,
     pub update_map: c_int,
     pub temporal: c_int,
@@ -1858,7 +1861,7 @@ pub struct Dav1dFrameHeader_delta_q {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_delta_q {
+pub struct Rav1dFrameHeader_delta_q {
     pub present: c_int,
     pub res_log2: c_int,
 }
@@ -1887,7 +1890,7 @@ pub struct Dav1dFrameHeader_delta_lf {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_delta_lf {
+pub struct Rav1dFrameHeader_delta_lf {
     pub present: c_int,
     pub res_log2: c_int,
     pub multi: c_int,
@@ -1932,7 +1935,7 @@ pub struct Dav1dFrameHeader_delta {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_delta {
+pub struct Rav1dFrameHeader_delta {
     pub q: Rav1dFrameHeader_delta_q,
     pub lf: Rav1dFrameHeader_delta_lf,
 }
@@ -1971,7 +1974,7 @@ pub struct Dav1dFrameHeader_loopfilter {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_loopfilter {
+pub struct Rav1dFrameHeader_loopfilter {
     pub level_y: [c_int; 2],
     pub level_u: c_int,
     pub level_v: c_int,
@@ -2038,7 +2041,7 @@ pub struct Dav1dFrameHeader_cdef {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_cdef {
+pub struct Rav1dFrameHeader_cdef {
     pub damping: c_int,
     pub n_bits: c_int,
     pub y_strength: [c_int; RAV1D_MAX_CDEF_STRENGTHS],
@@ -2088,7 +2091,7 @@ pub struct Dav1dFrameHeader_restoration {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader_restoration {
+pub struct Rav1dFrameHeader_restoration {
     pub r#type: [Rav1dRestorationType; 3],
     pub unit_size: [c_int; 2],
 }
@@ -2164,7 +2167,7 @@ pub struct Dav1dFrameHeader {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameSize {
+pub struct Rav1dFrameSize {
     pub width: [c_int; 2],
     pub height: c_int,
     pub render_width: c_int,
@@ -2175,7 +2178,7 @@ pub(crate) struct Rav1dFrameSize {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameSkipMode {
+pub struct Rav1dFrameSkipMode {
     pub allowed: c_int,
     pub enabled: c_int,
     pub refs: [c_int; 2],
@@ -2183,7 +2186,7 @@ pub(crate) struct Rav1dFrameSkipMode {
 
 #[derive(Clone)]
 #[repr(C)]
-pub(crate) struct Rav1dFrameHeader {
+pub struct Rav1dFrameHeader {
     pub size: Rav1dFrameSize,
     pub film_grain: Rav1dFrameHeader_film_grain,
     pub frame_type: Rav1dFrameType,
diff --git a/include/dav1d/picture.rs b/include/dav1d/picture.rs
index 4cfe014ac..8f20209d4 100644
--- a/include/dav1d/picture.rs
+++ b/include/dav1d/picture.rs
@@ -21,7 +21,6 @@ use libc::uintptr_t;
 use std::ffi::c_int;
 use std::ffi::c_void;
 use std::ptr;
-use std::ptr::addr_of_mut;
 use std::ptr::NonNull;
 use std::sync::Arc;
 
@@ -69,8 +68,8 @@ impl From<Rav1dPictureParameters> for Dav1dPictureParameters {
 
 #[repr(C)]
 pub struct Dav1dPicture {
-    pub seq_hdr: *mut Dav1dSequenceHeader,
-    pub frame_hdr: *mut Dav1dFrameHeader,
+    pub seq_hdr: Option<NonNull<Dav1dSequenceHeader>>,
+    pub frame_hdr: Option<NonNull<Dav1dFrameHeader>>,
     pub data: [*mut c_void; 3],
     pub stride: [ptrdiff_t; 2],
     pub p: Dav1dPictureParameters,
@@ -79,8 +78,8 @@ pub struct Dav1dPicture {
     pub mastering_display: Option<NonNull<Rav1dMasteringDisplay>>,
     pub itut_t35: Option<NonNull<Dav1dITUTT35>>,
     pub reserved: [uintptr_t; 4],
-    pub frame_hdr_ref: *mut Dav1dRef,
-    pub seq_hdr_ref: *mut Dav1dRef,
+    pub frame_hdr_ref: Option<RawArc<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>>, // opaque, so we can change this
+    pub seq_hdr_ref: Option<RawArc<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>>, // opaque, so we can change this
     pub content_light_ref: Option<RawArc<Rav1dContentLightLevel>>, // opaque, so we can change this
     pub mastering_display_ref: Option<RawArc<Rav1dMasteringDisplay>>, // opaque, so we can change this
     pub itut_t35_ref: Option<RawArc<DRav1d<Rav1dITUTT35, Dav1dITUTT35>>>, // opaque, so we can change this
@@ -92,8 +91,8 @@ pub struct Dav1dPicture {
 #[derive(Clone)]
 #[repr(C)]
 pub(crate) struct Rav1dPicture {
-    pub seq_hdr: *mut Rav1dSequenceHeader,
-    pub frame_hdr: *mut Rav1dFrameHeader,
+    pub seq_hdr: Option<Arc<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>>,
+    pub frame_hdr: Option<Arc<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>>,
     pub data: [*mut c_void; 3],
     pub stride: [ptrdiff_t; 2],
     pub p: Rav1dPictureParameters,
@@ -101,8 +100,6 @@ pub(crate) struct Rav1dPicture {
     pub content_light: Option<Arc<Rav1dContentLightLevel>>,
     pub mastering_display: Option<Arc<Rav1dMasteringDisplay>>,
     pub itut_t35: Option<Arc<DRav1d<Rav1dITUTT35, Dav1dITUTT35>>>,
-    pub frame_hdr_ref: *mut Rav1dRef,
-    pub seq_hdr_ref: *mut Rav1dRef,
     pub r#ref: *mut Rav1dRef,
     pub allocator_data: *mut c_void,
 }
@@ -110,8 +107,8 @@ pub(crate) struct Rav1dPicture {
 impl From<Dav1dPicture> for Rav1dPicture {
     fn from(value: Dav1dPicture) -> Self {
         let Dav1dPicture {
-            seq_hdr,
-            frame_hdr,
+            seq_hdr: _,
+            frame_hdr: _,
             data,
             stride,
             p,
@@ -129,35 +126,13 @@ impl From<Dav1dPicture> for Rav1dPicture {
             r#ref,
             allocator_data,
         } = value;
-        assert_eq!(seq_hdr.is_null(), seq_hdr_ref.is_null());
-        assert_eq!(frame_hdr.is_null(), frame_hdr_ref.is_null());
         Self {
-            // `.update_rav1d()` happens in `#[no_mangle] extern "C"`/`DAV1D_API` calls
-            seq_hdr: if seq_hdr.is_null() {
-                ptr::null_mut()
-            } else {
-                unsafe {
-                    addr_of_mut!(
-                        (*(seq_hdr_ref.read())
-                            .data
-                            .cast::<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>())
-                        .rav1d
-                    )
-                }
-            },
-            // `.update_rav1d()` happens in `#[no_mangle] extern "C"`/`DAV1D_API` calls
-            frame_hdr: if frame_hdr.is_null() {
-                ptr::null_mut()
-            } else {
-                unsafe {
-                    addr_of_mut!(
-                        (*(frame_hdr_ref.read())
-                            .data
-                            .cast::<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>())
-                        .rav1d
-                    )
-                }
-            },
+            // We don't `.update_rav1d()` [`Rav1dSequenceHeader`] because it's meant to be read-only.
+            // Safety: `raw` came from [`RawArc::from_arc`].
+            seq_hdr: seq_hdr_ref.map(|raw| unsafe { raw.into_arc() }),
+            // We don't `.update_rav1d()` [`Rav1dFrameHeader`] because it's meant to be read-only.
+            // Safety: `raw` came from [`RawArc::from_arc`].
+            frame_hdr: frame_hdr_ref.map(|raw| unsafe { raw.into_arc() }),
             data,
             stride,
             p: p.into(),
@@ -169,8 +144,6 @@ impl From<Dav1dPicture> for Rav1dPicture {
             // We don't `.update_rav1d` [`Rav1dITUTT35`] because never read it.
             // Safety: `raw` came from [`RawArc::from_arc`].
             itut_t35: itut_t35_ref.map(|raw| unsafe { raw.into_arc() }),
-            frame_hdr_ref,
-            seq_hdr_ref,
             r#ref,
             allocator_data,
         }
@@ -189,51 +162,25 @@ impl From<Rav1dPicture> for Dav1dPicture {
             content_light,
             mastering_display,
             itut_t35,
-            frame_hdr_ref,
-            seq_hdr_ref,
             r#ref,
             allocator_data,
         } = value;
-        assert_eq!(seq_hdr.is_null(), seq_hdr_ref.is_null());
-        assert_eq!(frame_hdr.is_null(), frame_hdr_ref.is_null());
         Self {
-            // `DRav1d::from_rav1d` is called right after [`parse_seq_hdr`].
-            seq_hdr: if seq_hdr.is_null() {
-                ptr::null_mut()
-            } else {
-                unsafe {
-                    addr_of_mut!(
-                        (*(seq_hdr_ref.read())
-                            .data
-                            .cast::<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>())
-                        .dav1d
-                    )
-                }
-            },
-            // `DRav1d::from_rav1d` is called in [`parse_frame_hdr`].
-            frame_hdr: if frame_hdr.is_null() {
-                ptr::null_mut()
-            } else {
-                unsafe {
-                    addr_of_mut!(
-                        (*(frame_hdr_ref.read())
-                            .data
-                            .cast::<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>())
-                        .dav1d
-                    )
-                }
-            },
+            // [`DRav1d::from_rav1d`] is called right after [`parse_seq_hdr`].
+            seq_hdr: seq_hdr.as_ref().map(|arc| (&arc.as_ref().dav1d).into()),
+            // [`DRav1d::from_rav1d`] is called in [`parse_frame_hdr`].
+            frame_hdr: frame_hdr.as_ref().map(|arc| (&arc.as_ref().dav1d).into()),
             data,
             stride,
             p: p.into(),
             m: m.into(),
             content_light: content_light.as_ref().map(|arc| arc.as_ref().into()),
             mastering_display: mastering_display.as_ref().map(|arc| arc.as_ref().into()),
-            // `DRav1d::from_rav1d` is called in [`rav1d_parse_obus`].
+            // [`DRav1d::from_rav1d`] is called in [`rav1d_parse_obus`].
             itut_t35: itut_t35.as_ref().map(|arc| (&arc.as_ref().dav1d).into()),
             reserved: Default::default(),
-            frame_hdr_ref,
-            seq_hdr_ref,
+            frame_hdr_ref: frame_hdr.map(RawArc::from_arc),
+            seq_hdr_ref: seq_hdr.map(RawArc::from_arc),
             content_light_ref: content_light.map(RawArc::from_arc),
             mastering_display_ref: mastering_display.map(RawArc::from_arc),
             itut_t35_ref: itut_t35.map(RawArc::from_arc),
@@ -252,8 +199,8 @@ impl From<Rav1dPicture> for Dav1dPicture {
 impl Default for Rav1dPicture {
     fn default() -> Self {
         Self {
-            seq_hdr: ptr::null_mut(),
-            frame_hdr: ptr::null_mut(),
+            seq_hdr: None,
+            frame_hdr: None,
             data: [ptr::null_mut(); 3],
             stride: Default::default(),
             p: Default::default(),
@@ -261,8 +208,6 @@ impl Default for Rav1dPicture {
             content_light: None,
             mastering_display: None,
             itut_t35: None,
-            frame_hdr_ref: ptr::null_mut(),
-            seq_hdr_ref: ptr::null_mut(),
             r#ref: ptr::null_mut(),
             allocator_data: ptr::null_mut(),
         }
diff --git a/src/cdef_apply.rs b/src/cdef_apply.rs
index 88bcc124a..b2d3ffc65 100644
--- a/src/cdef_apply.rs
+++ b/src/cdef_apply.rs
@@ -206,7 +206,8 @@ pub(crate) unsafe fn rav1d_cdef_brow<BD: BitDepth>(
     let mut ptrs: [*mut BD::Pixel; 3] = [*p.offset(0), *p.offset(1), *p.offset(2)];
     let sbsz = 16;
     let sb64w = (*f).sb128w << 1;
-    let damping = (*(*f).frame_hdr).cdef.damping + bitdepth_min_8;
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    let damping = frame_hdr.cdef.damping + bitdepth_min_8;
     let layout: Rav1dPixelLayout = (*f).cur.p.layout;
     let uv_idx =
         (Rav1dPixelLayout::I444 as c_int as c_uint).wrapping_sub(layout as c_uint) as c_int;
@@ -217,8 +218,8 @@ pub(crate) unsafe fn rav1d_cdef_brow<BD: BitDepth>(
         [(layout as c_uint == Rav1dPixelLayout::I422 as c_int as c_uint) as c_int as usize])
         .as_ptr();
     let have_tt = ((*(*f).c).n_tc > 1 as c_uint) as c_int;
-    let sb128 = (*(*f).seq_hdr).sb128;
-    let resize = ((*(*f).frame_hdr).size.width[0] != (*(*f).frame_hdr).size.width[1]) as c_int;
+    let sb128 = (*f).seq_hdr.as_ref().unwrap().sb128;
+    let resize = (frame_hdr.size.width[0] != frame_hdr.size.width[1]) as c_int;
     let y_stride: ptrdiff_t = BD::pxstride((*f).cur.stride[0] as usize) as isize;
     let uv_stride: ptrdiff_t = BD::pxstride((*f).cur.stride[1] as usize) as isize;
     let mut bit = 0;
@@ -276,8 +277,8 @@ pub(crate) unsafe fn rav1d_cdef_brow<BD: BitDepth>(
             let sb64_idx = ((by & sbsz) >> 3) + (sbx & 1);
             let cdef_idx = (*lflvl.offset(sb128x as isize)).cdef_idx[sb64_idx as usize] as c_int;
             if cdef_idx == -(1 as c_int)
-                || (*(*f).frame_hdr).cdef.y_strength[cdef_idx as usize] == 0
-                    && (*(*f).frame_hdr).cdef.uv_strength[cdef_idx as usize] == 0
+                || frame_hdr.cdef.y_strength[cdef_idx as usize] == 0
+                    && frame_hdr.cdef.uv_strength[cdef_idx as usize] == 0
             {
                 last_skip = 1 as c_int;
             } else {
@@ -286,8 +287,8 @@ pub(crate) unsafe fn rav1d_cdef_brow<BD: BitDepth>(
                     .offset(by_idx as isize) as *const [u16; 2];
                 noskip_mask = ((*noskip_row.offset(0))[1] as c_uint) << 16
                     | (*noskip_row.offset(0))[0] as c_uint;
-                y_lvl = (*(*f).frame_hdr).cdef.y_strength[cdef_idx as usize];
-                uv_lvl = (*(*f).frame_hdr).cdef.uv_strength[cdef_idx as usize];
+                y_lvl = frame_hdr.cdef.y_strength[cdef_idx as usize];
+                uv_lvl = frame_hdr.cdef.uv_strength[cdef_idx as usize];
                 flag = ((y_lvl != 0) as c_int + (((uv_lvl != 0) as c_int) << 1)) as Backup2x8Flags;
                 y_pri_lvl = (y_lvl >> 2) << bitdepth_min_8;
                 y_sec_lvl = y_lvl & 3;
diff --git a/src/decode.rs b/src/decode.rs
index c1a3b78ca..7567fe515 100644
--- a/src/decode.rs
+++ b/src/decode.rs
@@ -288,7 +288,7 @@ unsafe fn read_mv_component_diff(
 ) -> c_int {
     let ts = &mut *t.ts;
     let f = &*t.f;
-    let have_hp = (*f.frame_hdr).hp != 0;
+    let have_hp = f.frame_hdr.as_ref().unwrap().hp != 0;
     let sign = rav1d_msac_decode_bool_adapt(&mut ts.msac, &mut mv_comp.sign.0);
     let cl = rav1d_msac_decode_symbol_adapt16(&mut ts.msac, &mut mv_comp.classes.0, 10);
     let mut up;
@@ -1072,9 +1072,10 @@ unsafe fn read_vartx_tree(
     // var-tx tree coding
     let mut tx_split = [0u16; 2];
     *b.max_ytx_mut() = dav1d_max_txfm_size_for_bs[bs as usize][0];
-    let txfm_mode = (*f.frame_hdr).txfm_mode as Dav1dTxfmMode;
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+    let txfm_mode = frame_hdr.txfm_mode as Dav1dTxfmMode;
     if b.skip == 0
-        && ((*f.frame_hdr).segmentation.lossless[b.seg_id as usize] != 0
+        && (frame_hdr.segmentation.lossless[b.seg_id as usize] != 0
             || b.max_ytx() as TxfmSize == TX_4X4)
     {
         b.uvtx = TX_4X4 as u8;
@@ -1154,7 +1155,7 @@ unsafe fn get_prev_frame_segid(
     ref_seg_map: *const u8,
     stride: ptrdiff_t,
 ) -> u8 {
-    assert!((*f.frame_hdr).primary_ref_frame != RAV1D_PRIMARY_REF_NONE);
+    assert!(f.frame_hdr.as_ref().unwrap().primary_ref_frame != RAV1D_PRIMARY_REF_NONE);
 
     // Need checked casts here because an overflowing cast
     // would give a too large `len` to [`std::slice::from_raw_parts`], which would UB.
@@ -1461,7 +1462,7 @@ unsafe fn decode_b(
 
     let ts = &mut *t.ts;
     let f = &*t.f;
-    let frame_hdr = &mut *f.frame_hdr;
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
     let mut b_mem = Default::default();
     let b = if t.frame_thread.pass != 0 {
         &mut *f
@@ -1693,7 +1694,7 @@ unsafe fn decode_b(
     if seg
         .map(|seg| seg.globalmv == 0 && seg.r#ref == -1 && seg.skip == 0)
         .unwrap_or(true)
-        && (*f.frame_hdr).skip_mode.enabled != 0
+        && frame_hdr.skip_mode.enabled != 0
         && cmp::min(bw4, bh4) > 1
     {
         let smctx = (*t.a).skip_mode.0[bx4 as usize] + t.l.skip_mode.0[by4 as usize];
@@ -1720,11 +1721,11 @@ unsafe fn decode_b(
     }
 
     // segment_id
-    if (*f.frame_hdr).segmentation.enabled != 0
-        && (*f.frame_hdr).segmentation.update_map != 0
-        && (*f.frame_hdr).segmentation.seg_data.preskip == 0
+    if frame_hdr.segmentation.enabled != 0
+        && frame_hdr.segmentation.update_map != 0
+        && frame_hdr.segmentation.seg_data.preskip == 0
     {
-        if b.skip == 0 && (*f.frame_hdr).segmentation.temporal != 0 && {
+        if b.skip == 0 && frame_hdr.segmentation.temporal != 0 && {
             let index = (*t.a).seg_pred.0[bx4 as usize] + t.l.seg_pred.0[by4 as usize];
             seg_pred = rav1d_msac_decode_bool_adapt(
                 &mut ts.msac,
@@ -1754,7 +1755,7 @@ unsafe fn decode_b(
                     &mut ts.cdf.m.seg_id[seg_ctx as usize],
                     RAV1D_MAX_SEGMENTS as usize - 1,
                 );
-                let last_active_seg_id = (*f.frame_hdr).segmentation.seg_data.last_active_segid;
+                let last_active_seg_id = frame_hdr.segmentation.seg_data.last_active_segid;
                 b.seg_id =
                     neg_deinterleave(diff as c_int, pred_seg_id as c_int, last_active_seg_id + 1)
                         as u8;
@@ -1767,16 +1768,17 @@ unsafe fn decode_b(
             }
         }
 
-        seg = Some(&(*f.frame_hdr).segmentation.seg_data.d[b.seg_id as usize]);
+        seg = Some(&frame_hdr.segmentation.seg_data.d[b.seg_id as usize]);
 
         if DEBUG_BLOCK_INFO(f, t) {
             println!("Post-segid[postskip;{}]: r={}", b.seg_id, ts.msac.rng);
         }
     }
 
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
     // cdef index
     if b.skip == 0 {
-        let idx = if (*f.seq_hdr).sb128 != 0 {
+        let idx = if seq_hdr.sb128 != 0 {
             ((t.bx & 16) >> 4) + ((t.by & 16) >> 3)
         } else {
             0
@@ -1804,12 +1806,12 @@ unsafe fn decode_b(
     }
 
     // delta-q/lf
-    let not_sb128 = ((*f.seq_hdr).sb128 == 0) as c_int;
+    let not_sb128 = (seq_hdr.sb128 == 0) as c_int;
     if t.bx & (31 >> not_sb128) == 0 && t.by & (31 >> not_sb128) == 0 {
         let prev_qidx = ts.last_qidx;
         let have_delta_q = frame_hdr.delta.q.present != 0
             && (bs
-                != (if (*f.seq_hdr).sb128 != 0 {
+                != (if seq_hdr.sb128 != 0 {
                     BS_128x128
                 } else {
                     BS_64x64
@@ -1883,7 +1885,7 @@ unsafe fn decode_b(
             ts.dq = f.dq.as_ptr();
         } else if ts.last_qidx != prev_qidx {
             // find sb-specific quant parameters
-            init_quant_tables(&*f.seq_hdr, frame_hdr, ts.last_qidx, &mut ts.dqmem);
+            init_quant_tables(seq_hdr, frame_hdr, ts.last_qidx, &mut ts.dqmem);
             ts.dq = ts.dqmem.as_ptr();
         }
         if ts.last_delta_lf == [0, 0, 0, 0] {
@@ -2052,7 +2054,7 @@ unsafe fn decode_b(
         if b.y_mode() == DC_PRED
             && b.pal_sz()[0] == 0
             && cmp::max(b_dim[2], b_dim[3]) <= 3
-            && (*f.seq_hdr).filter_intra != 0
+            && seq_hdr.filter_intra != 0
         {
             let is_filter = rav1d_msac_decode_bool_adapt(
                 &mut ts.msac,
@@ -2292,11 +2294,11 @@ unsafe fn decode_b(
             b.mv_mut()[0] = mvstack[0].mv.mv[0];
         } else if mvstack[1].mv.mv[0] != mv::ZERO {
             b.mv_mut()[0] = mvstack[1].mv.mv[0];
-        } else if t.by - (16 << (*f.seq_hdr).sb128) < ts.tiling.row_start {
+        } else if t.by - (16 << seq_hdr.sb128) < ts.tiling.row_start {
             b.mv_mut()[0].y = 0;
-            b.mv_mut()[0].x = (-(512 << (*f.seq_hdr).sb128) - 2048) as i16;
+            b.mv_mut()[0].x = (-(512 << seq_hdr.sb128) - 2048) as i16;
         } else {
-            b.mv_mut()[0].y = -(512 << (*f.seq_hdr).sb128) as i16;
+            b.mv_mut()[0].y = -(512 << seq_hdr.sb128) as i16;
             b.mv_mut()[0].x = 0;
         }
 
@@ -2334,9 +2336,9 @@ unsafe fn decode_b(
             src_top += border_top - src_top;
         }
 
-        let sbx = t.bx >> 4 + (*f.seq_hdr).sb128 << 6 + (*f.seq_hdr).sb128;
-        let sby = t.by >> 4 + (*f.seq_hdr).sb128 << 6 + (*f.seq_hdr).sb128;
-        let sb_size = 1 << 6 + (*f.seq_hdr).sb128;
+        let sbx = t.bx >> 4 + seq_hdr.sb128 << 6 + seq_hdr.sb128;
+        let sby = t.by >> 4 + seq_hdr.sb128 << 6 + seq_hdr.sb128;
+        let sb_size = 1 << 6 + seq_hdr.sb128;
         // check for overlap with current superblock
         if src_bottom > sby && src_right > sbx {
             if src_top - border_top >= src_bottom - sby {
@@ -2676,7 +2678,7 @@ unsafe fn decode_b(
 
             // jnt_comp vs. seg vs. wedge
             let mut is_segwedge = false;
-            if (*f.seq_hdr).masked_compound != 0 {
+            if seq_hdr.masked_compound != 0 {
                 let mask_ctx = get_mask_comp_ctx(&*t.a, &t.l, by4, bx4);
                 is_segwedge = rav1d_msac_decode_bool_adapt(
                     &mut ts.msac,
@@ -2691,12 +2693,20 @@ unsafe fn decode_b(
             }
 
             if !is_segwedge {
-                if (*f.seq_hdr).jnt_comp != 0 {
+                if seq_hdr.jnt_comp != 0 {
+                    let [ref0poc, ref1poc] = b.r#ref().map(|r#ref| {
+                        f.refp[r#ref as usize]
+                            .p
+                            .frame_hdr
+                            .as_ref()
+                            .unwrap()
+                            .frame_offset as c_uint
+                    });
                     let jnt_ctx = get_jnt_comp_ctx(
-                        (*f.seq_hdr).order_hint_n_bits,
-                        (*f.cur.frame_hdr).frame_offset as c_uint,
-                        (*f.refp[b.r#ref()[0] as usize].p.frame_hdr).frame_offset as c_uint,
-                        (*f.refp[b.r#ref()[1] as usize].p.frame_hdr).frame_offset as c_uint,
+                        seq_hdr.order_hint_n_bits,
+                        f.cur.frame_hdr.as_ref().unwrap().frame_offset as c_uint,
+                        ref0poc,
+                        ref1poc,
                         &*t.a,
                         &t.l,
                         by4,
@@ -2952,7 +2962,7 @@ unsafe fn decode_b(
 
             // interintra flags
             let ii_sz_grp = dav1d_ymode_size_context[bs as usize] as c_int;
-            if (*f.seq_hdr).inter_intra != 0
+            if seq_hdr.inter_intra != 0
                 && interintra_allowed_mask & (1 << bs) != 0
                 && rav1d_msac_decode_bool_adapt(
                     &mut ts.msac,
@@ -2981,7 +2991,7 @@ unsafe fn decode_b(
                 *b.interintra_type_mut() = INTER_INTRA_NONE;
             }
             if DEBUG_BLOCK_INFO(f, t)
-                && (*f.seq_hdr).inter_intra != 0
+                && seq_hdr.inter_intra != 0
                 && interintra_allowed_mask & (1 << bs) != 0
             {
                 println!(
@@ -3092,7 +3102,7 @@ unsafe fn decode_b(
                     &mut ts.cdf.m.filter.0[0][ctx1 as usize],
                     RAV1D_N_SWITCHABLE_FILTERS as usize - 1,
                 ) as Dav1dFilterMode;
-                if (*f.seq_hdr).dual_filter != 0 {
+                if seq_hdr.dual_filter != 0 {
                     let ctx2 = get_filter_ctx(&*t.a, &t.l, comp, true, b.r#ref()[0], by4, bx4);
                     if DEBUG_BLOCK_INFO(f, t) {
                         println!(
@@ -3481,6 +3491,8 @@ unsafe fn decode_sb(
         return decode_sb(t, bl + 1, (*(node as *const EdgeBranch)).split[0]);
     }
 
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+
     let bp;
     let mut ctx = 0;
     let mut bx8 = 0;
@@ -3491,11 +3503,7 @@ unsafe fn decode_sb(
         if false && bl == BL_64X64 {
             println!(
                 "poc={},y={},x={},bl={},r={}",
-                (*f.frame_hdr).frame_offset,
-                t.by,
-                t.bx,
-                bl,
-                ts.msac.rng,
+                frame_hdr.frame_offset, t.by, t.bx, bl, ts.msac.rng,
             );
         }
         bx8 = (t.bx & 31) >> 1;
@@ -3522,13 +3530,7 @@ unsafe fn decode_sb(
             if DEBUG_BLOCK_INFO(f, t) {
                 println!(
                     "poc={},y={},x={},bl={},ctx={},bp={}: r={}",
-                    (*f.frame_hdr).frame_offset,
-                    t.by,
-                    t.bx,
-                    bl,
-                    ctx,
-                    bp,
-                    ts.msac.rng,
+                    frame_hdr.frame_offset, t.by, t.bx, bl, ctx, bp, ts.msac.rng,
                 );
             }
         } else {
@@ -3669,7 +3671,7 @@ unsafe fn decode_sb(
             if DEBUG_BLOCK_INFO(f, t) {
                 println!(
                     "poc={},y={},x={},bl={},ctx={},bp={}: r={}",
-                    (*f.frame_hdr).frame_offset,
+                    frame_hdr.frame_offset,
                     t.by,
                     t.bx,
                     bl,
@@ -3716,7 +3718,7 @@ unsafe fn decode_sb(
             if DEBUG_BLOCK_INFO(f, t) {
                 println!(
                     "poc={},y={},x={},bl={},ctx={},bp={}: r={}",
-                    (*f.frame_hdr).frame_offset,
+                    frame_hdr.frame_offset,
                     t.by,
                     t.bx,
                     bl,
@@ -3827,11 +3829,14 @@ unsafe fn setup_tile(
     tile_col: usize,
     tile_start_off: usize,
 ) {
-    let col_sb_start = (*f.frame_hdr).tiling.col_start_sb[tile_col] as c_int;
-    let col_sb128_start = col_sb_start >> ((*f.seq_hdr).sb128 == 0) as c_int;
-    let col_sb_end = (*f.frame_hdr).tiling.col_start_sb[tile_col + 1] as c_int;
-    let row_sb_start = (*f.frame_hdr).tiling.row_start_sb[tile_row] as c_int;
-    let row_sb_end = (*f.frame_hdr).tiling.row_start_sb[tile_row + 1] as c_int;
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+
+    let col_sb_start = frame_hdr.tiling.col_start_sb[tile_col] as c_int;
+    let col_sb128_start = col_sb_start >> (seq_hdr.sb128 == 0) as c_int;
+    let col_sb_end = frame_hdr.tiling.col_start_sb[tile_col + 1] as c_int;
+    let row_sb_start = frame_hdr.tiling.row_start_sb[tile_row] as c_int;
+    let row_sb_end = frame_hdr.tiling.row_start_sb[tile_row + 1] as c_int;
     let sb_shift = f.sb_shift;
 
     let size_mul = &ss_size_mul[f.cur.p.layout];
@@ -3848,8 +3853,7 @@ unsafe fn setup_tile(
                 .cf
                 .cast::<u8>()
                 .offset(
-                    (tile_start_off * size_mul[0] as usize >> ((*f.seq_hdr).hbd == 0) as c_int)
-                        as isize,
+                    (tile_start_off * size_mul[0] as usize >> (seq_hdr.hbd == 0) as c_int) as isize,
                 )
                 .cast::<DynCoef>()
         } else {
@@ -3858,14 +3862,14 @@ unsafe fn setup_tile(
     }
 
     rav1d_cdf_thread_copy(&mut ts.cdf, &f.in_cdf);
-    ts.last_qidx = (*f.frame_hdr).quant.yac;
+    ts.last_qidx = frame_hdr.quant.yac;
     ts.last_delta_lf.fill(0);
 
     rav1d_msac_init(
         &mut ts.msac,
         data.as_ptr(),
         data.len(),
-        (*f.frame_hdr).disable_cdf_update != 0,
+        frame_hdr.disable_cdf_update != 0,
     );
 
     ts.tiling.row = tile_row as c_int;
@@ -3874,7 +3878,7 @@ unsafe fn setup_tile(
     ts.tiling.col_end = cmp::min(col_sb_end << sb_shift, f.bw);
     ts.tiling.row_start = row_sb_start << sb_shift;
     ts.tiling.row_end = cmp::min(row_sb_end << sb_shift, f.bh);
-    let diff_width = (*f.frame_hdr).size.width[0] != (*f.frame_hdr).size.width[1];
+    let diff_width = frame_hdr.size.width[0] != frame_hdr.size.width[1];
 
     // Reference Restoration Unit (used for exp coding)
     let (sb_idx, unit_idx) = if diff_width {
@@ -3896,8 +3900,8 @@ unsafe fn setup_tile(
 
         let lr_ref = if diff_width {
             let ss_hor = (p != 0 && f.cur.p.layout != Rav1dPixelLayout::I444) as c_int;
-            let d = (*f.frame_hdr).size.super_res.width_scale_denominator;
-            let unit_size_log2 = (*f.frame_hdr).restoration.unit_size[(p != 0) as usize];
+            let d = frame_hdr.size.super_res.width_scale_denominator;
+            let unit_size_log2 = frame_hdr.restoration.unit_size[(p != 0) as usize];
             let rnd = (8 << unit_size_log2) - 1;
             let shift = unit_size_log2 + 3;
             let x = (4 * ts.tiling.col_start * d >> ss_hor) + rnd >> shift;
@@ -4028,7 +4032,8 @@ unsafe fn read_restoration_info(
 
 pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result<(), ()> {
     let f = &*t.f;
-    let root_bl = if (*f.seq_hdr).sb128 != 0 {
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
+    let root_bl = if seq_hdr.sb128 != 0 {
         BL_128X128
     } else {
         BL_64X64
@@ -4038,10 +4043,11 @@ pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result
     let sb_step = f.sb_step;
     let tile_row = ts.tiling.row;
     let tile_col = ts.tiling.col;
-    let col_sb_start = (*f.frame_hdr).tiling.col_start_sb[tile_col as usize] as c_int;
-    let col_sb128_start = col_sb_start >> ((*f.seq_hdr).sb128 == 0) as c_int;
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+    let col_sb_start = frame_hdr.tiling.col_start_sb[tile_col as usize] as c_int;
+    let col_sb128_start = col_sb_start >> (seq_hdr.sb128 == 0) as c_int;
 
-    if (*f.frame_hdr).frame_type.is_inter_or_switch() || (*f.frame_hdr).allow_intrabc != 0 {
+    if frame_hdr.frame_type.is_inter_or_switch() || frame_hdr.allow_intrabc != 0 {
         rav1d_refmvs_tile_sbrow_init(
             &mut t.rt,
             &f.rf,
@@ -4055,19 +4061,19 @@ pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result
         );
     }
 
-    if (*f.frame_hdr).frame_type.is_inter_or_switch() && c.n_fc > 1 {
+    if frame_hdr.frame_type.is_inter_or_switch() && c.n_fc > 1 {
         let sby = t.by - ts.tiling.row_start >> f.sb_shift;
         *ts.lowest_pixel.offset(sby as isize) = [[i32::MIN; 2]; 7];
     }
 
     reset_context(
         &mut t.l,
-        (*f.frame_hdr).frame_type.is_key_or_intra(),
+        frame_hdr.frame_type.is_key_or_intra(),
         t.frame_thread.pass,
     );
     if t.frame_thread.pass == 2 {
         let off_2pass = if c.n_tc > 1 {
-            f.sb128w * (*f.frame_hdr).tiling.rows
+            f.sb128w * frame_hdr.tiling.rows
         } else {
             0
         };
@@ -4079,7 +4085,7 @@ pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result
                 return Err(());
             }
             decode_sb(t, root_bl, c.intra_edge.root[root_bl as usize])?;
-            if t.bx & 16 != 0 || (*f.seq_hdr).sb128 != 0 {
+            if t.bx & 16 != 0 || seq_hdr.sb128 != 0 {
                 t.a = (t.a).offset(1);
             }
         }
@@ -4092,7 +4098,7 @@ pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result
         return Err(());
     }
 
-    if (*f.c).n_tc > 1 && (*f.frame_hdr).use_ref_frame_mvs != 0 {
+    if (*f.c).n_tc > 1 && frame_hdr.use_ref_frame_mvs != 0 {
         (*f.c)
             .refmvs_dsp
             .load_tmvs
@@ -4133,7 +4139,7 @@ pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result
 
             let ss_ver = (p != 0 && f.cur.p.layout == Rav1dPixelLayout::I420) as c_int;
             let ss_hor = (p != 0 && f.cur.p.layout != Rav1dPixelLayout::I444) as c_int;
-            let unit_size_log2 = (*f.frame_hdr).restoration.unit_size[(p != 0) as usize];
+            let unit_size_log2 = frame_hdr.restoration.unit_size[(p != 0) as usize];
             let y = t.by * 4 >> ss_ver;
             let h = f.cur.p.h + ss_ver >> ss_ver;
 
@@ -4149,13 +4155,13 @@ pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result
                 continue;
             }
 
-            let frame_type = (*f.frame_hdr).restoration.r#type[p as usize];
+            let frame_type = frame_hdr.restoration.r#type[p as usize];
 
-            if (*f.frame_hdr).size.width[0] != (*f.frame_hdr).size.width[1] {
+            if frame_hdr.size.width[0] != frame_hdr.size.width[1] {
                 let w = f.sr_cur.p.p.w + ss_hor >> ss_hor;
                 let n_units = cmp::max(1, w + half_unit >> unit_size_log2);
 
-                let d = (*f.frame_hdr).size.super_res.width_scale_denominator;
+                let d = frame_hdr.size.super_res.width_scale_denominator;
                 let rnd = unit_size * 8 - 1;
                 let shift = unit_size_log2 + 3;
                 let x0 = (4 * t.bx * d >> ss_hor) + rnd >> shift;
@@ -4189,16 +4195,13 @@ pub(crate) unsafe fn rav1d_decode_tile_sbrow(t: &mut Rav1dTaskContext) -> Result
             }
         }
         decode_sb(t, root_bl, c.intra_edge.root[root_bl as usize])?;
-        if t.bx & 16 != 0 || (*f.seq_hdr).sb128 != 0 {
+        if t.bx & 16 != 0 || seq_hdr.sb128 != 0 {
             t.a = (t.a).offset(1);
             t.lf_mask = (t.lf_mask).offset(1);
         }
     }
 
-    if (*f.seq_hdr).ref_frame_mvs != 0
-        && (*f.c).n_tc > 1
-        && (*f.frame_hdr).frame_type.is_inter_or_switch()
-    {
+    if seq_hdr.ref_frame_mvs != 0 && (*f.c).n_tc > 1 && frame_hdr.frame_type.is_inter_or_switch() {
         rav1d_refmvs_save_tmvs(
             &(*f.c).refmvs_dsp,
             &mut t.rt,
@@ -4251,17 +4254,18 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
         }
         f.lf.start_of_tile_row_sz = f.sbh;
     }
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
     let mut sby = 0;
-    for tile_row in 0..(*f.frame_hdr).tiling.rows {
+    for tile_row in 0..frame_hdr.tiling.rows {
         *f.lf.start_of_tile_row.offset(sby as isize) = tile_row as u8;
         sby += 1;
-        while sby < (*f.frame_hdr).tiling.row_start_sb[(tile_row + 1) as usize] as c_int {
+        while sby < frame_hdr.tiling.row_start_sb[(tile_row + 1) as usize] as c_int {
             *f.lf.start_of_tile_row.offset(sby as isize) = 0;
             sby += 1;
         }
     }
 
-    let n_ts = (*f.frame_hdr).tiling.cols * (*f.frame_hdr).tiling.rows;
+    let n_ts = frame_hdr.tiling.cols * frame_hdr.tiling.rows;
     if n_ts != f.n_ts {
         if c.n_fc > 1 {
             freep(&mut f.frame_thread.tile_start_off as *mut *mut c_int as *mut c_void);
@@ -4281,7 +4285,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
         f.n_ts = n_ts;
     }
 
-    let a_sz = f.sb128w * (*f.frame_hdr).tiling.rows * (1 + (c.n_fc > 1 && c.n_tc > 1) as c_int);
+    let a_sz = f.sb128w * frame_hdr.tiling.rows * (1 + (c.n_fc > 1 && c.n_tc > 1) as c_int);
     if a_sz != f.a_sz {
         freep(&mut f.a as *mut *mut BlockContext as *mut c_void);
         f.a = malloc(::core::mem::size_of::<BlockContext>() * a_sz as usize) as *mut BlockContext;
@@ -4294,23 +4298,24 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
 
     let num_sb128 = f.sb128w * f.sb128h;
     let size_mul = &ss_size_mul[f.cur.p.layout];
-    let hbd = ((*f.seq_hdr).hbd != 0) as c_int;
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
+    let hbd = (seq_hdr.hbd != 0) as c_int;
     if c.n_fc > 1 {
         let mut tile_idx = 0;
-        for tile_row in 0..(*f.frame_hdr).tiling.rows {
-            let row_off = (*f.frame_hdr).tiling.row_start_sb[tile_row as usize] as c_int
+        for tile_row in 0..frame_hdr.tiling.rows {
+            let row_off = frame_hdr.tiling.row_start_sb[tile_row as usize] as c_int
                 * f.sb_step
                 * 4
                 * f.sb128w
                 * 128;
-            let b_diff = ((*f.frame_hdr).tiling.row_start_sb[(tile_row + 1) as usize] as c_int
-                - (*f.frame_hdr).tiling.row_start_sb[tile_row as usize] as c_int)
+            let b_diff = (frame_hdr.tiling.row_start_sb[(tile_row + 1) as usize] as c_int
+                - frame_hdr.tiling.row_start_sb[tile_row as usize] as c_int)
                 * f.sb_step
                 * 4;
-            for tile_col in 0..(*f.frame_hdr).tiling.cols {
+            for tile_col in 0..frame_hdr.tiling.cols {
                 *f.frame_thread.tile_start_off.offset(tile_idx as isize) = row_off
                     + b_diff
-                        * (*f.frame_hdr).tiling.col_start_sb[tile_col as usize] as c_int
+                        * frame_hdr.tiling.col_start_sb[tile_col as usize] as c_int
                         * f.sb_step
                         * 4;
 
@@ -4318,7 +4323,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
             }
         }
 
-        let lowest_pixel_mem_sz = (*f.frame_hdr).tiling.cols * f.sbh;
+        let lowest_pixel_mem_sz = frame_hdr.tiling.cols * f.sbh;
         if lowest_pixel_mem_sz != f.tile_thread.lowest_pixel_mem_sz {
             free(f.tile_thread.lowest_pixel_mem as *mut c_void);
             f.tile_thread.lowest_pixel_mem =
@@ -4331,12 +4336,11 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
             f.tile_thread.lowest_pixel_mem_sz = lowest_pixel_mem_sz;
         }
         let mut lowest_pixel_ptr = f.tile_thread.lowest_pixel_mem;
-        for tile_row in 0..(*f.frame_hdr).tiling.rows {
-            let tile_row_base = tile_row * (*f.frame_hdr).tiling.cols;
-            let tile_row_sb_h = (*f.frame_hdr).tiling.row_start_sb[(tile_row + 1) as usize]
-                as c_int
-                - (*f.frame_hdr).tiling.row_start_sb[tile_row as usize] as c_int;
-            for tile_col in 0..(*f.frame_hdr).tiling.cols {
+        for tile_row in 0..frame_hdr.tiling.rows {
+            let tile_row_base = tile_row * frame_hdr.tiling.cols;
+            let tile_row_sb_h = frame_hdr.tiling.row_start_sb[(tile_row + 1) as usize] as c_int
+                - frame_hdr.tiling.row_start_sb[tile_row as usize] as c_int;
+            for tile_col in 0..frame_hdr.tiling.cols {
                 (*f.ts.offset((tile_row_base + tile_col) as isize)).lowest_pixel = lowest_pixel_ptr;
                 lowest_pixel_ptr = lowest_pixel_ptr.offset(tile_row_sb_h as isize);
             }
@@ -4359,7 +4363,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
             f.frame_thread.cf_sz = cf_sz;
         }
 
-        if (*f.frame_hdr).allow_screen_content_tools != 0 {
+        if frame_hdr.allow_screen_content_tools != 0 {
             if num_sb128 != f.frame_thread.pal_sz {
                 rav1d_freep_aligned(
                     &mut f.frame_thread.pal as *mut *mut [[u16; 8]; 3] as *mut c_void,
@@ -4399,7 +4403,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
     // update allocation of block contexts for above
     let mut y_stride = f.cur.stride[0];
     let mut uv_stride = f.cur.stride[1];
-    let has_resize = ((*f.frame_hdr).size.width[0] != (*f.frame_hdr).size.width[1]) as c_int;
+    let has_resize = (frame_hdr.size.width[0] != frame_hdr.size.width[1]) as c_int;
     let need_cdef_lpf_copy = (c.n_tc > 1 && has_resize != 0) as c_int;
     if y_stride * f.sbh as isize * 4 != f.lf.cdef_buf_plane_sz[0] as isize
         || uv_stride * f.sbh as isize * 8 != f.lf.cdef_buf_plane_sz[1] as isize
@@ -4471,7 +4475,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
         f.lf.cdef_buf_sbh = f.sbh;
     }
 
-    let sb128 = (*f.seq_hdr).sb128;
+    let sb128 = seq_hdr.sb128;
     let num_lines = if c.n_tc > 1 { (f.sbh * 4) << sb128 } else { 12 };
     y_stride = f.sr_cur.p.stride[0];
     uv_stride = f.sr_cur.p.stride[1];
@@ -4556,7 +4560,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
         }
         f.lf.lr_mask_sz = lr_mask_sz;
     }
-    f.lf.restore_planes = (*f.frame_hdr)
+    f.lf.restore_planes = frame_hdr
         .restoration
         .r#type
         .iter()
@@ -4564,11 +4568,11 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
         .map(|(i, &r#type)| ((r#type != RAV1D_RESTORATION_NONE) as u8) << i)
         .sum::<u8>()
         .into();
-    if (*f.frame_hdr).loopfilter.sharpness != f.lf.last_sharpness {
-        rav1d_calc_eih(&mut f.lf.lim_lut.0, (*f.frame_hdr).loopfilter.sharpness);
-        f.lf.last_sharpness = (*f.frame_hdr).loopfilter.sharpness;
+    if frame_hdr.loopfilter.sharpness != f.lf.last_sharpness {
+        rav1d_calc_eih(&mut f.lf.lim_lut.0, frame_hdr.loopfilter.sharpness);
+        f.lf.last_sharpness = frame_hdr.loopfilter.sharpness;
     }
-    rav1d_calc_lf_values(&mut f.lf.lvl, &*f.frame_hdr, &[0, 0, 0, 0]);
+    rav1d_calc_lf_values(&mut f.lf.lvl, &frame_hdr, &[0, 0, 0, 0]);
     slice::from_raw_parts_mut(f.lf.mask, num_sb128.try_into().unwrap()).fill_with(Default::default);
 
     let ipred_edge_sz = f.sbh * f.sb128w << hbd;
@@ -4588,7 +4592,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
         f.ipred_edge_sz = ipred_edge_sz;
     }
 
-    let re_sz = f.sb128h * (*f.frame_hdr).tiling.cols;
+    let re_sz = f.sb128h * frame_hdr.tiling.cols;
     if re_sz != f.lf.re_sz {
         freep(&mut *f.lf.tx_lpf_right_edge.as_mut_ptr().offset(0) as *mut *mut u8 as *mut c_void);
         f.lf.tx_lpf_right_edge[0] = malloc(re_sz as usize * 32 * 2) as *mut u8;
@@ -4601,11 +4605,11 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
     }
 
     // init ref mvs
-    if (*f.frame_hdr).frame_type.is_inter_or_switch() || (*f.frame_hdr).allow_intrabc != 0 {
+    if frame_hdr.frame_type.is_inter_or_switch() || frame_hdr.allow_intrabc != 0 {
         let ret = rav1d_refmvs_init_frame(
             &mut f.rf,
-            f.seq_hdr,
-            f.frame_hdr,
+            seq_hdr,
+            frame_hdr,
             f.refpoc.as_ptr(),
             f.mvs,
             f.refrefpoc.as_ptr(),
@@ -4619,19 +4623,14 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
     }
 
     // setup dequant tables
-    init_quant_tables(
-        &*f.seq_hdr,
-        &*f.frame_hdr,
-        (*f.frame_hdr).quant.yac,
-        &mut f.dq,
-    );
-    if (*f.frame_hdr).quant.qm != 0 {
+    init_quant_tables(&seq_hdr, &frame_hdr, frame_hdr.quant.yac, &mut f.dq);
+    if frame_hdr.quant.qm != 0 {
         for i in 0..N_RECT_TX_SIZES {
-            f.qm[i][0] = dav1d_qm_tbl[(*f.frame_hdr).quant.qm_y as usize][0][i]
+            f.qm[i][0] = dav1d_qm_tbl[frame_hdr.quant.qm_y as usize][0][i]
                 .map_or(std::ptr::null(), |qm| qm.as_ptr());
-            f.qm[i][1] = dav1d_qm_tbl[(*f.frame_hdr).quant.qm_u as usize][1][i]
+            f.qm[i][1] = dav1d_qm_tbl[frame_hdr.quant.qm_u as usize][1][i]
                 .map_or(std::ptr::null(), |qm| qm.as_ptr());
-            f.qm[i][2] = dav1d_qm_tbl[(*f.frame_hdr).quant.qm_v as usize][1][i]
+            f.qm[i][2] = dav1d_qm_tbl[frame_hdr.quant.qm_v as usize][1][i]
                 .map_or(std::ptr::null(), |qm| qm.as_ptr());
         }
     } else {
@@ -4639,16 +4638,17 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
     }
 
     // setup jnt_comp weights
-    if (*f.frame_hdr).switchable_comp_refs != 0 {
-        let ref_pocs: [_; 7] = array::from_fn(|i| (*f.refp[i].p.frame_hdr).frame_offset);
+    if frame_hdr.switchable_comp_refs != 0 {
+        let ref_pocs: [_; 7] =
+            array::from_fn(|i| f.refp[i].p.frame_hdr.as_ref().unwrap().frame_offset);
         for i in 0..ref_pocs.len() {
             for j in i + 1..ref_pocs.len() {
                 let d = [j, i].map(|ij| {
                     cmp::min(
                         (get_poc_diff(
-                            (*f.seq_hdr).order_hint_n_bits,
+                            seq_hdr.order_hint_n_bits,
                             ref_pocs[ij],
-                            (*f.cur.frame_hdr).frame_offset,
+                            f.cur.frame_hdr.as_ref().unwrap().frame_offset,
                         ))
                         .unsigned_abs(),
                         31,
@@ -4688,14 +4688,15 @@ pub(crate) unsafe fn rav1d_decode_frame_init(f: &mut Rav1dFrameContext) -> Rav1d
 
 pub(crate) unsafe fn rav1d_decode_frame_init_cdf(f: &mut Rav1dFrameContext) -> Rav1dResult {
     let c = &*f.c;
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
 
-    if (*f.frame_hdr).refresh_context != 0 {
+    if frame_hdr.refresh_context != 0 {
         rav1d_cdf_thread_copy(f.out_cdf.data.cdf, &mut f.in_cdf);
     }
 
     let uses_2pass = c.n_fc > 1;
 
-    let tiling = &(*f.frame_hdr).tiling;
+    let tiling = &frame_hdr.tiling;
 
     let n_bytes = tiling.n_bytes.try_into().unwrap();
     let rows: usize = tiling.rows.try_into().unwrap();
@@ -4753,7 +4754,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init_cdf(f: &mut Rav1dFrameContext) -> R
                 tile_col = 0;
                 tile_row += 1;
             }
-            if j == tiling.update as usize && (*f.frame_hdr).refresh_context != 0 {
+            if j == tiling.update as usize && frame_hdr.refresh_context != 0 {
                 f.task_thread.update_set = true;
             }
             data = rest_data;
@@ -4767,7 +4768,7 @@ pub(crate) unsafe fn rav1d_decode_frame_init_cdf(f: &mut Rav1dFrameContext) -> R
         {
             reset_context(
                 ctx,
-                (*f.frame_hdr).frame_type.is_key_or_intra(),
+                frame_hdr.frame_type.is_key_or_intra(),
                 if uses_2pass {
                     1 + (n >= sb128w * rows) as c_int
                 } else {
@@ -4789,19 +4790,22 @@ unsafe fn rav1d_decode_frame_main(f: &mut Rav1dFrameContext) -> Rav1dResult {
     t.f = f;
     t.frame_thread.pass = 0;
 
-    for ctx in slice::from_raw_parts_mut(
-        f.a,
-        (f.sb128w * (*f.frame_hdr).tiling.rows).try_into().unwrap(),
-    ) {
-        reset_context(ctx, (*f.frame_hdr).frame_type.is_key_or_intra(), 0);
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+
+    for ctx in
+        slice::from_raw_parts_mut(f.a, (f.sb128w * frame_hdr.tiling.rows).try_into().unwrap())
+    {
+        reset_context(ctx, frame_hdr.frame_type.is_key_or_intra(), 0);
     }
 
     // no threading - we explicitly interleave tile/sbrow decoding
     // and post-filtering, so that the full process runs in-line
-    let Rav1dFrameHeader_tiling { rows, cols, .. } = (*f.frame_hdr).tiling;
+    let Rav1dFrameHeader_tiling { rows, cols, .. } = frame_hdr.tiling;
     let [rows, cols] = [rows, cols].map(|it| it.try_into().unwrap());
+    // Need to clone this because `(f.bd_fn.filter_sbrow)(f, sby);` takes a `&mut` to `f` within the loop.
+    let row_start_sb = frame_hdr.tiling.row_start_sb.clone();
     for (tile_row, (sbh_start_end, ts)) in iter::zip(
-        (*f.frame_hdr).tiling.row_start_sb[..rows + 1].windows(2),
+        row_start_sb[..rows + 1].windows(2),
         slice::from_raw_parts_mut(f.ts, rows * cols).chunks_exact_mut(cols),
     )
     .enumerate()
@@ -4812,9 +4816,11 @@ unsafe fn rav1d_decode_frame_main(f: &mut Rav1dFrameContext) -> Rav1dResult {
         let sbh_end = cmp::min(sbh_end.into(), f.sbh);
 
         for sby in sbh_start.into()..sbh_end {
-            t.by = sby << 4 + (*f.seq_hdr).sb128;
+            let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
+            let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+            t.by = sby << 4 + seq_hdr.sb128;
             let by_end = t.by + f.sb_step >> 1;
-            if (*f.frame_hdr).use_ref_frame_mvs != 0 {
+            if frame_hdr.use_ref_frame_mvs != 0 {
                 ((*f.c).refmvs_dsp.load_tmvs).expect("non-null function pointer")(
                     &mut f.rf,
                     tile_row as c_int,
@@ -4828,7 +4834,7 @@ unsafe fn rav1d_decode_frame_main(f: &mut Rav1dFrameContext) -> Rav1dResult {
                 t.ts = tile;
                 rav1d_decode_tile_sbrow(t).map_err(|()| EINVAL)?;
             }
-            if (*f.frame_hdr).frame_type.is_inter_or_switch() {
+            if frame_hdr.frame_type.is_inter_or_switch() {
                 rav1d_refmvs_save_tmvs(
                     &(*f.c).refmvs_dsp,
                     &mut t.rt,
@@ -4861,7 +4867,7 @@ pub(crate) unsafe fn rav1d_decode_frame_exit(f: &mut Rav1dFrameContext, retval:
     }
     // TODO(kkysen) use array::zip when stable
     for i in 0..7 {
-        if !f.refp[i].p.frame_hdr.is_null() {
+        if f.refp[i].p.frame_hdr.is_some() {
             rav1d_thread_picture_unref(&mut f.refp[i]);
         }
         rav1d_ref_dec(&mut f.ref_mvs_ref[i]);
@@ -4869,20 +4875,23 @@ pub(crate) unsafe fn rav1d_decode_frame_exit(f: &mut Rav1dFrameContext, retval:
     rav1d_picture_unref_internal(&mut f.cur);
     rav1d_thread_picture_unref(&mut f.sr_cur);
     rav1d_cdf_thread_unref(&mut f.in_cdf);
-    if !f.frame_hdr.is_null() && (*f.frame_hdr).refresh_context != 0 {
-        if !f.out_cdf.progress.is_null() {
-            ::core::intrinsics::atomic_store_seqcst(
-                f.out_cdf.progress,
-                if retval.is_ok() { 1 } else { TILE_ERROR as u32 },
-            );
+    if let Some(frame_hdr) = &f.frame_hdr {
+        if frame_hdr.refresh_context != 0 {
+            if !f.out_cdf.progress.is_null() {
+                ::core::intrinsics::atomic_store_seqcst(
+                    f.out_cdf.progress,
+                    if retval.is_ok() { 1 } else { TILE_ERROR as u32 },
+                );
+            }
+            rav1d_cdf_thread_unref(&mut f.out_cdf);
         }
-        rav1d_cdf_thread_unref(&mut f.out_cdf);
     }
+
     rav1d_ref_dec(&mut f.cur_segmap_ref);
     rav1d_ref_dec(&mut f.prev_segmap_ref);
     rav1d_ref_dec(&mut f.mvs_ref);
-    rav1d_ref_dec(&mut f.seq_hdr_ref);
-    rav1d_ref_dec(&mut f.frame_hdr_ref);
+    let _ = mem::take(&mut f.seq_hdr);
+    let _ = mem::take(&mut f.frame_hdr);
     for tile in &mut f.tiles {
         rav1d_data_unref_internal(&mut tile.data);
     }
@@ -4918,11 +4927,12 @@ pub(crate) unsafe fn rav1d_decode_frame(f: &mut Rav1dFrameContext) -> Rav1dResul
             res = f.task_thread.retval;
         } else {
             res = rav1d_decode_frame_main(f);
-            if res.is_ok() && (*f.frame_hdr).refresh_context != 0 && f.task_thread.update_set {
+            let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+            if res.is_ok() && frame_hdr.refresh_context != 0 && f.task_thread.update_set {
                 rav1d_cdf_thread_update(
-                    f.frame_hdr,
+                    frame_hdr,
                     f.out_cdf.data.cdf,
-                    &mut (*f.ts.offset((*f.frame_hdr).tiling.update as isize)).cdf,
+                    &mut (*f.ts.offset(frame_hdr.tiling.update as isize)).cdf,
                 );
             }
         }
@@ -4992,16 +5002,12 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
         (&mut *c.fc, ptr::null_mut())
     };
 
-    f.seq_hdr = c.seq_hdr;
-    f.seq_hdr_ref = c.seq_hdr_ref;
-    rav1d_ref_inc(f.seq_hdr_ref);
-    f.frame_hdr = c.frame_hdr;
-    f.frame_hdr_ref = c.frame_hdr_ref;
-    c.frame_hdr = ptr::null_mut();
-    c.frame_hdr_ref = ptr::null_mut();
-    f.dsp = &mut c.dsp[(*f.seq_hdr).hbd as usize];
+    f.seq_hdr = c.seq_hdr.clone();
+    f.frame_hdr = mem::take(&mut c.frame_hdr);
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
+    f.dsp = &mut c.dsp[seq_hdr.hbd as usize];
 
-    let bpc = 8 + 2 * (*f.seq_hdr).hbd;
+    let bpc = 8 + 2 * seq_hdr.hbd;
 
     unsafe fn on_error(
         f: &mut Rav1dFrameContext,
@@ -5010,11 +5016,11 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     ) {
         f.task_thread.error = 1;
         rav1d_cdf_thread_unref(&mut f.in_cdf);
-        if (*f.frame_hdr).refresh_context != 0 {
+        if f.frame_hdr.as_ref().unwrap().refresh_context != 0 {
             rav1d_cdf_thread_unref(&mut f.out_cdf);
         }
         for i in 0..7 {
-            if !f.refp[i].p.frame_hdr.is_null() {
+            if f.refp[i].p.frame_hdr.is_some() {
                 rav1d_thread_picture_unref(&mut f.refp[i]);
             }
             rav1d_ref_dec(&mut f.ref_mvs_ref[i]);
@@ -5027,8 +5033,8 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
         rav1d_picture_unref_internal(&mut f.cur);
         rav1d_thread_picture_unref(&mut f.sr_cur);
         rav1d_ref_dec(&mut f.mvs_ref);
-        rav1d_ref_dec(&mut f.seq_hdr_ref);
-        rav1d_ref_dec(&mut f.frame_hdr_ref);
+        let _ = mem::take(&mut f.seq_hdr);
+        let _ = mem::take(&mut f.frame_hdr);
         c.cached_error_props = c.in_0.m.clone();
 
         for mut tile in f.tiles.drain(..) {
@@ -5044,7 +5050,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     // we should probably initialize all the fn ptrs
     // when `c` is allocated during [`rav1d_open`].
     if !(*f.dsp).initialized {
-        let dsp = &mut c.dsp[(*f.seq_hdr).hbd as usize];
+        let dsp = &mut c.dsp[seq_hdr.hbd as usize];
         dsp.initialized = true;
 
         match bpc {
@@ -5072,14 +5078,14 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
                 writeln!(
                     c.logger,
                     "Compiled without support for {}-bit decoding",
-                    8 + 2 * (*f.seq_hdr).hbd
+                    8 + 2 * seq_hdr.hbd
                 );
                 on_error(f, c, out_delayed);
                 return Err(ENOPROTOOPT);
             }
         }
     }
-    if (*f.seq_hdr).hbd == 0 {
+    if seq_hdr.hbd == 0 {
         #[cfg(feature = "bitdepth_8")]
         {
             f.bd_fn = Rav1dFrameContext_bd_fn::new::<BitDepth8>();
@@ -5096,22 +5102,23 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     }
 
     let mut ref_coded_width = <[i32; 7]>::default();
-    if (*f.frame_hdr).frame_type.is_inter_or_switch() {
-        if (*f.frame_hdr).primary_ref_frame != RAV1D_PRIMARY_REF_NONE {
-            let pri_ref = (*f.frame_hdr).refidx[(*f.frame_hdr).primary_ref_frame as usize] as usize;
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+    if frame_hdr.frame_type.is_inter_or_switch() {
+        if frame_hdr.primary_ref_frame != RAV1D_PRIMARY_REF_NONE {
+            let pri_ref = frame_hdr.refidx[frame_hdr.primary_ref_frame as usize] as usize;
             if c.refs[pri_ref].p.p.data[0].is_null() {
                 on_error(f, c, out_delayed);
                 return Err(EINVAL);
             }
         }
         for i in 0..7 {
-            let refidx = (*f.frame_hdr).refidx[i] as usize;
+            let refidx = frame_hdr.refidx[i] as usize;
             if c.refs[refidx].p.p.data[0].is_null()
-                || ((*f.frame_hdr).size.width[0] * 2) < c.refs[refidx].p.p.p.w
-                || ((*f.frame_hdr).size.height * 2) < c.refs[refidx].p.p.p.h
-                || (*f.frame_hdr).size.width[0] > c.refs[refidx].p.p.p.w * 16
-                || (*f.frame_hdr).size.height > c.refs[refidx].p.p.p.h * 16
-                || (*f.seq_hdr).layout != c.refs[refidx].p.p.p.layout
+                || (frame_hdr.size.width[0] * 2) < c.refs[refidx].p.p.p.w
+                || (frame_hdr.size.height * 2) < c.refs[refidx].p.p.p.h
+                || frame_hdr.size.width[0] > c.refs[refidx].p.p.p.w * 16
+                || frame_hdr.size.height > c.refs[refidx].p.p.p.h * 16
+                || seq_hdr.layout != c.refs[refidx].p.p.p.layout
                 || bpc != c.refs[refidx].p.p.p.bpc
             {
                 for j in 0..i {
@@ -5121,33 +5128,33 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
                 return Err(EINVAL);
             }
             rav1d_thread_picture_ref(&mut f.refp[i], &mut c.refs[refidx].p);
-            ref_coded_width[i] = (*c.refs[refidx].p.p.frame_hdr).size.width[0];
-            if (*f.frame_hdr).size.width[0] != c.refs[refidx].p.p.p.w
-                || (*f.frame_hdr).size.height != c.refs[refidx].p.p.p.h
+            ref_coded_width[i] = c.refs[refidx].p.p.frame_hdr.as_ref().unwrap().size.width[0];
+            if frame_hdr.size.width[0] != c.refs[refidx].p.p.p.w
+                || frame_hdr.size.height != c.refs[refidx].p.p.p.h
             {
-                f.svc[i][0].scale = scale_fac(c.refs[refidx].p.p.p.w, (*f.frame_hdr).size.width[0]);
-                f.svc[i][1].scale = scale_fac(c.refs[refidx].p.p.p.h, (*f.frame_hdr).size.height);
+                f.svc[i][0].scale = scale_fac(c.refs[refidx].p.p.p.w, frame_hdr.size.width[0]);
+                f.svc[i][1].scale = scale_fac(c.refs[refidx].p.p.p.h, frame_hdr.size.height);
                 f.svc[i][0].step = f.svc[i][0].scale + 8 >> 4;
                 f.svc[i][1].step = f.svc[i][1].scale + 8 >> 4;
             } else {
                 f.svc[i][1].scale = 0;
                 f.svc[i][0].scale = f.svc[i][1].scale;
             }
-            f.gmv_warp_allowed[i] = ((*f.frame_hdr).gmv[i].r#type > RAV1D_WM_TYPE_TRANSLATION
-                && (*f.frame_hdr).force_integer_mv == 0
-                && !rav1d_get_shear_params(&mut (*f.frame_hdr).gmv[i])
+            f.gmv_warp_allowed[i] = (frame_hdr.gmv[i].r#type > RAV1D_WM_TYPE_TRANSLATION
+                && frame_hdr.force_integer_mv == 0
+                && !rav1d_get_shear_params(&frame_hdr.gmv[i])
                 && f.svc[i][0].scale == 0) as u8;
         }
     }
 
     // setup entropy
-    if (*f.frame_hdr).primary_ref_frame == RAV1D_PRIMARY_REF_NONE {
-        rav1d_cdf_thread_init_static(&mut f.in_cdf, (*f.frame_hdr).quant.yac);
+    if frame_hdr.primary_ref_frame == RAV1D_PRIMARY_REF_NONE {
+        rav1d_cdf_thread_init_static(&mut f.in_cdf, frame_hdr.quant.yac);
     } else {
-        let pri_ref = (*f.frame_hdr).refidx[(*f.frame_hdr).primary_ref_frame as usize] as usize;
+        let pri_ref = frame_hdr.refidx[frame_hdr.primary_ref_frame as usize] as usize;
         rav1d_cdf_thread_ref(&mut f.in_cdf, &mut c.cdf[pri_ref]);
     }
-    if (*f.frame_hdr).refresh_context != 0 {
+    if frame_hdr.refresh_context != 0 {
         let res = rav1d_cdf_thread_alloc(c, &mut f.out_cdf, (c.n_fc > 1) as c_int);
         if res.is_err() {
             on_error(f, c, out_delayed);
@@ -5166,9 +5173,11 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
         return res;
     }
 
-    if (*f.frame_hdr).size.width[0] != (*f.frame_hdr).size.width[1] {
-        let res =
-            rav1d_picture_alloc_copy(c, &mut f.cur, (*f.frame_hdr).size.width[0], &mut f.sr_cur.p);
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+
+    if frame_hdr.size.width[0] != frame_hdr.size.width[1] {
+        let res = rav1d_picture_alloc_copy(c, &mut f.cur, frame_hdr.size.width[0], &mut f.sr_cur.p);
         if res.is_err() {
             on_error(f, c, out_delayed);
             return res;
@@ -5176,7 +5185,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     } else {
         rav1d_picture_ref(&mut f.cur, &mut f.sr_cur.p);
     }
-    if (*f.frame_hdr).size.width[0] != (*f.frame_hdr).size.width[1] {
+    if frame_hdr.size.width[0] != frame_hdr.size.width[1] {
         f.resize_step[0] = scale_fac(f.cur.p.w, f.sr_cur.p.p.w);
         let ss_hor = (f.cur.p.layout != Rav1dPixelLayout::I444) as c_int;
         let in_cw = f.cur.p.w + ss_hor >> ss_hor;
@@ -5188,7 +5197,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
 
     // move f->cur into output queue
     if c.n_fc == 1 {
-        if (*f.frame_hdr).show_frame != 0 || c.output_invisible_frames {
+        if frame_hdr.show_frame != 0 || c.output_invisible_frames {
             rav1d_thread_picture_ref(&mut c.out, &mut f.sr_cur);
             c.event_flags |= f.sr_cur.flags.into();
         }
@@ -5196,28 +5205,28 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
         rav1d_thread_picture_ref(out_delayed, &mut f.sr_cur);
     }
 
-    f.w4 = (*f.frame_hdr).size.width[0] + 3 >> 2;
-    f.h4 = (*f.frame_hdr).size.height + 3 >> 2;
-    f.bw = ((*f.frame_hdr).size.width[0] + 7 >> 3) << 1;
-    f.bh = ((*f.frame_hdr).size.height + 7 >> 3) << 1;
+    f.w4 = frame_hdr.size.width[0] + 3 >> 2;
+    f.h4 = frame_hdr.size.height + 3 >> 2;
+    f.bw = (frame_hdr.size.width[0] + 7 >> 3) << 1;
+    f.bh = (frame_hdr.size.height + 7 >> 3) << 1;
     f.sb128w = f.bw + 31 >> 5;
     f.sb128h = f.bh + 31 >> 5;
-    f.sb_shift = 4 + (*f.seq_hdr).sb128;
-    f.sb_step = 16 << (*f.seq_hdr).sb128;
+    f.sb_shift = 4 + seq_hdr.sb128;
+    f.sb_step = 16 << seq_hdr.sb128;
     f.sbh = f.bh + f.sb_step - 1 >> f.sb_shift;
     f.b4_stride = (f.bw + 31 & !31) as ptrdiff_t;
     f.bitdepth_max = (1 << f.cur.p.bpc) - 1;
     *&mut f.task_thread.error = 0;
     let uses_2pass = (c.n_fc > 1) as c_int;
-    let cols = (*f.frame_hdr).tiling.cols;
-    let rows = (*f.frame_hdr).tiling.rows;
+    let cols = frame_hdr.tiling.cols;
+    let rows = frame_hdr.tiling.rows;
     ::core::intrinsics::atomic_store_seqcst(
         &mut f.task_thread.task_counter,
         cols * rows + f.sbh << uses_2pass,
     );
 
     // ref_mvs
-    if (*f.frame_hdr).frame_type.is_inter_or_switch() || (*f.frame_hdr).allow_intrabc != 0 {
+    if frame_hdr.frame_type.is_inter_or_switch() || frame_hdr.allow_intrabc != 0 {
         f.mvs_ref = rav1d_ref_create_using_pool(
             c.refmvs_pool,
             ::core::mem::size_of::<refmvs_temporal_block>()
@@ -5230,16 +5239,16 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
             return Err(ENOMEM);
         }
         f.mvs = (*f.mvs_ref).data.cast::<refmvs_temporal_block>();
-        if (*f.frame_hdr).allow_intrabc == 0 {
+        if frame_hdr.allow_intrabc == 0 {
             for i in 0..7 {
-                f.refpoc[i] = (*f.refp[i].p.frame_hdr).frame_offset as c_uint;
+                f.refpoc[i] = f.refp[i].p.frame_hdr.as_ref().unwrap().frame_offset as c_uint;
             }
         } else {
             f.refpoc.fill(0);
         }
-        if (*f.frame_hdr).use_ref_frame_mvs != 0 {
+        if frame_hdr.use_ref_frame_mvs != 0 {
             for i in 0..7 {
-                let refidx = (*f.frame_hdr).refidx[i] as usize;
+                let refidx = frame_hdr.refidx[i] as usize;
                 let ref_w = (ref_coded_width[i] + 7 >> 3) << 1;
                 let ref_h = (f.refp[i].p.p.h + 7 >> 3) << 1;
                 if !c.refs[refidx].refmvs.is_null() && ref_w == f.bw && ref_h == f.bh {
@@ -5263,21 +5272,20 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     }
 
     // segmap
-    if (*f.frame_hdr).segmentation.enabled != 0 {
+    if frame_hdr.segmentation.enabled != 0 {
         // By default, the previous segmentation map is not initialised.
         f.prev_segmap_ref = ptr::null_mut();
         f.prev_segmap = ptr::null();
 
         // We might need a previous frame's segmentation map.
         // This happens if there is either no update or a temporal update.
-        if (*f.frame_hdr).segmentation.temporal != 0 || (*f.frame_hdr).segmentation.update_map == 0
-        {
-            let pri_ref = (*f.frame_hdr).primary_ref_frame as usize;
+        if frame_hdr.segmentation.temporal != 0 || frame_hdr.segmentation.update_map == 0 {
+            let pri_ref = frame_hdr.primary_ref_frame as usize;
             assert!(pri_ref != RAV1D_PRIMARY_REF_NONE as usize);
             let ref_w = (ref_coded_width[pri_ref] + 7 >> 3) << 1;
             let ref_h = (f.refp[pri_ref].p.p.h + 7 >> 3) << 1;
             if ref_w == f.bw && ref_h == f.bh {
-                f.prev_segmap_ref = c.refs[(*f.frame_hdr).refidx[pri_ref] as usize].segmap;
+                f.prev_segmap_ref = c.refs[frame_hdr.refidx[pri_ref] as usize].segmap;
                 if !f.prev_segmap_ref.is_null() {
                     rav1d_ref_inc(f.prev_segmap_ref);
                     f.prev_segmap = (*f.prev_segmap_ref).data.cast::<u8>();
@@ -5285,7 +5293,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
             }
         }
 
-        if (*f.frame_hdr).segmentation.update_map != 0 {
+        if frame_hdr.segmentation.update_map != 0 {
             // We're updating an existing map,
             // but need somewhere to put the new values.
             // Allocate them here (the data actually gets set elsewhere).
@@ -5324,16 +5332,16 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
     }
 
     // update references etc.
-    let refresh_frame_flags = (*f.frame_hdr).refresh_frame_flags as c_uint;
+    let refresh_frame_flags = frame_hdr.refresh_frame_flags as c_uint;
     for i in 0..8 {
         if refresh_frame_flags & (1 << i) != 0 {
-            if !c.refs[i].p.p.frame_hdr.is_null() {
+            if c.refs[i].p.p.frame_hdr.is_some() {
                 rav1d_thread_picture_unref(&mut c.refs[i].p);
             }
             rav1d_thread_picture_ref(&mut c.refs[i].p, &mut f.sr_cur);
 
             rav1d_cdf_thread_unref(&mut c.cdf[i]);
-            if (*f.frame_hdr).refresh_context != 0 {
+            if frame_hdr.refresh_context != 0 {
                 rav1d_cdf_thread_ref(&mut c.cdf[i], &mut f.out_cdf);
             } else {
                 rav1d_cdf_thread_ref(&mut c.cdf[i], &mut f.in_cdf);
@@ -5345,7 +5353,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
                 rav1d_ref_inc(f.cur_segmap_ref);
             }
             rav1d_ref_dec(&mut c.refs[i].refmvs);
-            if (*f.frame_hdr).allow_intrabc == 0 {
+            if frame_hdr.allow_intrabc == 0 {
                 c.refs[i].refmvs = f.mvs_ref;
                 if !f.mvs_ref.is_null() {
                     rav1d_ref_inc(f.mvs_ref);
@@ -5361,7 +5369,7 @@ pub unsafe fn rav1d_submit_frame(c: &mut Rav1dContext) -> Rav1dResult {
             rav1d_thread_picture_unref(&mut c.out);
             for i in 0..8 {
                 if refresh_frame_flags & (1 << i) != 0 {
-                    if !c.refs[i].p.p.frame_hdr.is_null() {
+                    if c.refs[i].p.p.frame_hdr.is_some() {
                         rav1d_thread_picture_unref(&mut c.refs[i].p);
                     }
                     rav1d_cdf_thread_unref(&mut c.cdf[i]);
diff --git a/src/fg_apply.rs b/src/fg_apply.rs
index 41844d55f..39f87c700 100644
--- a/src/fg_apply.rs
+++ b/src/fg_apply.rs
@@ -78,7 +78,8 @@ pub(crate) unsafe fn rav1d_prep_grain<BD: BitDepth>(
     grain: &mut GrainBD<BD>,
 ) {
     let GrainBD { grain_lut, scaling } = grain;
-    let data = &(*out.frame_hdr).film_grain.data;
+    let frame_hdr = &***out.frame_hdr.as_ref().unwrap();
+    let data = &frame_hdr.film_grain.data;
     let bitdepth_max = (1 << out.p.bpc) - 1;
     let bd = BD::from_c(bitdepth_max);
 
@@ -179,12 +180,14 @@ pub(crate) unsafe fn rav1d_apply_grain_row<BD: BitDepth>(
 ) {
     // Synthesize grain for the affected planes
     let GrainBD { grain_lut, scaling } = grain;
-    let data = &(*out.frame_hdr).film_grain.data;
+    let seq_hdr = &***out.seq_hdr.as_ref().unwrap();
+    let frame_hdr = &***out.frame_hdr.as_ref().unwrap();
+    let data = &frame_hdr.film_grain.data;
     let data_c = &data.clone().into();
     let ss_y = (r#in.p.layout == Rav1dPixelLayout::I420) as usize;
     let ss_x = (r#in.p.layout != Rav1dPixelLayout::I444) as usize;
     let cpw = out.p.w as usize + ss_x >> ss_x;
-    let is_id = (*out.seq_hdr).mtrx == RAV1D_MC_IDENTITY;
+    let is_id = seq_hdr.mtrx == RAV1D_MC_IDENTITY;
     let luma_src = (r#in.data[0] as *mut BD::Pixel)
         .offset(((row * 32) as isize * BD::pxstride(r#in.stride[0] as usize) as isize) as isize);
     let bitdepth_max = (1 << out.p.bpc) - 1;
diff --git a/src/internal.rs b/src/internal.rs
index 2bdf65753..bb24f06ff 100644
--- a/src/internal.rs
+++ b/src/internal.rs
@@ -9,7 +9,9 @@ use crate::include::dav1d::dav1d::Rav1dDecodeFrameType;
 use crate::include::dav1d::dav1d::Rav1dEventFlags;
 use crate::include::dav1d::dav1d::Rav1dInloopFilterType;
 use crate::include::dav1d::headers::DRav1d;
+use crate::include::dav1d::headers::Dav1dFrameHeader;
 use crate::include::dav1d::headers::Dav1dITUTT35;
+use crate::include::dav1d::headers::Dav1dSequenceHeader;
 use crate::include::dav1d::headers::Rav1dContentLightLevel;
 use crate::include::dav1d::headers::Rav1dFrameHeader;
 use crate::include::dav1d::headers::Rav1dITUTT35;
@@ -207,12 +209,8 @@ pub struct Rav1dContext {
     /// to a frame worker to be decoded.
     pub(crate) tiles: Vec<Rav1dTileGroup>,
     pub(crate) n_tiles: c_int,
-    pub(crate) seq_hdr_pool: *mut Rav1dMemPool,
-    pub(crate) seq_hdr_ref: *mut Rav1dRef,
-    pub(crate) seq_hdr: *mut Rav1dSequenceHeader,
-    pub(crate) frame_hdr_pool: *mut Rav1dMemPool,
-    pub(crate) frame_hdr_ref: *mut Rav1dRef,
-    pub(crate) frame_hdr: *mut Rav1dFrameHeader,
+    pub(crate) seq_hdr: Option<Arc<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>>, // TODO(kkysen) Previously pooled.
+    pub(crate) frame_hdr: Option<Arc<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>>, // TODO(kkysen) Previously pooled.
     pub(crate) content_light: Option<Arc<Rav1dContentLightLevel>>,
     pub(crate) mastering_display: Option<Arc<Rav1dMasteringDisplay>>,
     pub(crate) itut_t35: Option<Arc<DRav1d<Rav1dITUTT35, Dav1dITUTT35>>>,
@@ -456,10 +454,8 @@ pub struct FrameTileThreadData {
 
 #[repr(C)]
 pub(crate) struct Rav1dFrameContext {
-    pub seq_hdr_ref: *mut Rav1dRef,
-    pub seq_hdr: *mut Rav1dSequenceHeader,
-    pub frame_hdr_ref: *mut Rav1dRef,
-    pub frame_hdr: *mut Rav1dFrameHeader,
+    pub seq_hdr: Option<Arc<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>>,
+    pub frame_hdr: Option<Arc<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>>,
     pub refp: [Rav1dThreadPicture; 7],
     // during block coding / reconstruction
     pub cur: Rav1dPicture,
diff --git a/src/lf_apply.rs b/src/lf_apply.rs
index 4b765be5f..8e53c7035 100644
--- a/src/lf_apply.rs
+++ b/src/lf_apply.rs
@@ -33,8 +33,9 @@ unsafe fn backup_lpf<BD: BitDepth>(
     lr_backup: c_int,
 ) {
     let cdef_backup = (lr_backup == 0) as c_int;
-    let dst_w = if (*(*f).frame_hdr).size.super_res.enabled != 0 {
-        (*(*f).frame_hdr).size.width[1] + ss_hor >> ss_hor
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    let dst_w = if frame_hdr.size.super_res.enabled != 0 {
+        frame_hdr.size.width[1] + ss_hor >> ss_hor
     } else {
         src_w
     };
@@ -104,7 +105,8 @@ unsafe fn backup_lpf<BD: BitDepth>(
         }
         dst = dst.offset(4 * BD::pxstride(dst_stride as usize) as isize);
     }
-    if lr_backup != 0 && (*(*f).frame_hdr).size.width[0] != (*(*f).frame_hdr).size.width[1] {
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    if lr_backup != 0 && frame_hdr.size.width[0] != frame_hdr.size.width[1] {
         while row + stripe_h <= row_h {
             let n_lines = 4 - (row + stripe_h + 1 == h) as c_int;
             ((*(*f).dsp).mc.resize)(
@@ -171,11 +173,13 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
     sby: c_int,
 ) {
     let have_tt = ((*(*f).c).n_tc > 1 as c_uint) as c_int;
-    let resize = ((*(*f).frame_hdr).size.width[0] != (*(*f).frame_hdr).size.width[1]) as c_int;
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    let resize = (frame_hdr.size.width[0] != frame_hdr.size.width[1]) as c_int;
     let offset = 8 * (sby != 0) as c_int;
     let src_stride: *const ptrdiff_t = ((*f).cur.stride).as_mut_ptr();
     let lr_stride: *const ptrdiff_t = ((*f).sr_cur.p.stride).as_mut_ptr();
-    let tt_off = have_tt * sby * ((4 as c_int) << (*(*f).seq_hdr).sb128);
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+    let tt_off = have_tt * sby * ((4 as c_int) << seq_hdr.sb128);
     let dst: [*mut BD::Pixel; 3] = [
         ((*f).lf.lr_lpf_line[0] as *mut BD::Pixel)
             .offset(tt_off as isize * BD::pxstride(*lr_stride.offset(0) as usize) as isize),
@@ -185,11 +189,11 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
             .offset(tt_off as isize * BD::pxstride(*lr_stride.offset(1) as usize) as isize),
     ];
     let restore_planes = (*f).lf.restore_planes;
-    if (*(*f).seq_hdr).cdef != 0 || restore_planes & LR_RESTORE_Y as c_int != 0 {
+    if seq_hdr.cdef != 0 || restore_planes & LR_RESTORE_Y as c_int != 0 {
         let h = (*f).cur.p.h;
         let w = (*f).bw << 2;
-        let row_h = cmp::min((sby + 1) << 6 + (*(*f).seq_hdr).sb128, h - 1);
-        let y_stripe = (sby << 6 + (*(*f).seq_hdr).sb128) - offset;
+        let row_h = cmp::min((sby + 1) << 6 + seq_hdr.sb128, h - 1);
+        let y_stripe = (sby << 6 + seq_hdr.sb128) - offset;
         if restore_planes & LR_RESTORE_Y as c_int != 0 || resize == 0 {
             backup_lpf::<BD>(
                 f,
@@ -200,7 +204,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
                 ),
                 *src_stride.offset(0),
                 0 as c_int,
-                (*(*f).seq_hdr).sb128,
+                seq_hdr.sb128,
                 y_stripe,
                 row_h,
                 w,
@@ -221,7 +225,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
                 ),
                 *src_stride.offset(0),
                 0 as c_int,
-                (*(*f).seq_hdr).sb128,
+                seq_hdr.sb128,
                 y_stripe,
                 row_h,
                 w,
@@ -231,8 +235,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
             );
         }
     }
-    if ((*(*f).seq_hdr).cdef != 0
-        || restore_planes & (LR_RESTORE_U as c_int | LR_RESTORE_V as c_int) != 0)
+    if (seq_hdr.cdef != 0 || restore_planes & (LR_RESTORE_U as c_int | LR_RESTORE_V as c_int) != 0)
         && (*f).cur.p.layout as c_uint != Rav1dPixelLayout::I400 as c_int as c_uint
     {
         let ss_ver = ((*f).sr_cur.p.p.layout as c_uint == Rav1dPixelLayout::I420 as c_int as c_uint)
@@ -241,12 +244,12 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
             as c_int;
         let h_0 = (*f).cur.p.h + ss_ver >> ss_ver;
         let w_0 = (*f).bw << 2 - ss_hor;
-        let row_h_0 = cmp::min((sby + 1) << 6 - ss_ver + (*(*f).seq_hdr).sb128, h_0 - 1);
+        let row_h_0 = cmp::min((sby + 1) << 6 - ss_ver + seq_hdr.sb128, h_0 - 1);
         let offset_uv = offset >> ss_ver;
-        let y_stripe_0 = (sby << 6 - ss_ver + (*(*f).seq_hdr).sb128) - offset_uv;
+        let y_stripe_0 = (sby << 6 - ss_ver + seq_hdr.sb128) - offset_uv;
         let cdef_off_uv: ptrdiff_t =
             sby as isize * 4 * BD::pxstride(*src_stride.offset(1) as usize) as isize;
-        if (*(*f).seq_hdr).cdef != 0 || restore_planes & LR_RESTORE_U as c_int != 0 {
+        if seq_hdr.cdef != 0 || restore_planes & LR_RESTORE_U as c_int != 0 {
             if restore_planes & LR_RESTORE_U as c_int != 0 || resize == 0 {
                 backup_lpf::<BD>(
                     f,
@@ -257,7 +260,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
                     ),
                     *src_stride.offset(1),
                     ss_ver,
-                    (*(*f).seq_hdr).sb128,
+                    seq_hdr.sb128,
                     y_stripe_0,
                     row_h_0,
                     w_0,
@@ -276,7 +279,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
                     ),
                     *src_stride.offset(1),
                     ss_ver,
-                    (*(*f).seq_hdr).sb128,
+                    seq_hdr.sb128,
                     y_stripe_0,
                     row_h_0,
                     w_0,
@@ -286,7 +289,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
                 );
             }
         }
-        if (*(*f).seq_hdr).cdef != 0 || restore_planes & LR_RESTORE_V as c_int != 0 {
+        if seq_hdr.cdef != 0 || restore_planes & LR_RESTORE_V as c_int != 0 {
             if restore_planes & LR_RESTORE_V as c_int != 0 || resize == 0 {
                 backup_lpf::<BD>(
                     f,
@@ -297,7 +300,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
                     ),
                     *src_stride.offset(1),
                     ss_ver,
-                    (*(*f).seq_hdr).sb128,
+                    seq_hdr.sb128,
                     y_stripe_0,
                     row_h_0,
                     w_0,
@@ -316,7 +319,7 @@ pub(crate) unsafe fn rav1d_copy_lpf<BD: BitDepth>(
                     ),
                     *src_stride.offset(1),
                     ss_ver,
-                    (*(*f).seq_hdr).sb128,
+                    seq_hdr.sb128,
                     y_stripe_0,
                     row_h_0,
                     w_0,
@@ -540,7 +543,8 @@ pub(crate) unsafe fn rav1d_loopfilter_sbrow_cols<BD: BitDepth>(
 ) {
     let mut x;
     let mut have_left;
-    let is_sb64 = ((*(*f).seq_hdr).sb128 == 0) as c_int;
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+    let is_sb64 = (seq_hdr.sb128 == 0) as c_int;
     let starty4 = (sby & is_sb64) << 4;
     let sbsz = 32 >> is_sb64;
     let sbl2 = 5 - is_sb64;
@@ -559,9 +563,10 @@ pub(crate) unsafe fn rav1d_loopfilter_sbrow_cols<BD: BitDepth>(
         .offset((sby << sbl2) as isize) as *mut u8;
     let mut lpf_uv: *const u8 = &mut *(*((*f).lf.tx_lpf_right_edge).as_ptr().offset(1))
         .offset((sby << sbl2 - ss_ver) as isize) as *mut u8;
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
     let mut tile_col = 1;
     loop {
-        x = (*(*f).frame_hdr).tiling.col_start_sb[tile_col as usize] as c_int;
+        x = frame_hdr.tiling.col_start_sb[tile_col as usize] as c_int;
         if x << sbl2 >= (*f).bw {
             break;
         }
@@ -706,7 +711,7 @@ pub(crate) unsafe fn rav1d_loopfilter_sbrow_cols<BD: BitDepth>(
         ptr = ptr.offset(128);
         level_ptr = level_ptr.offset(32);
     }
-    if (*(*f).frame_hdr).loopfilter.level_u == 0 && (*(*f).frame_hdr).loopfilter.level_v == 0 {
+    if frame_hdr.loopfilter.level_u == 0 && frame_hdr.loopfilter.level_v == 0 {
         return;
     }
     let mut uv_off: ptrdiff_t;
@@ -744,7 +749,8 @@ pub(crate) unsafe fn rav1d_loopfilter_sbrow_rows<BD: BitDepth>(
 ) {
     let mut x;
     let have_top = (sby > 0) as c_int;
-    let is_sb64 = ((*(*f).seq_hdr).sb128 == 0) as c_int;
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+    let is_sb64 = (seq_hdr.sb128 == 0) as c_int;
     let starty4 = (sby & is_sb64) << 4;
     let sbsz = 32 >> is_sb64;
     let ss_ver =
@@ -775,7 +781,8 @@ pub(crate) unsafe fn rav1d_loopfilter_sbrow_rows<BD: BitDepth>(
         ptr = ptr.offset(128);
         level_ptr = level_ptr.offset(32);
     }
-    if (*(*f).frame_hdr).loopfilter.level_u == 0 && (*(*f).frame_hdr).loopfilter.level_v == 0 {
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    if frame_hdr.loopfilter.level_u == 0 && frame_hdr.loopfilter.level_v == 0 {
         return;
     }
     let mut uv_off: ptrdiff_t;
diff --git a/src/lib.rs b/src/lib.rs
index 292d76559..419c321ea 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -15,10 +15,8 @@ use crate::include::dav1d::dav1d::RAV1D_DECODEFRAMETYPE_ALL;
 use crate::include::dav1d::dav1d::RAV1D_DECODEFRAMETYPE_KEY;
 use crate::include::dav1d::dav1d::RAV1D_INLOOPFILTER_ALL;
 use crate::include::dav1d::headers::DRav1d;
-use crate::include::dav1d::headers::Dav1dFrameHeader;
 use crate::include::dav1d::headers::Dav1dSequenceHeader;
 use crate::include::dav1d::headers::Rav1dFilmGrainData;
-use crate::include::dav1d::headers::Rav1dFrameHeader;
 use crate::include::dav1d::headers::Rav1dSequenceHeader;
 use crate::include::dav1d::picture::Dav1dPicture;
 use crate::include::dav1d::picture::Rav1dPicture;
@@ -107,6 +105,7 @@ use std::mem;
 use std::mem::MaybeUninit;
 use std::process::abort;
 use std::ptr::NonNull;
+use std::sync::Arc;
 use std::sync::Once;
 use to_method::To as _;
 
@@ -283,9 +282,7 @@ pub(crate) unsafe fn rav1d_open(c_out: &mut *mut Rav1dContext, s: &Rav1dSettings
     (*c).inloop_filters = s.inloop_filters;
     (*c).decode_frame_type = s.decode_frame_type;
     (*c).cached_error_props = Default::default();
-    if rav1d_mem_pool_init(&mut (*c).seq_hdr_pool).is_err()
-        || rav1d_mem_pool_init(&mut (*c).frame_hdr_pool).is_err()
-        || rav1d_mem_pool_init(&mut (*c).segmap_pool).is_err()
+    if rav1d_mem_pool_init(&mut (*c).segmap_pool).is_err()
         || rav1d_mem_pool_init(&mut (*c).refmvs_pool).is_err()
         || rav1d_mem_pool_init(&mut (*c).cdf_pool).is_err()
     {
@@ -488,7 +485,7 @@ unsafe extern "C" fn dummy_free(data: *const u8, user_data: *mut c_void) {
 pub(crate) unsafe fn rav1d_parse_sequence_header(
     ptr: *const u8,
     sz: usize,
-) -> Rav1dResult<Rav1dSequenceHeader> {
+) -> Rav1dResult<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>> {
     let mut buf = Rav1dData::default();
     let s = Rav1dSettings {
         n_threads: 1,
@@ -497,7 +494,7 @@ pub(crate) unsafe fn rav1d_parse_sequence_header(
     };
     let mut c: *mut Rav1dContext = 0 as *mut Rav1dContext;
     rav1d_open(&mut c, &s)?;
-    || -> Rav1dResult<Rav1dSequenceHeader> {
+    || -> Rav1dResult<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>> {
         if !ptr.is_null() {
             rav1d_data_wrap_internal(&mut buf, ptr, sz, Some(dummy_free), 0 as *mut c_void)?;
         }
@@ -509,11 +506,15 @@ pub(crate) unsafe fn rav1d_parse_sequence_header(
             buf.data = buf.data.add(len);
         }
 
-        if (*c).seq_hdr.is_null() {
+        if (*c).seq_hdr.is_none() {
             return Err(ENOENT);
         }
 
-        Ok((*(*c).seq_hdr).clone())
+        (*c).seq_hdr
+            .take()
+            .and_then(Arc::into_inner)
+            .map(Ok)
+            .unwrap()
     }()
     .inspect_err(|_| {
         rav1d_data_unref_internal(&mut buf);
@@ -530,7 +531,7 @@ pub unsafe extern "C" fn dav1d_parse_sequence_header(
     (|| {
         validate_input!((!out.is_null(), EINVAL))?;
         let seq_hdr = rav1d_parse_sequence_header(ptr, sz)?;
-        out.write(seq_hdr.into());
+        out.write(seq_hdr.dav1d);
         Ok(())
     })()
     .into()
@@ -547,7 +548,7 @@ impl Rav1dFilmGrainData {
 
 impl Rav1dPicture {
     unsafe fn has_grain(&self) -> bool {
-        (*self.frame_hdr).film_grain.data.has_grain()
+        self.frame_hdr.as_ref().unwrap().film_grain.data.has_grain()
     }
 }
 
@@ -578,7 +579,7 @@ unsafe fn output_picture_ready(c: &mut Rav1dContext, drain: bool) -> bool {
     }
     if !c.all_layers && c.max_spatial_id {
         if !c.out.p.data[0].is_null() && !c.cache.p.data[0].is_null() {
-            if c.max_spatial_id == ((*c.cache.p.frame_hdr).spatial_id != 0)
+            if c.max_spatial_id == (c.cache.p.frame_hdr.as_ref().unwrap().spatial_id != 0)
                 || c.out.flags.contains(PictureFlags::NEW_TEMPORAL_UNIT)
             {
                 return true;
@@ -829,20 +830,8 @@ pub unsafe extern "C" fn dav1d_apply_grain(
         validate_input!((!in_0.is_null(), EINVAL))?;
         let c = &mut *c;
         let in_0 = in_0.read();
-        if let Some(mut seq_hdr_ref) = NonNull::new(in_0.seq_hdr_ref) {
-            (*seq_hdr_ref
-                .as_mut()
-                .data
-                .cast::<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>())
-            .update_rav1d();
-        }
-        if let Some(mut frame_hdr_ref) = NonNull::new(in_0.frame_hdr_ref) {
-            (*frame_hdr_ref
-                .as_mut()
-                .data
-                .cast::<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>())
-            .update_rav1d();
-        }
+        // Don't `.update_rav1d()` [`Rav1dSequenceHeader`] because it's meant to be read-only.
+        // Don't `.update_rav1d()` [`Rav1dFrameHeader`] because it's meant to be read-only.
         // Don't `.update_rav1d()` [`Rav1dITUTT35`] because we never read it.
         let mut out_rust = MaybeUninit::zeroed().assume_init(); // TODO(kkysen) Temporary until we return it directly.
         let in_rust = in_0.into();
@@ -855,17 +844,17 @@ pub unsafe extern "C" fn dav1d_apply_grain(
 
 pub(crate) unsafe fn rav1d_flush(c: *mut Rav1dContext) {
     rav1d_data_unref_internal(&mut (*c).in_0);
-    if !((*c).out.p.frame_hdr).is_null() {
+    if (*c).out.p.frame_hdr.is_some() {
         rav1d_thread_picture_unref(&mut (*c).out);
     }
-    if !((*c).cache.p.frame_hdr).is_null() {
+    if (*c).cache.p.frame_hdr.is_some() {
         rav1d_thread_picture_unref(&mut (*c).cache);
     }
     (*c).drain = 0 as c_int;
     (*c).cached_error = Ok(());
     let mut i = 0;
     while i < 8 {
-        if !((*c).refs[i as usize].p.p.frame_hdr).is_null() {
+        if (*c).refs[i as usize].p.p.frame_hdr.is_some() {
             rav1d_thread_picture_unref(&mut (*((*c).refs).as_mut_ptr().offset(i as isize)).p);
         }
         rav1d_ref_dec(&mut (*((*c).refs).as_mut_ptr().offset(i as isize)).segmap);
@@ -873,9 +862,8 @@ pub(crate) unsafe fn rav1d_flush(c: *mut Rav1dContext) {
         rav1d_cdf_thread_unref(&mut *((*c).cdf).as_mut_ptr().offset(i as isize));
         i += 1;
     }
-    (*c).frame_hdr = 0 as *mut Rav1dFrameHeader;
-    (*c).seq_hdr = 0 as *mut Rav1dSequenceHeader;
-    rav1d_ref_dec(&mut (*c).seq_hdr_ref);
+    let _ = mem::take(&mut (*c).frame_hdr); // TODO(kkysen) Why wasn't [`rav1d_ref_dec`] called on it?
+    let _ = mem::take(&mut (*c).seq_hdr);
     let _ = mem::take(&mut (*c).content_light);
     let _ = mem::take(&mut (*c).mastering_display);
     let _ = mem::take(&mut (*c).itut_t35);
@@ -939,7 +927,7 @@ pub(crate) unsafe fn rav1d_flush(c: *mut Rav1dContext) {
             let out_delayed: *mut Rav1dThreadPicture = &mut *((*c).frame_thread.out_delayed)
                 .offset(next as isize)
                 as *mut Rav1dThreadPicture;
-            if !((*out_delayed).p.frame_hdr).is_null() {
+            if (*out_delayed).p.frame_hdr.is_some() {
                 rav1d_thread_picture_unref(out_delayed);
             }
             n = n.wrapping_add(1);
@@ -1054,10 +1042,10 @@ unsafe fn close_internal(c_out: &mut *mut Rav1dContext, flush: c_int) {
     if (*c).n_fc > 1 as c_uint && !((*c).frame_thread.out_delayed).is_null() {
         let mut n_2: c_uint = 0 as c_int as c_uint;
         while n_2 < (*c).n_fc {
-            if !((*((*c).frame_thread.out_delayed).offset(n_2 as isize))
+            if (*((*c).frame_thread.out_delayed).offset(n_2 as isize))
                 .p
-                .frame_hdr)
-                .is_null()
+                .frame_hdr
+                .is_some()
             {
                 rav1d_thread_picture_unref(
                     &mut *((*c).frame_thread.out_delayed).offset(n_2 as isize),
@@ -1074,20 +1062,18 @@ unsafe fn close_internal(c_out: &mut *mut Rav1dContext, flush: c_int) {
     let mut n_4 = 0;
     while n_4 < 8 {
         rav1d_cdf_thread_unref(&mut *((*c).cdf).as_mut_ptr().offset(n_4 as isize));
-        if !((*c).refs[n_4 as usize].p.p.frame_hdr).is_null() {
+        if (*c).refs[n_4 as usize].p.p.frame_hdr.is_some() {
             rav1d_thread_picture_unref(&mut (*((*c).refs).as_mut_ptr().offset(n_4 as isize)).p);
         }
         rav1d_ref_dec(&mut (*((*c).refs).as_mut_ptr().offset(n_4 as isize)).refmvs);
         rav1d_ref_dec(&mut (*((*c).refs).as_mut_ptr().offset(n_4 as isize)).segmap);
         n_4 += 1;
     }
-    rav1d_ref_dec(&mut (*c).seq_hdr_ref);
-    rav1d_ref_dec(&mut (*c).frame_hdr_ref);
+    let _ = mem::take(&mut (*c).seq_hdr);
+    let _ = mem::take(&mut (*c).frame_hdr);
     let _ = mem::take(&mut (*c).mastering_display);
     let _ = mem::take(&mut (*c).content_light);
     let _ = mem::take(&mut (*c).itut_t35);
-    rav1d_mem_pool_end((*c).seq_hdr_pool);
-    rav1d_mem_pool_end((*c).frame_hdr_pool);
     rav1d_mem_pool_end((*c).segmap_pool);
     rav1d_mem_pool_end((*c).refmvs_pool);
     rav1d_mem_pool_end((*c).cdf_pool);
diff --git a/src/lr_apply.rs b/src/lr_apply.rs
index c17cf1622..8a86e604e 100644
--- a/src/lr_apply.rs
+++ b/src/lr_apply.rs
@@ -38,6 +38,7 @@ unsafe fn lr_stripe<BD: BitDepth>(
     lr: Av1RestorationUnit,
     mut edges: LrEdgeFlags,
 ) {
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
     let dsp: *const Rav1dDSPContext = (*f).dsp;
     let chroma = (plane != 0) as c_int;
     let ss_ver = chroma
@@ -48,11 +49,11 @@ unsafe fn lr_stripe<BD: BitDepth>(
             (8 as c_int) << ss_ver
         } else {
             0 as c_int
-        }) >> 6 - ss_ver + (*(*f).seq_hdr).sb128;
+        }) >> 6 - ss_ver + seq_hdr.sb128;
     let have_tt = ((*(*f).c).n_tc > 1 as c_uint) as c_int;
     let mut lpf: *const BD::Pixel = ((*f).lf.lr_lpf_line[plane as usize] as *mut BD::Pixel)
         .offset(
-            (have_tt * (sby * ((4 as c_int) << (*(*f).seq_hdr).sb128) - 4)) as isize
+            (have_tt * (sby * ((4 as c_int) << seq_hdr.sb128) - 4)) as isize
                 * BD::pxstride(stride as usize) as isize,
         )
         .offset(x as isize);
@@ -179,7 +180,8 @@ unsafe fn lr_sbrow<BD: BitDepth>(
     let ss_hor = chroma
         & ((*f).sr_cur.p.p.layout as c_uint != Rav1dPixelLayout::I444 as c_int as c_uint) as c_int;
     let p_stride: ptrdiff_t = (*f).sr_cur.p.stride[chroma as usize];
-    let unit_size_log2 = (*(*f).frame_hdr).restoration.unit_size[(plane != 0) as c_int as usize];
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    let unit_size_log2 = frame_hdr.restoration.unit_size[(plane != 0) as c_int as usize];
     let unit_size = (1 as c_int) << unit_size_log2;
     let half_unit_size = unit_size >> 1;
     let max_unit_size = unit_size + half_unit_size;
@@ -270,12 +272,13 @@ pub(crate) unsafe fn rav1d_lr_sbrow<BD: BitDepth>(
     let dst_stride: *const ptrdiff_t = ((*f).sr_cur.p.stride).as_mut_ptr();
     let restore_planes = (*f).lf.restore_planes;
     let not_last = ((sby + 1) < (*f).sbh) as c_int;
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
     if restore_planes & LR_RESTORE_Y as c_int != 0 {
         let h = (*f).sr_cur.p.p.h;
         let w = (*f).sr_cur.p.p.w;
-        let next_row_y = (sby + 1) << 6 + (*(*f).seq_hdr).sb128;
+        let next_row_y = (sby + 1) << 6 + seq_hdr.sb128;
         let row_h = cmp::min(next_row_y - 8 * not_last, h);
-        let y_stripe = (sby << 6 + (*(*f).seq_hdr).sb128) - offset_y;
+        let y_stripe = (sby << 6 + seq_hdr.sb128) - offset_y;
         lr_sbrow::<BD>(
             f,
             (*dst.offset(0)).offset(
@@ -295,10 +298,10 @@ pub(crate) unsafe fn rav1d_lr_sbrow<BD: BitDepth>(
             as c_int;
         let h_0 = (*f).sr_cur.p.p.h + ss_ver >> ss_ver;
         let w_0 = (*f).sr_cur.p.p.w + ss_hor >> ss_hor;
-        let next_row_y_0 = (sby + 1) << 6 - ss_ver + (*(*f).seq_hdr).sb128;
+        let next_row_y_0 = (sby + 1) << 6 - ss_ver + seq_hdr.sb128;
         let row_h_0 = cmp::min(next_row_y_0 - (8 >> ss_ver) * not_last, h_0);
         let offset_uv = offset_y >> ss_ver;
-        let y_stripe_0 = (sby << 6 - ss_ver + (*(*f).seq_hdr).sb128) - offset_uv;
+        let y_stripe_0 = (sby << 6 - ss_ver + seq_hdr.sb128) - offset_uv;
         if restore_planes & LR_RESTORE_U as c_int != 0 {
             lr_sbrow::<BD>(
                 f,
diff --git a/src/obu.rs b/src/obu.rs
index ddf4a8bc4..7a85f20af 100644
--- a/src/obu.rs
+++ b/src/obu.rs
@@ -4,9 +4,7 @@ use crate::include::dav1d::data::Rav1dData;
 use crate::include::dav1d::dav1d::RAV1D_DECODEFRAMETYPE_INTRA;
 use crate::include::dav1d::dav1d::RAV1D_DECODEFRAMETYPE_REFERENCE;
 use crate::include::dav1d::headers::DRav1d;
-use crate::include::dav1d::headers::Dav1dFrameHeader;
 use crate::include::dav1d::headers::Dav1dITUTT35;
-use crate::include::dav1d::headers::Dav1dSequenceHeader;
 use crate::include::dav1d::headers::Rav1dAdaptiveBoolean;
 use crate::include::dav1d::headers::Rav1dChromaSamplePosition;
 use crate::include::dav1d::headers::Rav1dColorPrimaries;
@@ -102,10 +100,8 @@ use crate::src::picture::rav1d_thread_picture_ref;
 use crate::src::picture::rav1d_thread_picture_unref;
 use crate::src::picture::PictureFlags;
 use crate::src::r#ref::rav1d_ref_create;
-use crate::src::r#ref::rav1d_ref_create_using_pool;
 use crate::src::r#ref::rav1d_ref_dec;
 use crate::src::r#ref::rav1d_ref_inc;
-use crate::src::r#ref::rav1d_ref_is_writable;
 use crate::src::thread_task::FRAME_ERROR;
 use libc::pthread_cond_wait;
 use libc::pthread_mutex_lock;
@@ -594,10 +590,7 @@ unsafe fn parse_frame_size(
         for i in 0..7 {
             if gb.get_bit() {
                 let r#ref = &c.refs[refidx[i as usize] as usize].p;
-                if (*r#ref).p.frame_hdr.is_null() {
-                    return Err(EINVAL);
-                }
-                let ref_size = &(*(*r#ref).p.frame_hdr).size;
+                let ref_size = &r#ref.p.frame_hdr.as_ref().ok_or(EINVAL)?.size;
                 let width1 = ref_size.width[1];
                 let height = ref_size.height;
                 let render_width = ref_size.render_width;
@@ -704,13 +697,16 @@ unsafe fn parse_refidx(
         let mut shifted_frame_offset = [0; 8];
         let current_frame_offset = 1 << seqhdr.order_hint_n_bits - 1;
         for i in 0..8 {
-            if c.refs[i as usize].p.p.frame_hdr.is_null() {
-                return Err(EINVAL);
-            }
             shifted_frame_offset[i as usize] = current_frame_offset
                 + get_poc_diff(
                     seqhdr.order_hint_n_bits,
-                    (*c.refs[i as usize].p.p.frame_hdr).frame_offset,
+                    c.refs[i as usize]
+                        .p
+                        .p
+                        .frame_hdr
+                        .as_ref()
+                        .ok_or(EINVAL)?
+                        .frame_offset,
                     frame_offset,
                 );
         }
@@ -805,12 +801,15 @@ unsafe fn parse_refidx(
         if seqhdr.frame_id_numbers_present != 0 {
             let delta_ref_frame_id_minus_1 = gb.get_bits(seqhdr.delta_frame_id_n_bits) as c_int;
             let ref_frame_id =
-                frame_id + ((1) << seqhdr.frame_id_n_bits) - delta_ref_frame_id_minus_1 - 1
-                    & ((1) << seqhdr.frame_id_n_bits) - 1;
-            let ref_frame_hdr = c.refs[refidx[i as usize] as usize].p.p.frame_hdr;
-            if ref_frame_hdr.is_null() || (*ref_frame_hdr).frame_id != ref_frame_id {
-                return Err(EINVAL);
-            }
+                frame_id + (1 << seqhdr.frame_id_n_bits) - delta_ref_frame_id_minus_1 - 1
+                    & (1 << seqhdr.frame_id_n_bits) - 1;
+            c.refs[refidx[i as usize] as usize]
+                .p
+                .p
+                .frame_hdr
+                .as_ref()
+                .filter(|ref_frame_hdr| ref_frame_hdr.frame_id == ref_frame_id)
+                .ok_or(EINVAL)?;
         }
     }
     Ok(refidx)
@@ -1117,10 +1116,12 @@ unsafe fn parse_segmentation(
             // segmentation data from the reference frame.
             assert!(primary_ref_frame != RAV1D_PRIMARY_REF_NONE);
             let pri_ref = refidx[primary_ref_frame as usize];
-            if (c.refs[pri_ref as usize].p.p.frame_hdr).is_null() {
-                return Err(EINVAL);
-            }
-            (*c.refs[pri_ref as usize].p.p.frame_hdr)
+            c.refs[pri_ref as usize]
+                .p
+                .p
+                .frame_hdr
+                .as_ref()
+                .ok_or(EINVAL)?
                 .segmentation
                 .seg_data
                 .clone()
@@ -1246,10 +1247,12 @@ unsafe fn parse_loopfilter(
             mode_ref_deltas = default_mode_ref_deltas.clone();
         } else {
             let r#ref = refidx[primary_ref_frame as usize];
-            if (c.refs[r#ref as usize].p.p.frame_hdr).is_null() {
-                return Err(EINVAL);
-            }
-            mode_ref_deltas = (*c.refs[r#ref as usize].p.p.frame_hdr)
+            mode_ref_deltas = c.refs[r#ref as usize]
+                .p
+                .p
+                .frame_hdr
+                .as_ref()
+                .ok_or(EINVAL)?
                 .loopfilter
                 .mode_ref_deltas
                 .clone();
@@ -1394,11 +1397,13 @@ unsafe fn parse_skip_mode(
         let mut off_before_idx = 0;
         let mut off_after_idx = 0;
         for i in 0..7 {
-            if c.refs[refidx[i as usize] as usize].p.p.frame_hdr.is_null() {
-                return Err(EINVAL);
-            }
-            let refpoc =
-                (*c.refs[refidx[i as usize] as usize].p.p.frame_hdr).frame_offset as c_uint;
+            let refpoc = c.refs[refidx[i as usize] as usize]
+                .p
+                .p
+                .frame_hdr
+                .as_ref()
+                .ok_or(EINVAL)?
+                .frame_offset as c_uint;
 
             let diff = get_poc_diff(seqhdr.order_hint_n_bits, refpoc as c_int, poc as c_int);
             if diff > 0 {
@@ -1431,11 +1436,13 @@ unsafe fn parse_skip_mode(
             let mut off_before2 = 0xffffffff;
             let mut off_before2_idx = 0;
             for i in 0..7 {
-                if (c.refs[refidx[i as usize] as usize].p.p.frame_hdr).is_null() {
-                    return Err(EINVAL);
-                }
-                let refpoc =
-                    (*c.refs[refidx[i as usize] as usize].p.p.frame_hdr).frame_offset as c_uint;
+                let refpoc = c.refs[refidx[i as usize] as usize]
+                    .p
+                    .p
+                    .frame_hdr
+                    .as_ref()
+                    .ok_or(EINVAL)?
+                    .frame_offset as c_uint;
                 if get_poc_diff(
                     seqhdr.order_hint_n_bits,
                     refpoc as c_int,
@@ -1508,10 +1515,13 @@ unsafe fn parse_gmv(
                 &default_gmv
             } else {
                 let pri_ref = refidx[primary_ref_frame as usize];
-                if (c.refs[pri_ref as usize].p.p.frame_hdr).is_null() {
-                    return Err(EINVAL);
-                }
-                &(*c.refs[pri_ref as usize].p.p.frame_hdr).gmv[i]
+                &c.refs[pri_ref as usize]
+                    .p
+                    .p
+                    .frame_hdr
+                    .as_ref()
+                    .ok_or(EINVAL)?
+                    .gmv[i]
             };
             let mat = &mut gmv.matrix;
             let ref_mat = &ref_gmv.matrix;
@@ -1680,12 +1690,17 @@ unsafe fn parse_film_grain(
                     break;
                 }
             }
-            if !found || c.refs[refidx as usize].p.p.frame_hdr.is_null() {
+            if !found {
                 return Err(EINVAL);
             }
             Rav1dFilmGrainData {
                 seed,
-                ..(*c.refs[refidx as usize].p.p.frame_hdr)
+                ..c.refs[refidx as usize]
+                    .p
+                    .p
+                    .frame_hdr
+                    .as_ref()
+                    .ok_or(EINVAL)?
                     .film_grain
                     .data
                     .clone()
@@ -1731,10 +1746,13 @@ unsafe fn parse_frame_hdr(
         let frame_id;
         if seqhdr.frame_id_numbers_present != 0 {
             frame_id = gb.get_bits(seqhdr.frame_id_n_bits) as c_int;
-            let ref_frame_hdr = c.refs[existing_frame_idx as usize].p.p.frame_hdr;
-            if ref_frame_hdr.is_null() || (*ref_frame_hdr).frame_id != frame_id {
-                return Err(EINVAL);
-            }
+            c.refs[existing_frame_idx as usize]
+                .p
+                .p
+                .frame_hdr
+                .as_ref()
+                .filter(|ref_frame_hdr| ref_frame_hdr.frame_id == frame_id)
+                .ok_or(EINVAL)?;
         } else {
             // Default initialization.
             frame_id = Default::default();
@@ -2106,19 +2124,14 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
     unsafe fn skip(c: &mut Rav1dContext, len: usize, init_byte_pos: usize) -> usize {
         // update refs with only the headers in case we skip the frame
         for i in 0..8 {
-            if (*c.frame_hdr).refresh_frame_flags & (1 << i) != 0 {
+            if c.frame_hdr.as_ref().unwrap().refresh_frame_flags & (1 << i) != 0 {
                 rav1d_thread_picture_unref(&mut c.refs[i as usize].p);
-                c.refs[i as usize].p.p.frame_hdr = c.frame_hdr;
-                c.refs[i as usize].p.p.seq_hdr = c.seq_hdr;
-                c.refs[i as usize].p.p.frame_hdr_ref = c.frame_hdr_ref;
-                c.refs[i as usize].p.p.seq_hdr_ref = c.seq_hdr_ref;
-                rav1d_ref_inc(c.frame_hdr_ref);
-                rav1d_ref_inc(c.seq_hdr_ref);
+                c.refs[i as usize].p.p.frame_hdr = c.frame_hdr.clone();
+                c.refs[i as usize].p.p.seq_hdr = c.seq_hdr.clone();
             }
         }
 
-        rav1d_ref_dec(&mut c.frame_hdr_ref);
-        c.frame_hdr = 0 as *mut Rav1dFrameHeader;
+        let _ = mem::take(&mut c.frame_hdr);
         c.n_tiles = 0;
 
         len + init_byte_pos
@@ -2186,11 +2199,7 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
         init_byte_pos: usize,
         len: usize,
     ) -> Rav1dResult {
-        if c.frame_hdr.is_null() {
-            return Err(EINVAL);
-        }
-
-        let hdr = parse_tile_hdr(&(*c.frame_hdr).tiling, gb);
+        let hdr = parse_tile_hdr(&c.frame_hdr.as_ref().ok_or(EINVAL)?.tiling, gb);
         // Align to the next byte boundary and check for overrun.
         gb.bytealign();
         if check_for_overrun(c, gb, init_bit_pos, len) != 0 {
@@ -2227,84 +2236,62 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
 
     match r#type {
         RAV1D_OBU_SEQ_HDR => {
-            let mut r#ref = rav1d_ref_create_using_pool(
-                c.seq_hdr_pool,
-                ::core::mem::size_of::<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>(),
-            );
-            if r#ref.is_null() {
-                return Err(ENOMEM);
-            }
-            let seq_hdrs = (*r#ref)
-                .data
-                .cast::<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>();
             let seq_hdr = parse_seq_hdr(c, &mut gb).inspect_err(|_| {
                 writeln!(c.logger, "Error parsing sequence header");
-                rav1d_ref_dec(&mut r#ref);
             })?;
-            (*seq_hdrs) = DRav1d::from_rav1d(seq_hdr);
-            let seq_hdr = &mut (*seq_hdrs).rav1d as *mut Rav1dSequenceHeader;
             if check_for_overrun(c, &mut gb, init_bit_pos, len) != 0 {
-                rav1d_ref_dec(&mut r#ref);
                 return Err(EINVAL);
             }
             // If we have read a sequence header which is different from the old one,
             // this is a new video sequence and can't use any previous state.
             // Free that state.
 
-            if c.seq_hdr.is_null() {
-                c.frame_hdr = 0 as *mut Rav1dFrameHeader;
-                c.frame_flags |= PictureFlags::NEW_SEQUENCE;
-            } else if !(*seq_hdr).eq_without_operating_parameter_info(&*c.seq_hdr) {
-                // See 7.5, `operating_parameter_info` is allowed to change in
-                // sequence headers of a single sequence.
-                c.frame_hdr = 0 as *mut Rav1dFrameHeader;
-                let _ = mem::take(&mut c.content_light);
-                let _ = mem::take(&mut c.mastering_display);
-                for i in 0..8 {
-                    if !c.refs[i as usize].p.p.frame_hdr.is_null() {
-                        rav1d_thread_picture_unref(&mut c.refs[i as usize].p);
+            match &c.seq_hdr {
+                None => {
+                    c.frame_hdr = None;
+                    c.frame_flags |= PictureFlags::NEW_SEQUENCE;
+                }
+                Some(c_seq_hdr) if !seq_hdr.eq_without_operating_parameter_info(&c_seq_hdr) => {
+                    // See 7.5, `operating_parameter_info` is allowed to change in
+                    // sequence headers of a single sequence.
+                    c.frame_hdr = None;
+                    let _ = mem::take(&mut c.content_light);
+                    let _ = mem::take(&mut c.mastering_display);
+                    for i in 0..8 {
+                        if c.refs[i as usize].p.p.frame_hdr.is_some() {
+                            rav1d_thread_picture_unref(&mut c.refs[i as usize].p);
+                        }
+                        rav1d_ref_dec(&mut c.refs[i as usize].segmap);
+                        rav1d_ref_dec(&mut c.refs[i as usize].refmvs);
+                        rav1d_cdf_thread_unref(&mut c.cdf[i as usize]);
                     }
-                    rav1d_ref_dec(&mut c.refs[i as usize].segmap);
-                    rav1d_ref_dec(&mut c.refs[i as usize].refmvs);
-                    rav1d_cdf_thread_unref(&mut c.cdf[i as usize]);
+                    c.frame_flags |= PictureFlags::NEW_SEQUENCE;
+                }
+                Some(c_seq_hdr)
+                    if seq_hdr.operating_parameter_info != c_seq_hdr.operating_parameter_info =>
+                {
+                    // If operating_parameter_info changed, signal it
+                    c.frame_flags |= PictureFlags::NEW_OP_PARAMS_INFO;
                 }
-                c.frame_flags |= PictureFlags::NEW_SEQUENCE;
-            } else if (*seq_hdr).operating_parameter_info != (*c.seq_hdr).operating_parameter_info {
-                // If operating_parameter_info changed, signal it
-                c.frame_flags |= PictureFlags::NEW_OP_PARAMS_INFO;
+                _ => {}
             }
-            rav1d_ref_dec(&mut c.seq_hdr_ref);
-            c.seq_hdr_ref = r#ref;
-            c.seq_hdr = seq_hdr;
+            c.seq_hdr = Some(Arc::new(DRav1d::from_rav1d(seq_hdr))); // TODO(kkysen) fallible allocation
         }
-        RAV1D_OBU_REDUNDANT_FRAME_HDR if !c.frame_hdr.is_null() => {}
+        RAV1D_OBU_REDUNDANT_FRAME_HDR if c.frame_hdr.is_some() => {}
         RAV1D_OBU_REDUNDANT_FRAME_HDR | RAV1D_OBU_FRAME | RAV1D_OBU_FRAME_HDR if global => {}
         RAV1D_OBU_REDUNDANT_FRAME_HDR | RAV1D_OBU_FRAME | RAV1D_OBU_FRAME_HDR => {
-            if c.seq_hdr.is_null() {
-                return Err(EINVAL);
-            }
-            let frame_hdr = parse_frame_hdr(c, &*c.seq_hdr, temporal_id, spatial_id, &mut gb)
-                .inspect_err(|_| {
-                    writeln!(c.logger, "Error parsing frame header");
-                    c.frame_hdr = 0 as *mut Rav1dFrameHeader;
-                })?;
-            let drav1d_frame_hdr = DRav1d::from_rav1d(frame_hdr);
-            if c.frame_hdr_ref.is_null() {
-                c.frame_hdr_ref = rav1d_ref_create_using_pool(
-                    c.frame_hdr_pool,
-                    ::core::mem::size_of::<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>(),
-                );
-                if c.frame_hdr_ref.is_null() {
-                    return Err(ENOMEM);
-                }
-            }
-            // ensure that the reference is writable
-            debug_assert!(rav1d_ref_is_writable(c.frame_hdr_ref) != 0);
-            let drav1d_frame_hdr_ptr = (*c.frame_hdr_ref)
-                .data
-                .cast::<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>();
-            drav1d_frame_hdr_ptr.write(drav1d_frame_hdr);
-            c.frame_hdr = &mut (*drav1d_frame_hdr_ptr).rav1d;
+            c.frame_hdr = None;
+            // TODO(kkysen) C originally re-used this allocation,
+            // but it was also pooling, which we've dropped for now.
+
+            let frame_hdr = parse_frame_hdr(
+                c,
+                c.seq_hdr.as_ref().ok_or(EINVAL)?,
+                temporal_id,
+                spatial_id,
+                &mut gb,
+            )
+            .inspect_err(|_| writeln!(c.logger, "Error parsing frame header"))?;
 
             for mut tile in c.tiles.drain(..) {
                 rav1d_data_unref_internal(&mut tile.data);
@@ -2315,33 +2302,32 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
                 // so read the trailing bit and check for overrun.
                 gb.get_bit();
                 if check_for_overrun(c, &mut gb, init_bit_pos, len) != 0 {
-                    c.frame_hdr = 0 as *mut Rav1dFrameHeader;
                     return Err(EINVAL);
                 }
             }
 
             if c.frame_size_limit != 0
-                && (*c.frame_hdr).size.width[1] as i64 * (*c.frame_hdr).size.height as i64
+                && frame_hdr.size.width[1] as i64 * frame_hdr.size.height as i64
                     > c.frame_size_limit as i64
             {
                 writeln!(
                     c.logger,
                     "Frame size {}x{} exceeds limit {}",
-                    (*c.frame_hdr).size.width[1],
-                    (*c.frame_hdr).size.height,
-                    c.frame_size_limit,
+                    frame_hdr.size.width[1], frame_hdr.size.height, c.frame_size_limit,
                 );
-                c.frame_hdr = 0 as *mut Rav1dFrameHeader;
                 return Err(ERANGE);
             }
 
             if r#type == RAV1D_OBU_FRAME {
                 // OBU_FRAMEs shouldn't be signaled with `show_existing_frame`.
-                if (*c.frame_hdr).show_existing_frame != 0 {
-                    c.frame_hdr = 0 as *mut Rav1dFrameHeader;
+                if frame_hdr.show_existing_frame != 0 {
                     return Err(EINVAL);
                 }
+            }
 
+            c.frame_hdr = Some(Arc::new(DRav1d::from_rav1d(frame_hdr))); // TODO(kkysen) fallible allocation
+
+            if r#type == RAV1D_OBU_FRAME {
                 // This is the frame header at the start of a frame OBU.
                 // There's no trailing bit at the end to skip,
                 // but we do need to align to the next byte.
@@ -2489,20 +2475,15 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
         }
     }
 
-    if !c.seq_hdr.is_null() && !c.frame_hdr.is_null() {
-        if (*c.frame_hdr).show_existing_frame != 0 {
-            if c.refs[(*c.frame_hdr).existing_frame_idx as usize]
+    if let (Some(_), Some(frame_hdr)) = (c.seq_hdr.as_ref(), c.frame_hdr.as_ref()) {
+        let frame_hdr = &***frame_hdr;
+        if frame_hdr.show_existing_frame != 0 {
+            match c.refs[frame_hdr.existing_frame_idx as usize]
                 .p
                 .p
                 .frame_hdr
-                .is_null()
-            {
-                return Err(EINVAL);
-            }
-            match (*c.refs[(*c.frame_hdr).existing_frame_idx as usize]
-                .p
-                .p
-                .frame_hdr)
+                .as_ref()
+                .ok_or(EINVAL)?
                 .frame_type
             {
                 Rav1dFrameType::Inter | Rav1dFrameType::Switch => {
@@ -2517,20 +2498,17 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
                 }
                 _ => {}
             }
-            if c.refs[(*c.frame_hdr).existing_frame_idx as usize].p.p.data[0].is_null() {
+            if c.refs[frame_hdr.existing_frame_idx as usize].p.p.data[0].is_null() {
                 return Err(EINVAL);
             }
-            if c.strict_std_compliance
-                && !c.refs[(*c.frame_hdr).existing_frame_idx as usize]
-                    .p
-                    .showable
+            if c.strict_std_compliance && !c.refs[frame_hdr.existing_frame_idx as usize].p.showable
             {
                 return Err(EINVAL);
             }
             if c.n_fc == 1 {
                 rav1d_thread_picture_ref(
                     &mut c.out,
-                    &mut c.refs[(*c.frame_hdr).existing_frame_idx as usize].p,
+                    &mut c.refs[frame_hdr.existing_frame_idx as usize].p,
                 );
                 rav1d_picture_copy_props(
                     &mut (*c).out.p,
@@ -2541,10 +2519,7 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
                 );
                 // Must be removed from the context after being attached to the frame
                 let _ = mem::take(&mut c.itut_t35);
-                c.event_flags |= c.refs[(*c.frame_hdr).existing_frame_idx as usize]
-                    .p
-                    .flags
-                    .into();
+                c.event_flags |= c.refs[frame_hdr.existing_frame_idx as usize].p.flags.into();
             } else {
                 pthread_mutex_lock(&mut c.task_thread.lock);
                 // Need to append this to the frame output queue.
@@ -2602,7 +2577,7 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
                 }
                 rav1d_thread_picture_ref(
                     out_delayed,
-                    &mut c.refs[(*c.frame_hdr).existing_frame_idx as usize].p,
+                    &mut c.refs[frame_hdr.existing_frame_idx as usize].p,
                 );
                 (*out_delayed).visible = true;
                 rav1d_picture_copy_props(
@@ -2616,21 +2591,23 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
                 let _ = mem::take(&mut c.itut_t35);
                 pthread_mutex_unlock(&mut c.task_thread.lock);
             }
-            if (*c.refs[(*c.frame_hdr).existing_frame_idx as usize]
+            if c.refs[frame_hdr.existing_frame_idx as usize]
                 .p
                 .p
-                .frame_hdr)
+                .frame_hdr
+                .as_ref()
+                .unwrap()
                 .frame_type
                 == Rav1dFrameType::Key
             {
-                let r = (*c.frame_hdr).existing_frame_idx;
+                let r = frame_hdr.existing_frame_idx;
                 c.refs[r as usize].p.showable = false;
                 for i in 0..8 {
                     if i == r {
                         continue;
                     }
 
-                    if !c.refs[i as usize].p.p.frame_hdr.is_null() {
+                    if c.refs[i as usize].p.p.frame_hdr.is_some() {
                         rav1d_thread_picture_unref(&mut c.refs[i as usize].p);
                     }
                     rav1d_thread_picture_ref(&mut c.refs[i as usize].p, &mut c.refs[r as usize].p);
@@ -2646,13 +2623,13 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
                     rav1d_ref_dec(&mut c.refs[i as usize].refmvs);
                 }
             }
-            c.frame_hdr = 0 as *mut Rav1dFrameHeader;
-        } else if c.n_tiles == (*c.frame_hdr).tiling.cols * (*c.frame_hdr).tiling.rows {
-            match (*c.frame_hdr).frame_type {
+            c.frame_hdr = None;
+        } else if c.n_tiles == frame_hdr.tiling.cols * frame_hdr.tiling.rows {
+            match frame_hdr.frame_type {
                 Rav1dFrameType::Inter | Rav1dFrameType::Switch => {
                     if c.decode_frame_type > RAV1D_DECODEFRAMETYPE_REFERENCE
                         || c.decode_frame_type == RAV1D_DECODEFRAMETYPE_REFERENCE
-                            && (*c.frame_hdr).refresh_frame_flags == 0
+                            && frame_hdr.refresh_frame_flags == 0
                     {
                         return Ok(skip(c, len, init_byte_pos));
                     }
@@ -2660,7 +2637,7 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
                 Rav1dFrameType::Intra => {
                     if c.decode_frame_type > RAV1D_DECODEFRAMETYPE_INTRA
                         || c.decode_frame_type == RAV1D_DECODEFRAMETYPE_REFERENCE
-                            && (*c.frame_hdr).refresh_frame_flags == 0
+                            && frame_hdr.refresh_frame_flags == 0
                     {
                         return Ok(skip(c, len, init_byte_pos));
                     }
@@ -2672,7 +2649,7 @@ unsafe fn parse_obus(c: &mut Rav1dContext, r#in: &Rav1dData, global: bool) -> Ra
             }
             rav1d_submit_frame(&mut *c)?;
             assert!(c.tiles.is_empty());
-            c.frame_hdr = 0 as *mut Rav1dFrameHeader;
+            c.frame_hdr = None;
             c.n_tiles = 0;
         }
     }
diff --git a/src/picture.rs b/src/picture.rs
index 256981f8e..5c6c55684 100644
--- a/src/picture.rs
+++ b/src/picture.rs
@@ -2,7 +2,9 @@ use crate::include::common::validate::validate_input;
 use crate::include::dav1d::common::Rav1dDataProps;
 use crate::include::dav1d::dav1d::Rav1dEventFlags;
 use crate::include::dav1d::headers::DRav1d;
+use crate::include::dav1d::headers::Dav1dFrameHeader;
 use crate::include::dav1d::headers::Dav1dITUTT35;
+use crate::include::dav1d::headers::Dav1dSequenceHeader;
 use crate::include::dav1d::headers::Rav1dContentLightLevel;
 use crate::include::dav1d::headers::Rav1dFrameHeader;
 use crate::include::dav1d::headers::Rav1dITUTT35;
@@ -28,7 +30,6 @@ use crate::src::mem::Rav1dMemPoolBuffer;
 use crate::src::r#ref::rav1d_ref_dec;
 use crate::src::r#ref::rav1d_ref_inc;
 use crate::src::r#ref::rav1d_ref_wrap;
-use crate::src::r#ref::Rav1dRef;
 use bitflags::bitflags;
 use libc::free;
 use libc::malloc;
@@ -174,10 +175,8 @@ unsafe fn picture_alloc_with_edges(
     p: *mut Rav1dPicture,
     w: c_int,
     h: c_int,
-    seq_hdr: *mut Rav1dSequenceHeader,
-    seq_hdr_ref: *mut Rav1dRef,
-    frame_hdr: *mut Rav1dFrameHeader,
-    frame_hdr_ref: *mut Rav1dRef,
+    seq_hdr: &Option<Arc<DRav1d<Rav1dSequenceHeader, Dav1dSequenceHeader>>>,
+    frame_hdr: &Option<Arc<DRav1d<Rav1dFrameHeader, Dav1dFrameHeader>>>,
     content_light: &Option<Arc<Rav1dContentLightLevel>>,
     mastering_display: &Option<Arc<Rav1dMasteringDisplay>>,
     itut_t35: &Option<Arc<DRav1d<Rav1dITUTT35, Dav1dITUTT35>>>,
@@ -202,13 +201,11 @@ unsafe fn picture_alloc_with_edges(
     }
     (*p).p.w = w;
     (*p).p.h = h;
-    (*p).seq_hdr = seq_hdr;
-    (*p).frame_hdr = frame_hdr;
-    (*p).p.layout = (*seq_hdr).layout;
+    (*p).seq_hdr = seq_hdr.clone();
+    (*p).frame_hdr = frame_hdr.clone();
+    (*p).p.layout = seq_hdr.as_ref().unwrap().layout;
     (*p).p.bpc = bpc;
     (*p).m = Default::default();
-    (*p).seq_hdr_ref = seq_hdr_ref;
-    (*p).frame_hdr_ref = frame_hdr_ref;
     let res = (*p_allocator).alloc_picture(p);
     if res.is_err() {
         free(pic_ctx as *mut c_void);
@@ -234,12 +231,6 @@ unsafe fn picture_alloc_with_edges(
         );
         return Err(ENOMEM);
     }
-    if !seq_hdr_ref.is_null() {
-        rav1d_ref_inc(seq_hdr_ref);
-    }
-    if !frame_hdr_ref.is_null() {
-        rav1d_ref_inc(frame_hdr_ref);
-    }
     rav1d_picture_copy_props(p, content_light, mastering_display, itut_t35, props);
 
     if extra != 0 && !extra_ptr.is_null() {
@@ -269,15 +260,14 @@ pub(crate) unsafe fn rav1d_thread_picture_alloc(
 ) -> Rav1dResult {
     let p: *mut Rav1dThreadPicture = &mut (*f).sr_cur;
     let have_frame_mt = ((*c).n_fc > 1 as c_uint) as c_int;
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
     let res = picture_alloc_with_edges(
         c,
         &mut (*p).p,
-        (*(*f).frame_hdr).size.width[1],
-        (*(*f).frame_hdr).size.height,
-        (*f).seq_hdr,
-        (*f).seq_hdr_ref,
-        (*f).frame_hdr,
-        (*f).frame_hdr_ref,
+        frame_hdr.size.width[1],
+        frame_hdr.size.height,
+        &(*f).seq_hdr,
+        &(*f).frame_hdr,
         &(*c).content_light,
         &(*c).mastering_display,
         &(*c).itut_t35,
@@ -295,15 +285,15 @@ pub(crate) unsafe fn rav1d_thread_picture_alloc(
         return res;
     }
     let _ = mem::take(&mut (*c).itut_t35);
-    let flags_mask = if (*(*f).frame_hdr).show_frame != 0 || (*c).output_invisible_frames {
+    let flags_mask = if frame_hdr.show_frame != 0 || (*c).output_invisible_frames {
         PictureFlags::empty()
     } else {
         PictureFlags::NEW_SEQUENCE | PictureFlags::NEW_OP_PARAMS_INFO
     };
     (*p).flags = (*c).frame_flags;
     (*c).frame_flags &= flags_mask;
-    (*p).visible = (*(*f).frame_hdr).show_frame != 0;
-    (*p).showable = (*(*f).frame_hdr).showable_frame != 0;
+    (*p).visible = frame_hdr.show_frame != 0;
+    (*p).showable = frame_hdr.showable_frame != 0;
     if have_frame_mt != 0 {
         *(&mut *((*p).progress).offset(0) as *mut atomic_uint) = 0 as c_int as c_uint;
         *(&mut *((*p).progress).offset(1) as *mut atomic_uint) = 0 as c_int as c_uint;
@@ -323,10 +313,8 @@ pub(crate) unsafe fn rav1d_picture_alloc_copy(
         dst,
         w,
         (*src).p.h,
-        (*src).seq_hdr,
-        (*src).seq_hdr_ref,
-        (*src).frame_hdr,
-        (*src).frame_hdr_ref,
+        &(*src).seq_hdr,
+        &(*src).frame_hdr,
         &(*src).content_light,
         &(*src).mastering_display,
         &(*src).itut_t35,
@@ -349,12 +337,6 @@ pub(crate) unsafe fn rav1d_picture_ref(dst: &mut Rav1dPicture, src: &Rav1dPictur
         }
         rav1d_ref_inc(src.r#ref);
     }
-    if !src.frame_hdr_ref.is_null() {
-        rav1d_ref_inc(src.frame_hdr_ref);
-    }
-    if !src.seq_hdr_ref.is_null() {
-        rav1d_ref_inc(src.seq_hdr_ref);
-    }
     *dst = src.clone();
 }
 
@@ -393,8 +375,6 @@ pub(crate) unsafe fn rav1d_picture_unref_internal(p: &mut Rav1dPicture) {
         m: _,
         data,
         mut r#ref,
-        mut frame_hdr_ref,
-        mut seq_hdr_ref,
         ..
     } = mem::take(p);
     if !r#ref.is_null() {
@@ -403,8 +383,6 @@ pub(crate) unsafe fn rav1d_picture_unref_internal(p: &mut Rav1dPicture) {
         }
         rav1d_ref_dec(&mut r#ref);
     }
-    rav1d_ref_dec(&mut seq_hdr_ref);
-    rav1d_ref_dec(&mut frame_hdr_ref);
 }
 
 pub(crate) unsafe fn rav1d_thread_picture_unref(p: *mut Rav1dThreadPicture) {
diff --git a/src/recon.rs b/src/recon.rs
index 9357b92b4..a4e0480a0 100644
--- a/src/recon.rs
+++ b/src/recon.rs
@@ -117,8 +117,13 @@ use std::ops::BitOr;
 use std::slice;
 
 /// TODO: add feature and compile-time guard around this code
-pub(crate) unsafe fn DEBUG_BLOCK_INFO(f: &Rav1dFrameContext, t: &Rav1dTaskContext) -> bool {
-    false && (*f.frame_hdr).frame_offset == 2 && t.by >= 0 && t.by < 4 && t.bx >= 8 && t.bx < 12
+pub(crate) fn DEBUG_BLOCK_INFO(f: &Rav1dFrameContext, t: &Rav1dTaskContext) -> bool {
+    false
+        && f.frame_hdr.as_ref().unwrap().frame_offset == 2
+        && t.by >= 0
+        && t.by < 4
+        && t.bx >= 8
+        && t.bx < 12
 }
 
 pub(crate) type recon_b_intra_fn =
@@ -451,7 +456,8 @@ unsafe fn decode_coefs<BD: BitDepth>(
     let ts: *mut Rav1dTileState = (*t).ts;
     let chroma = (plane != 0) as c_int;
     let f: *const Rav1dFrameContext = (*t).f;
-    let lossless = (*(*f).frame_hdr).segmentation.lossless[b.seg_id as usize];
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    let lossless = frame_hdr.segmentation.lossless[b.seg_id as usize];
     let t_dim = &dav1d_txfm_dimensions[tx as usize];
     let dbg = DEBUG_BLOCK_INFO(&*f, &*t) as c_int;
     if dbg != 0 {
@@ -492,7 +498,7 @@ unsafe fn decode_coefs<BD: BitDepth>(
         } else {
             get_uv_inter_txtp(&*t_dim, *txtp) as c_uint
         }) as TxfmType;
-    } else if (*(*f).frame_hdr).segmentation.qidx[b.seg_id as usize] == 0 {
+    } else if frame_hdr.segmentation.qidx[b.seg_id as usize] == 0 {
         *txtp = DCT_DCT;
     } else {
         let idx: c_uint;
@@ -504,8 +510,7 @@ unsafe fn decode_coefs<BD: BitDepth>(
                 } else {
                     b.c2rust_unnamed.c2rust_unnamed.y_mode as c_int
                 }) as IntraPredMode;
-            if (*(*f).frame_hdr).reduced_txtp_set != 0 || (*t_dim).min as c_int == TX_16X16 as c_int
-            {
+            if frame_hdr.reduced_txtp_set != 0 || (*t_dim).min as c_int == TX_16X16 as c_int {
                 idx = rav1d_msac_decode_symbol_adapt4(
                     &mut (*ts).msac,
                     &mut (*ts).cdf.m.txtp_intra2[(*t_dim).min as usize][y_mode_nofilt as usize],
@@ -534,8 +539,7 @@ unsafe fn decode_coefs<BD: BitDepth>(
                 );
             }
         } else {
-            if (*(*f).frame_hdr).reduced_txtp_set != 0 || (*t_dim).max as c_int == TX_32X32 as c_int
-            {
+            if frame_hdr.reduced_txtp_set != 0 || (*t_dim).max as c_int == TX_32X32 as c_int {
                 idx = rav1d_msac_decode_bool_adapt(
                     &mut (*ts).msac,
                     &mut (*ts).cdf.m.txtp_inter3[(*t_dim).min as usize],
@@ -2542,7 +2546,8 @@ pub(crate) unsafe fn rav1d_recon_b_intra<BD: BitDepth>(
         .as_mut_ptr();
     let cbw4 = bw4 + ss_hor >> ss_hor;
     let cbh4 = bh4 + ss_ver >> ss_ver;
-    let intra_edge_filter_flag = (*(*f).seq_hdr).intra_edge_filter << 10;
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+    let intra_edge_filter_flag = seq_hdr.intra_edge_filter << 10;
     let mut init_y = 0;
     while init_y < h4 {
         let sub_h4 = cmp::min(h4, 16 + init_y);
@@ -2665,7 +2670,7 @@ pub(crate) unsafe fn rav1d_recon_b_intra<BD: BitDepth>(
                             &mut angle,
                             (*t_dim).w as c_int,
                             (*t_dim).h as c_int,
-                            (*(*f).seq_hdr).intra_edge_filter,
+                            seq_hdr.intra_edge_filter,
                             edge,
                             BD::from_c((*f).bitdepth_max),
                         );
@@ -3070,7 +3075,7 @@ pub(crate) unsafe fn rav1d_recon_b_intra<BD: BitDepth>(
                                     &mut angle,
                                     (*uv_t_dim).w as c_int,
                                     (*uv_t_dim).h as c_int,
-                                    (*(*f).seq_hdr).intra_edge_filter,
+                                    seq_hdr.intra_edge_filter,
                                     edge,
                                     BD::from_c((*f).bitdepth_max),
                                 );
@@ -3282,8 +3287,9 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
     let uvdstoff: ptrdiff_t = 4
         * ((t.bx >> ss_hor) as isize
             + (t.by >> ss_ver) as isize * BD::pxstride((*f).cur.stride[1] as usize) as isize);
-    if (*(*f).frame_hdr).frame_type.is_key_or_intra() {
-        if (*(*f).frame_hdr).size.super_res.enabled != 0 {
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    if frame_hdr.frame_type.is_key_or_intra() {
+        if frame_hdr.size.super_res.enabled != 0 {
             unreachable!();
         }
         res = mc::<BD>(
@@ -3361,11 +3367,9 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
                 0 as c_int,
                 refp,
                 if b.c2rust_unnamed.c2rust_unnamed_0.motion_mode as c_int == MM_WARP as c_int {
-                    &mut t.warpmv
+                    &t.warpmv
                 } else {
-                    &mut *((*(*f).frame_hdr).gmv).as_mut_ptr().offset(
-                        *(b.c2rust_unnamed.c2rust_unnamed_0.r#ref).as_ptr().offset(0) as isize,
-                    )
+                    &frame_hdr.gmv[b.c2rust_unnamed.c2rust_unnamed_0.r#ref[0] as usize]
                 },
             );
             if res != 0 {
@@ -3744,12 +3748,9 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
                             if b.c2rust_unnamed.c2rust_unnamed_0.motion_mode as c_int
                                 == MM_WARP as c_int
                             {
-                                &mut t.warpmv
+                                &t.warpmv
                             } else {
-                                &mut *((*(*f).frame_hdr).gmv).as_mut_ptr().offset(
-                                    *(b.c2rust_unnamed.c2rust_unnamed_0.r#ref).as_ptr().offset(0)
-                                        as isize,
-                                )
+                                &frame_hdr.gmv[b.c2rust_unnamed.c2rust_unnamed_0.r#ref[0] as usize]
                             },
                         );
                         if res != 0 {
@@ -3949,11 +3950,7 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
                     b_dim,
                     0 as c_int,
                     refp,
-                    &mut *((*(*f).frame_hdr).gmv).as_mut_ptr().offset(
-                        *(b.c2rust_unnamed.c2rust_unnamed_0.r#ref)
-                            .as_ptr()
-                            .offset(i as isize) as isize,
-                    ),
+                    &frame_hdr.gmv[b.c2rust_unnamed.c2rust_unnamed_0.r#ref[i as usize] as usize],
                 );
                 if res != 0 {
                     return res;
@@ -4123,11 +4120,8 @@ pub(crate) unsafe fn rav1d_recon_b_inter<BD: BitDepth>(
                             b_dim,
                             1 + pl,
                             refp,
-                            &mut *((*(*f).frame_hdr).gmv).as_mut_ptr().offset(
-                                *(b.c2rust_unnamed.c2rust_unnamed_0.r#ref)
-                                    .as_ptr()
-                                    .offset(i as isize) as isize,
-                            ),
+                            &frame_hdr.gmv
+                                [b.c2rust_unnamed.c2rust_unnamed_0.r#ref[i as usize] as usize],
                         );
                         if res != 0 {
                             return res;
@@ -4462,8 +4456,9 @@ pub(crate) unsafe fn rav1d_filter_sbrow_deblock_cols<BD: BitDepth>(
     f: &mut Rav1dFrameContext,
     sby: c_int,
 ) {
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
     if (*f.c).inloop_filters as c_uint & RAV1D_INLOOPFILTER_DEBLOCK as c_int as c_uint == 0
-        || (*f.frame_hdr).loopfilter.level_y[0] == 0 && (*f.frame_hdr).loopfilter.level_y[1] == 0
+        || frame_hdr.loopfilter.level_y[0] == 0 && frame_hdr.loopfilter.level_y[1] == 0
     {
         return;
     }
@@ -4479,8 +4474,9 @@ pub(crate) unsafe fn rav1d_filter_sbrow_deblock_cols<BD: BitDepth>(
             (y as isize * BD::pxstride(f.cur.stride[1] as usize) as isize >> ss_ver) as isize,
         ),
     ];
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
     let mask: *mut Av1Filter =
-        (f.lf.mask).offset(((sby >> ((*f.seq_hdr).sb128 == 0) as c_int) * f.sb128w) as isize);
+        (f.lf.mask).offset(((sby >> (seq_hdr.sb128 == 0) as c_int) * f.sb128w) as isize);
     rav1d_loopfilter_sbrow_cols::<BD>(
         f,
         p.as_ptr(),
@@ -4506,14 +4502,16 @@ pub(crate) unsafe fn rav1d_filter_sbrow_deblock_rows<BD: BitDepth>(
             (y as isize * BD::pxstride(f.cur.stride[1] as usize) as isize >> ss_ver) as isize,
         ),
     ];
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
     let mask: *mut Av1Filter =
-        (f.lf.mask).offset(((sby >> ((*f.seq_hdr).sb128 == 0) as c_int) * f.sb128w) as isize);
+        (f.lf.mask).offset(((sby >> (seq_hdr.sb128 == 0) as c_int) * f.sb128w) as isize);
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
     if (*f.c).inloop_filters as c_uint & RAV1D_INLOOPFILTER_DEBLOCK as c_int as c_uint != 0
-        && ((*f.frame_hdr).loopfilter.level_y[0] != 0 || (*f.frame_hdr).loopfilter.level_y[1] != 0)
+        && (frame_hdr.loopfilter.level_y[0] != 0 || frame_hdr.loopfilter.level_y[1] != 0)
     {
         rav1d_loopfilter_sbrow_rows::<BD>(f, p.as_ptr(), mask, sby);
     }
-    if (*f.seq_hdr).cdef != 0 || f.lf.restore_planes != 0 {
+    if seq_hdr.cdef != 0 || f.lf.restore_planes != 0 {
         rav1d_copy_lpf::<BD>(f, p.as_ptr(), sby);
     }
 }
@@ -4537,10 +4535,11 @@ pub(crate) unsafe fn rav1d_filter_sbrow_cdef<BD: BitDepth>(tc: &mut Rav1dTaskCon
             (y as isize * BD::pxstride((*f).cur.stride[1] as usize) as isize >> ss_ver) as isize,
         ),
     ];
-    let prev_mask: *mut Av1Filter = ((*f).lf.mask)
-        .offset(((sby - 1 >> ((*(*f).seq_hdr).sb128 == 0) as c_int) * (*f).sb128w) as isize);
-    let mask: *mut Av1Filter = ((*f).lf.mask)
-        .offset(((sby >> ((*(*f).seq_hdr).sb128 == 0) as c_int) * (*f).sb128w) as isize);
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+    let prev_mask: *mut Av1Filter =
+        ((*f).lf.mask).offset(((sby - 1 >> (seq_hdr.sb128 == 0) as c_int) * (*f).sb128w) as isize);
+    let mask: *mut Av1Filter =
+        ((*f).lf.mask).offset(((sby >> (seq_hdr.sb128 == 0) as c_int) * (*f).sb128w) as isize);
     let start = sby * sbsz;
     if sby != 0 {
         let ss_ver =
@@ -4654,10 +4653,12 @@ pub(crate) unsafe fn rav1d_filter_sbrow_lr<BD: BitDepth>(f: &mut Rav1dFrameConte
 pub(crate) unsafe fn rav1d_filter_sbrow<BD: BitDepth>(f: &mut Rav1dFrameContext, sby: c_int) {
     rav1d_filter_sbrow_deblock_cols::<BD>(f, sby);
     rav1d_filter_sbrow_deblock_rows::<BD>(f, sby);
-    if (*f.seq_hdr).cdef != 0 {
+    let seq_hdr = &***f.seq_hdr.as_ref().unwrap();
+    let frame_hdr = &***f.frame_hdr.as_ref().unwrap();
+    if seq_hdr.cdef != 0 {
         rav1d_filter_sbrow_cdef::<BD>(&mut *(*f.c).tc, sby);
     }
-    if (*f.frame_hdr).size.width[0] != (*f.frame_hdr).size.width[1] {
+    if frame_hdr.size.width[0] != frame_hdr.size.width[1] {
         rav1d_filter_sbrow_resize::<BD>(f, sby);
     }
     if f.lf.restore_planes != 0 {
diff --git a/src/ref.rs b/src/ref.rs
index 2dbf7a830..dddb89bb3 100644
--- a/src/ref.rs
+++ b/src/ref.rs
@@ -125,8 +125,3 @@ pub unsafe fn rav1d_ref_dec(pref: *mut *mut Rav1dRef) {
         }
     }
 }
-
-pub unsafe fn rav1d_ref_is_writable(r#ref: *mut Rav1dRef) -> c_int {
-    return (::core::intrinsics::atomic_load_seqcst(&mut (*r#ref).ref_cnt as *mut atomic_int) == 1
-        && !((*r#ref).data).is_null()) as c_int;
-}
diff --git a/src/thread_task.rs b/src/thread_task.rs
index c89fd9d12..f317670f6 100644
--- a/src/thread_task.rs
+++ b/src/thread_task.rs
@@ -360,10 +360,12 @@ unsafe fn create_filter_sbrow(
     pass: c_int,
     res_t: *mut *mut Rav1dTask,
 ) -> c_int {
-    let has_deblock = ((*(*f).frame_hdr).loopfilter.level_y[0] != 0
-        || (*(*f).frame_hdr).loopfilter.level_y[1] != 0) as c_int;
-    let has_cdef = (*(*f).seq_hdr).cdef;
-    let has_resize = ((*(*f).frame_hdr).size.width[0] != (*(*f).frame_hdr).size.width[1]) as c_int;
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    let has_deblock =
+        (frame_hdr.loopfilter.level_y[0] != 0 || frame_hdr.loopfilter.level_y[1] != 0) as c_int;
+    let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+    let has_cdef = seq_hdr.cdef;
+    let has_resize = (frame_hdr.size.width[0] != frame_hdr.size.width[1]) as c_int;
     let has_lr = (*f).lf.restore_planes;
     let mut tasks: *mut Rav1dTask = (*f).task_thread.tasks;
     let uses_2pass = ((*(*f).c).n_fc > 1 as c_uint) as c_int;
@@ -423,7 +425,8 @@ pub(crate) unsafe fn rav1d_task_create_tile_sbrow(
 ) -> Rav1dResult {
     let mut tasks: *mut Rav1dTask = (*f).task_thread.tile_tasks[0];
     let uses_2pass = ((*(*f).c).n_fc > 1 as c_uint) as c_int;
-    let num_tasks = (*(*f).frame_hdr).tiling.cols * (*(*f).frame_hdr).tiling.rows;
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    let num_tasks = frame_hdr.tiling.cols * frame_hdr.tiling.rows;
     if pass < 2 {
         let alloc_num_tasks = num_tasks * (1 + uses_2pass);
         if alloc_num_tasks > (*f).task_thread.num_tile_tasks {
@@ -573,7 +576,8 @@ unsafe fn check_tile(t: *mut Rav1dTask, f: *mut Rav1dFrameContext, frame_mt: c_i
         error = (p2 == TILE_ERROR) as c_int;
         error |= ::core::intrinsics::atomic_or_seqcst(&mut (*f).task_thread.error, error);
     }
-    if error == 0 && frame_mt != 0 && !(*(*f).frame_hdr).frame_type.is_key_or_intra() {
+    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+    if error == 0 && frame_mt != 0 && !frame_hdr.frame_type.is_key_or_intra() {
         let p: *const Rav1dThreadPicture = &mut (*f).sr_cur;
         let ss_ver =
             ((*p).p.p.layout as c_uint == Rav1dPixelLayout::I420 as c_int as c_uint) as c_int;
@@ -937,7 +941,8 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                             {
                                                 unreachable!();
                                             }
-                                            let tile_row_base = (*(*f).frame_hdr).tiling.cols
+                                            let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+                                            let tile_row_base = frame_hdr.tiling.cols
                                                 * (*f).frame_thread.next_tile_row[p as usize];
                                             if p != 0 {
                                                 let p1_0 = (*f)
@@ -959,9 +964,11 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                             match current_block {
                                                 5395695591151878490 => {}
                                                 _ => {
+                                                    let frame_hdr =
+                                                        &***(*f).frame_hdr.as_ref().unwrap();
                                                     let mut tc_0 = 0;
                                                     loop {
-                                                        if !(tc_0 < (*(*f).frame_hdr).tiling.cols) {
+                                                        if !(tc_0 < frame_hdr.tiling.cols) {
                                                             current_block = 3222590281903869779;
                                                             break;
                                                         }
@@ -999,11 +1006,10 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                                                     (*f).frame_thread.next_tile_row
                                                                         [p as usize]
                                                                         + 1;
-                                                                let start = (*(*f).frame_hdr)
-                                                                    .tiling
-                                                                    .row_start_sb
-                                                                    [ntr as usize]
-                                                                    as c_int;
+                                                                let start =
+                                                                    frame_hdr.tiling.row_start_sb
+                                                                        [ntr as usize]
+                                                                        as c_int;
                                                                 if (*next_t).sby == start {
                                                                     (*f).frame_thread
                                                                         .next_tile_row
@@ -1144,7 +1150,8 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                     {
                                         res_0 = rav1d_decode_frame_init_cdf(&mut *f);
                                     }
-                                    if (*(*f).frame_hdr).refresh_context != 0
+                                    let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+                                    if frame_hdr.refresh_context != 0
                                         && !(*f).task_thread.update_set
                                     {
                                         ::core::intrinsics::atomic_store_seqcst(
@@ -1180,8 +1187,7 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                                 );
                                                 ::core::intrinsics::atomic_xsub_seqcst(
                                                     &mut (*f).task_thread.task_counter,
-                                                    (*(*f).frame_hdr).tiling.cols
-                                                        * (*(*f).frame_hdr).tiling.rows
+                                                    frame_hdr.tiling.cols * frame_hdr.tiling.rows
                                                         + (*f).sbh,
                                                 );
                                                 ::core::intrinsics::atomic_store_seqcst(
@@ -1304,18 +1310,18 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                         error_0 = ::core::intrinsics::atomic_load_seqcst(
                                             &mut (*f).task_thread.error,
                                         );
-                                        if (*(*f).frame_hdr).refresh_context != 0
+                                        let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+                                        if frame_hdr.refresh_context != 0
                                             && (*tc).frame_thread.pass <= 1
                                             && (*f).task_thread.update_set
-                                            && (*(*f).frame_hdr).tiling.update == tile_idx
+                                            && frame_hdr.tiling.update == tile_idx
                                         {
                                             if error_0 == 0 {
                                                 rav1d_cdf_thread_update(
-                                                    (*f).frame_hdr,
+                                                    frame_hdr,
                                                     (*f).out_cdf.data.cdf,
-                                                    &mut (*((*f).ts).offset(
-                                                        (*(*f).frame_hdr).tiling.update as isize,
-                                                    ))
+                                                    &mut (*((*f).ts)
+                                                        .offset(frame_hdr.tiling.update as isize))
                                                     .cdf,
                                                 );
                                             }
@@ -1438,8 +1444,10 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                 {
                                     ((*f).bd_fn.filter_sbrow_deblock_rows)(&mut *f, sby);
                                 }
-                                if (*(*f).frame_hdr).loopfilter.level_y[0] != 0
-                                    || (*(*f).frame_hdr).loopfilter.level_y[1] != 0
+                                let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+                                let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+                                if frame_hdr.loopfilter.level_y[0] != 0
+                                    || frame_hdr.loopfilter.level_y[1] != 0
                                 {
                                     error_0 = ::core::intrinsics::atomic_load_seqcst(
                                         &mut (*f).task_thread.error,
@@ -1456,7 +1464,7 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                                     {
                                         pthread_cond_signal(&mut (*ttd).cond);
                                     }
-                                } else if (*(*f).seq_hdr).cdef != 0 || (*f).lf.restore_planes != 0 {
+                                } else if seq_hdr.cdef != 0 || (*f).lf.restore_planes != 0 {
                                     (*f).frame_thread.copy_lpf_progress[(sby >> 5) as usize]
                                         .fetch_or((1 as c_uint) << (sby & 31), Ordering::SeqCst);
                                     if sby != 0 {
@@ -1480,7 +1488,8 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                         }
                         match current_block {
                             5292528706010880565 => {
-                                if (*(*f).seq_hdr).cdef != 0 {
+                                let seq_hdr = &***(*f).seq_hdr.as_ref().unwrap();
+                                if seq_hdr.cdef != 0 {
                                     if ::core::intrinsics::atomic_load_seqcst(
                                         &mut (*f).task_thread.error as *mut atomic_int,
                                     ) == 0
@@ -1502,9 +1511,8 @@ pub unsafe extern "C" fn rav1d_worker_task(data: *mut c_void) -> *mut c_void {
                         }
                         match current_block {
                             12196494833634779273 => {
-                                if (*(*f).frame_hdr).size.width[0]
-                                    != (*(*f).frame_hdr).size.width[1]
-                                {
+                                let frame_hdr = &***(*f).frame_hdr.as_ref().unwrap();
+                                if frame_hdr.size.width[0] != frame_hdr.size.width[1] {
                                     if ::core::intrinsics::atomic_load_seqcst(
                                         &mut (*f).task_thread.error as *mut atomic_int,
                                     ) == 0
diff --git a/tests/seek_stress.rs b/tests/seek_stress.rs
index fd1b63426..02d9a4eee 100644
--- a/tests/seek_stress.rs
+++ b/tests/seek_stress.rs
@@ -47,7 +47,6 @@ use rav1d::include::dav1d::dav1d::Dav1dSettings;
 use rav1d::include::dav1d::dav1d::DAV1D_DECODEFRAMETYPE_ALL;
 use rav1d::include::dav1d::dav1d::DAV1D_INLOOPFILTER_NONE;
 use rav1d::include::dav1d::headers::Dav1dColorPrimaries;
-use rav1d::include::dav1d::headers::Dav1dFrameHeader;
 use rav1d::include::dav1d::headers::Dav1dSequenceHeader;
 use rav1d::include::dav1d::headers::Dav1dSequenceHeaderOperatingParameterInfo;
 use rav1d::include::dav1d::headers::Dav1dSequenceHeaderOperatingPoint;
@@ -146,8 +145,8 @@ unsafe fn decode_rand(
 ) -> c_int {
     let mut res = 0;
     let mut p: Dav1dPicture = Dav1dPicture {
-        seq_hdr: 0 as *mut Dav1dSequenceHeader,
-        frame_hdr: 0 as *mut Dav1dFrameHeader,
+        seq_hdr: None,
+        frame_hdr: None,
         data: [0 as *mut c_void; 3],
         stride: [0; 2],
         p: Dav1dPictureParameters {
@@ -170,8 +169,8 @@ unsafe fn decode_rand(
         mastering_display: None,
         itut_t35: None,
         reserved: [0; 4],
-        frame_hdr_ref: 0 as *mut Dav1dRef,
-        seq_hdr_ref: 0 as *mut Dav1dRef,
+        frame_hdr_ref: None,
+        seq_hdr_ref: None,
         content_light_ref: None,
         mastering_display_ref: None,
         itut_t35_ref: None,
@@ -201,8 +200,8 @@ unsafe fn decode_all(
 ) -> c_int {
     let mut res: c_int;
     let mut p: Dav1dPicture = Dav1dPicture {
-        seq_hdr: 0 as *mut Dav1dSequenceHeader,
-        frame_hdr: 0 as *mut Dav1dFrameHeader,
+        seq_hdr: None,
+        frame_hdr: None,
         data: [0 as *mut c_void; 3],
         stride: [0; 2],
         p: Dav1dPictureParameters {
@@ -225,8 +224,8 @@ unsafe fn decode_all(
         mastering_display: None,
         itut_t35: None,
         reserved: [0; 4],
-        frame_hdr_ref: 0 as *mut Dav1dRef,
-        seq_hdr_ref: 0 as *mut Dav1dRef,
+        frame_hdr_ref: None,
+        seq_hdr_ref: None,
         content_light_ref: None,
         mastering_display_ref: None,
         itut_t35_ref: None,
diff --git a/tools/dav1d.rs b/tools/dav1d.rs
index b03d2df85..7988c8a19 100644
--- a/tools/dav1d.rs
+++ b/tools/dav1d.rs
@@ -64,7 +64,6 @@ use rav1d::include::dav1d::dav1d::Dav1dSettings;
 use rav1d::include::dav1d::dav1d::DAV1D_DECODEFRAMETYPE_ALL;
 use rav1d::include::dav1d::dav1d::DAV1D_INLOOPFILTER_NONE;
 use rav1d::include::dav1d::headers::Dav1dColorPrimaries;
-use rav1d::include::dav1d::headers::Dav1dFrameHeader;
 use rav1d::include::dav1d::headers::Dav1dSequenceHeader;
 use rav1d::include::dav1d::headers::Dav1dSequenceHeaderOperatingParameterInfo;
 use rav1d::include::dav1d::headers::Dav1dSequenceHeaderOperatingPoint;
@@ -303,8 +302,8 @@ unsafe fn main_0(argc: c_int, argv: *const *mut c_char) -> c_int {
     let mut in_0: *mut DemuxerContext = 0 as *mut DemuxerContext;
     let mut out: *mut MuxerContext = 0 as *mut MuxerContext;
     let mut p: Dav1dPicture = Dav1dPicture {
-        seq_hdr: 0 as *mut Dav1dSequenceHeader,
-        frame_hdr: 0 as *mut Dav1dFrameHeader,
+        seq_hdr: None,
+        frame_hdr: None,
         data: [0 as *mut c_void; 3],
         stride: [0; 2],
         p: Dav1dPictureParameters {
@@ -327,8 +326,8 @@ unsafe fn main_0(argc: c_int, argv: *const *mut c_char) -> c_int {
         mastering_display: None,
         itut_t35: None,
         reserved: [0; 4],
-        frame_hdr_ref: 0 as *mut Dav1dRef,
-        seq_hdr_ref: 0 as *mut Dav1dRef,
+        frame_hdr_ref: None,
+        seq_hdr_ref: None,
         content_light_ref: None,
         mastering_display_ref: None,
         itut_t35_ref: None,
diff --git a/tools/output/y4m2.rs b/tools/output/y4m2.rs
index 92d16d6e3..abc30fdfd 100644
--- a/tools/output/y4m2.rs
+++ b/tools/output/y4m2.rs
@@ -101,21 +101,23 @@ unsafe fn write_header(c: *mut Y4m2OutputContext, p: *const Dav1dPicture) -> c_i
         b"420mpeg2\0" as *const u8 as *const c_char,
         b"420\0" as *const u8 as *const c_char,
     ];
+    let seq_hdr = (*p).seq_hdr.unwrap().as_ref();
+    let frame_hdr = (*p).frame_hdr.unwrap().as_ref();
     let ss_name: *const c_char =
         if (*p).p.layout as c_uint == DAV1D_PIXEL_LAYOUT_I420 as c_int as c_uint && (*p).p.bpc == 8
         {
-            chr_names_8bpc_i420[(if (*(*p).seq_hdr).chr as c_uint > 2 as c_uint {
+            chr_names_8bpc_i420[(if seq_hdr.chr as c_uint > 2 as c_uint {
                 DAV1D_CHR_UNKNOWN as c_int as c_uint
             } else {
-                (*(*p).seq_hdr).chr as c_uint
+                seq_hdr.chr as c_uint
             }) as usize]
         } else {
-            ss_names[(*p).p.layout as usize][(*(*p).seq_hdr).hbd as usize]
+            ss_names[(*p).p.layout as usize][seq_hdr.hbd as usize]
         };
     let fw: c_uint = (*p).p.w as c_uint;
     let fh: c_uint = (*p).p.h as c_uint;
-    let mut aw: u64 = (fh as u64).wrapping_mul((*(*p).frame_hdr).render_width as u64);
-    let mut ah: u64 = (fw as u64).wrapping_mul((*(*p).frame_hdr).render_height as u64);
+    let mut aw: u64 = (fh as u64).wrapping_mul(frame_hdr.render_width as u64);
+    let mut ah: u64 = (fw as u64).wrapping_mul(frame_hdr.render_height as u64);
     let mut gcd: u64 = ah;
     let mut a: u64 = aw;
     let mut b: u64;