Question regarding residual luma and codedBlockPatternLuma
I am implementing an H.264/AVC decoder. I have purchased the necessary specifications (ISO_IEC_14496-10_2022(en)) for this purpose.
According to specification, the if-query in ResidualLuma should be as follows: if CodedBlockPatternLuma & ( 1 << i8x8 ).
In this case, an exception is thrown because codedBlockPatternLuma is a nullable integer in my implementation. It is only initialized in the Decode_MacroblockLayer function if
If mbPartPredMode <> MbPartPredModes.Intra_16x16 Then
If isCabac Then
coded_block_pattern = cabacDecoder.DecodeCoded_block_pattern()
Else
Dim ueVal As UInteger = ByteTools.DecodeExpGolomb(nalUnit, absoluteBitPos, AVC_bitstream)
coded_block_pattern = CodedBlockPatternMapping(ueVal, chromaArrayType, mbTyp)
End If
CodedBlockPatternLuma = GetCodedBlockPatternLumaFromMbType(mbTyp, coded_block_pattern.Value)
.
.
.
because coded_block_patthern is set here.
What I've tried
- Based on my understanding of the specifications, I could also do the following in ResidualLuma:
Dim hasLumaResidual As Boolean
If mbPartPredMode = MbPartPredModes.Intra_16x16 Then
' CBP does not apply; AC blocks are always decoded syntactically
hasLumaResidual = True
Else
hasLumaResidual = (codedBlockPatternLuma.Value And (1 << i8x8)) <> 0
End If
If hasLumaResidual Then
If mbPartPredMode = MbPartPredModes.Intra_16x16 Then
Intra16x16ACLevel(blkIdx) = DecodeResidualBlock_CAVLC
.
.
.
In that case, the problem is that it crashes a few functions later. I know that my bit twiddling functions work, so they can't be wrong. The bitstream is in the wrong place because I've read too much/too little somewhere.
- If I wrote
If codedBlockPatternLuma.HasValue AndAlso (codedBlockPatternLuma.Value And (1 << i8x8)) <> 0 Then(does not run into the if block), the program crashes a few functions later (but another function), too.
The macroblock type is: I_16x16_0_0_0.
Code excerpts
I can't provide you with all the code because I'm in the middle of the decoder. I also cannot show you the bitstream because the error occurs in the middle of the byte array. I can only show you excerpts here, which won't compile on their own, but the logic is inspectable.
Residual Luma:
Private Function Decode_Residual_Luma(startIdx As Integer,
endIdx As Integer,
entropy_coding_mode_flag As Boolean,
mbPartPredMode As MbPartPredModes,
transform_size_8x8_flag As Boolean?,
codedBlockPatternLuma As Integer?,
ByRef AVC_Bitstream As BitStream,
nalUnit() As Byte,
ByRef absoluteBitPos As UInteger,
chromaArrayType As Byte,
currMbAddr As Integer,
previousMacroblockLayers As List(Of MacroblockLayer),
picWidthInMbs As UInt32) As Residual_Luma
Dim Intra16x16DCLevel As Integer() = Nothing
If startIdx = 0 AndAlso mbPartPredMode = MbPartPredModes.Intra_16x16 Then
Dim blockA As NeighborBlockInfo = GetLeftLumaBlockInfo(currMbAddr, -1, previousMacroblockLayers, CInt(picWidthInMbs))
Dim blockB As NeighborBlockInfo = GetTopLumaBlockInfo(currMbAddr, -1, previousMacroblockLayers, CInt(picWidthInMbs))
Dim nC As Int16 = Derive_nC(blockA, blockB)
Intra16x16DCLevel = DecodeResidualBlock_CAVLC(0,
15,
16,
AVC_Bitstream,
nalUnit,
absoluteBitPos,
chromaArrayType,
False,
nC).CoeffLevel
End If
Dim Intra16x16ACLevel As Integer()() = New Integer(15)() {}
Dim LumaLevel4x4 As Integer()() = New Integer(15)() {}
Dim LumaLevel8x8 As Integer()() = New Integer(3)() {}
For i8x8 As Integer = 0 To 3
Dim use8x8 As Boolean = transform_size_8x8_flag.HasValue AndAlso transform_size_8x8_flag.Value
If Not use8x8 OrElse Not entropy_coding_mode_flag Then
For i4x4 As Integer = 0 To 3
Dim blkIdx As Integer = i8x8 * 4 + i4x4
If (codedBlockPatternLuma.Value And (1 << i8x8)) <> 0 Then
If mbPartPredMode = MbPartPredModes.Intra_16x16 Then
Dim blockA As NeighborBlockInfo = GetLeftLumaBlockInfo(currMbAddr, blkIdx, previousMacroblockLayers, CInt(picWidthInMbs))
Dim blockB As NeighborBlockInfo = GetTopLumaBlockInfo(currMbAddr, blkIdx, previousMacroblockLayers, CInt(picWidthInMbs))
Dim nC As Int16 = Derive_nC(blockA, blockB)
Intra16x16ACLevel(blkIdx) = DecodeResidualBlock_CAVLC(Math.Max(0, startIdx - 1),
endIdx - 1,
15,
AVC_Bitstream,
nalUnit,
absoluteBitPos,
chromaArrayType,
False,
nC).CoeffLevel
Else
Dim blockA As NeighborBlockInfo = GetLeftLumaBlockInfo(currMbAddr, blkIdx, previousMacroblockLayers, CInt(picWidthInMbs))
Dim blockB As NeighborBlockInfo = GetTopLumaBlockInfo(currMbAddr, blkIdx, previousMacroblockLayers, CInt(picWidthInMbs))
Dim nC As Int16 = Derive_nC(blockA, blockB)
LumaLevel4x4(blkIdx) = DecodeResidualBlock_CAVLC(startIdx,
endIdx,
16,
AVC_Bitstream,
nalUnit,
absoluteBitPos,
chromaArrayType,
False,
nC).CoeffLevel
End If
Else
If mbPartPredMode = MbPartPredModes.Intra_16x16 Then
Intra16x16ACLevel(blkIdx) = New Integer(14) {}
Else
LumaLevel4x4(blkIdx) = New Integer(15) {}
End If
End If
If Not entropy_coding_mode_flag AndAlso use8x8 Then
If LumaLevel8x8(i8x8) Is Nothing Then
LumaLevel8x8(i8x8) = New Integer(63) {}
End If
For i As Integer = 0 To 15 Step 1
Dim dstIdx As Integer = 4 * i + i4x4
LumaLevel8x8(i8x8)(dstIdx) = LumaLevel4x4(blkIdx)(i)
Next
End If
Next
ElseIf (codedBlockPatternLuma.Value And (1 << i8x8)) <> 0 AndAlso Not entropy_coding_mode_flag Then
LumaLevel8x8(i8x8) = DecodeResidualBlock_CAVLC(4 * startIdx,
4 * endIdx + 3,
64,
AVC_Bitstream,
nalUnit,
absoluteBitPos,
chromaArrayType,
False,
0S).CoeffLevel
Else
LumaLevel8x8(i8x8) = New Integer(63) {}
End If
Next
Return New Residual_Luma(Intra16x16DCLevel, Intra16x16ACLevel, LumaLevel4x4, LumaLevel8x8)
End Function
Macroblock:
Private Function Decode_MacroblockLayer(ByRef AVC_bitstream As BitStream,
ByRef absoluteBitPos As UInteger,
isCabac As Boolean,
nalUnit As Byte(),
sliceType As String,
bit_depth_luma As UInt16,
bit_depth_chroma As UInt16,
MbWidthC As UInt32,
MbHeightC As UInt32,
transform_8x8_mode_flag As Byte,
direct_8x8_inference_flag As Boolean,
chromaArrayType As Byte,
num_ref_idx_l0_active_minus1 As UInt32,
num_ref_idx_l1_active_minus1 As UInt32,
mb_field_decoding_flag As Boolean?,
field_pic_flag As Boolean,
subWidthC As UInt32?,
subHeightC As UInt32?,
currMbAddr As Integer,
picWidthInMbs As UInt32,
previousMacroblocklayers As List(Of MacroblockLayer)) As MacroblockLayer
Dim mbTyp As AvcDecoder.MbTypes
If isCabac Then
Dim mbTypeFromCabacDecoder As Cabac.CabacDecoder.MbTypes = Me.cabacDecoder.Decode_MbType(sliceType)
mbTyp = CType([Enum].Parse(GetType(AvcDecoder.MbTypes), mbTypeFromCabacDecoder.ToString()), AvcDecoder.MbTypes)
Else
Dim value As UInteger = ByteTools.DecodeExpGolomb(nalUnit, absoluteBitPos, AVC_bitstream)
mbTyp = MapUnsignedValueToMbType(value, sliceType)
#If DEBUG Then
Diagnostics.Debug.WriteLine($"Macroblock: {currMbAddr.ToString()}. Macroblock type: {mbTyp.ToString()}")
#End If
End If
Dim pcm_sample_luma As UInt16() = Nothing
Dim pcm_sample_chroma As UInt16() = Nothing
Dim transform_size_8x8_flag As Boolean = False ' default according to documentation
Dim mb_qp_delta As Int32 = 0 ' default according to documentation
Dim CodedBlockPatternLuma As Integer?
Dim CodedBlockPatternChroma As Integer?
Dim coded_block_pattern As UInt16?
Dim residual As ResidualData = Nothing
Dim subMacroblockPrediction As SubMacroblockPrediction = Nothing
Dim macroblockPrediction As MacroblockPrediction = Nothing
If mbTyp = MbTypes.I_PCM Then
While Not Byte_Aligned(AVC_bitstream)
Dim pcm_alignment_zero_bit As Byte = ByteTools.GetBit(nalUnit(AVC_bitstream.BytePos), AVC_bitstream.BitPos, AVC_bitstream, absoluteBitPos)
End While
pcm_sample_luma = New UInt16(255) {}
For i As Integer = 0 To 255 Step 1
pcm_sample_luma(i) = CUShort(ByteTools.ReadBits(nalUnit, absoluteBitPos, bit_depth_luma, AVC_bitstream))
Next
pcm_sample_chroma = New UInt16(2 * CInt(MbWidthC) * CInt(MbHeightC) - 1) {}
For i As Integer = 0 To 2 * CInt(MbWidthC) * CInt(MbHeightC) - 1 Step 1
pcm_sample_chroma(i) = CUShort(ByteTools.ReadBits(nalUnit, absoluteBitPos, bit_depth_chroma, AVC_bitstream))
Next
Return New MacroblockLayer(mbTyp, pcm_sample_luma, pcm_sample_chroma, False, 0, Nothing, Nothing, Nothing, Nothing, Nothing, Nothing, mb_field_decoding_flag)
Else
Dim noSubMbPartSizeLessThan8x8Flag As Boolean = True
Dim mbPartPredMode As MbPartPredModes = MapMbPartPredModeFromMbType(mbTyp, False)
Dim NumMbPart As Integer = MapMbTypeToInteger(mbTyp)
If mbTyp <> MbTypes.I_NxN AndAlso mbPartPredMode <> MbPartPredModes.Intra_16x16 AndAlso NumMbPart = 4 Then
subMacroblockPrediction = Decode_SubMacroblockPrediction(mbTyp,
num_ref_idx_l0_active_minus1,
num_ref_idx_l1_active_minus1,
mb_field_decoding_flag,
field_pic_flag,
isCabac,
nalUnit,
absoluteBitPos,
AVC_bitstream,
sliceType)
Dim sub_mb_type As SubMbTypes() = subMacroblockPrediction.Sub_mb_types
For mbPartIdx As Integer = 0 To 3
If sub_mb_type(mbPartIdx) <> SubMbTypes.B_Direct_8x8 Then
If MapSubMbTypeToInt(sub_mb_type(mbPartIdx)) > 1 Then
noSubMbPartSizeLessThan8x8Flag = False
End If
ElseIf Not direct_8x8_inference_flag Then
noSubMbPartSizeLessThan8x8Flag = False
End If
Next
Else
If transform_8x8_mode_flag = CByte(1) AndAlso mbTyp = MbTypes.I_NxN Then
If isCabac Then
transform_size_8x8_flag = cabacDecoder.Decode_transform_size_8x8_flag() > 0
Else
transform_size_8x8_flag = ByteTools.GetBitB(nalUnit(AVC_bitstream.BytePos), AVC_bitstream.BitPos, AVC_bitstream, absoluteBitPos)
End If
End If
macroblockPrediction = Decode_MacroblockPrediction(mbTyp,
chromaArrayType,
NumMbPart,
num_ref_idx_l0_active_minus1,
num_ref_idx_l1_active_minus1,
mb_field_decoding_flag,
field_pic_flag,
isCabac,
nalUnit,
absoluteBitPos,
AVC_bitstream)
End If
If mbPartPredMode <> MbPartPredModes.Intra_16x16 Then
If isCabac Then
coded_block_pattern = cabacDecoder.DecodeCoded_block_pattern()
Else
Dim ueVal As UInteger = ByteTools.DecodeExpGolomb(nalUnit, absoluteBitPos, AVC_bitstream)
coded_block_pattern = CodedBlockPatternMapping(ueVal, chromaArrayType, mbTyp)
End If
CodedBlockPatternLuma = GetCodedBlockPatternLumaFromMbType(mbTyp, coded_block_pattern.Value)
If CodedBlockPatternLuma.Value > 0 AndAlso
transform_8x8_mode_flag = 1 AndAlso
mbTyp <> MbTypes.I_NxN AndAlso
noSubMbPartSizeLessThan8x8Flag AndAlso
(mbTyp <> MbTypes.B_Direct_16x16 OrElse direct_8x8_inference_flag) Then
If isCabac Then
transform_size_8x8_flag = cabacDecoder.Decode_transform_size_8x8_flag() > 0
Else
transform_size_8x8_flag = ByteTools.GetBitB(nalUnit(AVC_bitstream.BytePos), AVC_bitstream.BitPos, AVC_bitstream, absoluteBitPos)
End If
End If
Else
coded_block_pattern = 0UI
End If
CodedBlockPatternChroma = GetCodedBlockPatternChromaFromMbType(mbTyp, coded_block_pattern.Value)
If CodedBlockPatternLuma.HasValue AndAlso CodedBlockPatternLuma.Value > 0 OrElse
CodedBlockPatternChroma.Value > 0 OrElse
mbPartPredMode = MbPartPredModes.Intra_16x16 Then
If isCabac Then
mb_qp_delta = cabacDecoder.Decode_mb_qp_delta()
Else
mb_qp_delta = ByteTools.DecodeExpGolombSigned(nalUnit, absoluteBitPos)
AVC_bitstream.UpdateBitPosition(absoluteBitPos)
End If
If isCabac Then
'residual = DecodeResidualBlock_CABAC()
Else
residual = DecodeResidual_CAVLC(0,
15,
isCabac,
mbTyp,
mbPartPredMode,
transform_size_8x8_flag,
CodedBlockPatternLuma,
CodedBlockPatternChroma,
chromaArrayType,
subWidthC,
subHeightC,
AVC_bitstream,
nalUnit,
absoluteBitPos,
currMbAddr,
previousMacroblocklayers,
picWidthInMbs)
#If DEBUG Then
Diagnostics.Debug.WriteLine($"After Residual (last thing done in macroblock): byte={AVC_bitstream.BytePos}, bit={AVC_bitstream.BitPos}")
#End If
End If
End If
End If
Return New MacroblockLayer(mbTyp,
Nothing,
Nothing,
transform_size_8x8_flag,
mb_qp_delta,
CodedBlockPatternLuma,
CodedBlockPatternChroma,
coded_block_pattern,
residual,
macroblockPrediction,
subMacroblockPrediction,
mb_field_decoding_flag)
End Function