Qualcomm H.264 Media Foundation encoder occasionally returns a zero-length output sample

Question

Qualcomm H.264 Media Foundation encoder occasionally returns a zero-length output sample

chunlai xie 0

I use the Qualcomm H.264 hardware encoder through Media Foundation on a Windows ARM64 device. The encoder is an asynchronous MFT ({7790EE16-08E3-426D-AADA-F96774308EA1}).

The encoder configuration is as follows.

#include <codecapi.h>
#include <mfapi.h>
#include <mftransform.h>
#include <wrl/client.h>
using Microsoft::WRL::ComPtr;
HRESULT SetU32(ICodecAPI* api, const GUID& property, ULONG value) {
    VARIANT v = {};
    v.vt = VT_UI4;
    v.ulVal = value;
    return api->SetValue(&property, &v);
}
HRESULT SetBool(ICodecAPI* api, const GUID& property, bool value) {
    VARIANT v = {};
    v.vt = VT_BOOL;
    v.boolVal = value ? VARIANT_TRUE : VARIANT_FALSE;
    return api->SetValue(&property, &v);
}
HRESULT ConfigureMft(IMFTransform* mft,
                     UINT32 maxBitrateKbps,
                     UINT32 h264Profile,
                     IMFDXGIDeviceManager* d3dManager) {
    HRESULT hr;
    ComPtr<IMFAttributes> attributes;
    ComPtr<ICodecAPI> codecApi;
    ComPtr<IMFMediaType> inputType;
    ComPtr<IMFMediaType> outputType;
#define CHECK_HR(expr)        \
    do {                      \
        hr = (expr);          \
        if (FAILED(hr))       \
            return hr;        \
    } while (false)
    CHECK_HR(mft->GetAttributes(&attributes));
    CHECK_HR(attributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE));
    CHECK_HR(attributes->SetUINT32(CODECAPI_AVLowLatencyMode, TRUE));
    CHECK_HR(mft->QueryInterface(IID_PPV_ARGS(&codecApi)));
    // Encoder settings from the build that produced the logs.
    // 60 fps × 10 seconds = 600 pictures per GOP.
    CHECK_HR(SetU32(codecApi.Get(), CODECAPI_AVEncMPVGOPSize, 600));
    CHECK_HR(SetU32(codecApi.Get(),
                    CODECAPI_AVEncMPVDefaultBPictureCount, 0));
    CHECK_HR(SetBool(codecApi.Get(),
                     CODECAPI_AVEncCommonLowLatency, true));
    CHECK_HR(SetU32(codecApi.Get(), CODECAPI_AVEncVideoMinQP, 10));
    CHECK_HR(SetU32(codecApi.Get(), CODECAPI_AVEncVideoMaxQP, 38));
    CHECK_HR(SetU32(codecApi.Get(),
                    CODECAPI_AVEncCommonRateControlMode,
                    eAVEncCommonRateControlMode_CBR));
    CHECK_HR(SetU32(codecApi.Get(),
                    CODECAPI_AVEncCommonMeanBitRate,
                    2'714'000));  // bits per second
    CHECK_HR(SetU32(codecApi.Get(),
                    CODECAPI_AVEncVideoMaxNumRefFrame, 1));
    CHECK_HR(MFCreateMediaType(&outputType));
    CHECK_HR(outputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
    CHECK_HR(outputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264));
    CHECK_HR(outputType->SetUINT32(MF_MT_MPEG2_PROFILE, h264Profile));
    CHECK_HR(MFSetAttributeSize(outputType.Get(),
                                MF_MT_FRAME_SIZE, 2304, 1440));
    CHECK_HR(MFSetAttributeRatio(outputType.Get(),
                                 MF_MT_FRAME_RATE, 60, 1));
    CHECK_HR(outputType->SetUINT32(
        MF_MT_AVG_BITRATE, maxBitrateKbps * 1000));
    CHECK_HR(MFCreateMediaType(&inputType));
    CHECK_HR(inputType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video));
    CHECK_HR(inputType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12));
    CHECK_HR(MFSetAttributeSize(inputType.Get(),
                                MF_MT_FRAME_SIZE, 2304, 1440));
    CHECK_HR(MFSetAttributeRatio(inputType.Get(),
                                 MF_MT_FRAME_RATE, 60, 1));
    CHECK_HR(mft->SetOutputType(0, outputType.Get(), 0));
    CHECK_HR(mft->SetInputType(0, inputType.Get(), 0));
    if (d3dManager) {
        CHECK_HR(mft->ProcessMessage(
            MFT_MESSAGE_SET_D3D_MANAGER,
            reinterpret_cast<ULONG_PTR>(d3dManager)));
    }
    CHECK_HR(mft->ProcessMessage(
        MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0));
    CHECK_HR(mft->ProcessMessage(
        MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0));
#undef CHECK_HR
    return S_OK;
}
// Later, when the target changes to 10000 kbps:
HRESULT UpdateTargetBitrate(IMFTransform* mft) {
    ComPtr<ICodecAPI> codecApi;
    HRESULT hr = mft->QueryInterface(IID_PPV_ARGS(&codecApi));
    if (FAILED(hr))
        return hr;
    return SetU32(codecApi.Get(),
                  CODECAPI_AVEncCommonMeanBitRate,
                  10'000'000);  // bits per second
}

The input is NV12, approximately 2304 × 1440 at up to 60 fps. I submit a sample when the MFT sends METransformNeedInput, and call ProcessOutput when it sends METransformHaveOutput. For D3D11 input, I set the input buffer's current length before calling ProcessInput.

The relevant output handling is:

// Called after receiving METransformHaveOutput.
// The MFT provides its own output samples.
void ReadEncodedOutput(IMFTransform* encoder) {
    MFT_OUTPUT_DATA_BUFFER output = {};
    DWORD outputStatus = 0;
    HRESULT hr = encoder->ProcessOutput(0, 1, &output, &outputStatus);
    Microsoft::WRL::ComPtr<IMFSample> sample;
    sample.Attach(output.pSample);
    Microsoft::WRL::ComPtr<IMFCollection> events;
    events.Attach(output.pEvents);
    if (FAILED(hr) || !sample) {
        printf("ProcessOutput: hr=0x%08X, sample=%p\n",
               static_cast<unsigned>(hr), sample.Get());
        return;
    }
    Microsoft::WRL::ComPtr<IMFMediaBuffer> buffer;
    hr = sample->ConvertToContiguousBuffer(&buffer);
    if (FAILED(hr)) {
        printf("ConvertToContiguousBuffer: 0x%08X\n",
               static_cast<unsigned>(hr));
        return;
    }
    DWORD length = 0;
    hr = buffer->GetCurrentLength(&length);
    LONGLONG pts = 0;
    sample->GetSampleTime(&pts);
    printf("ProcessOutput=0x%08X, length=%lu, pts=%lld, "
           "streamStatus=0x%08lX, outputStatus=0x%08lX\n",
           static_cast<unsigned>(hr),
           static_cast<unsigned long>(length),
           pts,
           static_cast<unsigned long>(output.dwStatus),
           static_cast<unsigned long>(outputStatus));
    if (SUCCEEDED(hr) && length == 0) {
        // No H.264 bytes are available to send for this sample.
        return;
    }
    // Consume the encoded H.264 data.
}

Most output samples contain H.264 data, but occasionally ProcessOutput succeeds and the returned sample's buffer has a current length of zero. Encoding continues and later samples contain data again. Our application skips these empty samples.

Questions:

Is a successful ProcessOutput with a zero-length sample expected behavior for this Qualcomm H.264 MFT?
Does it indicate an intentionally dropped frame, a rate-control decision, or a driver problem?
Is there a recommended way to configure the encoder or recover without losing that frame? Are there Media Foundation traces or sample attributes that would distinguish these cases?

0 comments

1 answer

Your answer

Answer 1

A zero-length sample is valid only if that successful ProcessOutput call also returned a non-empty in-band event collection; if it returned neither H.264 data nor events, the Qualcomm MFT is not following the documented IMFTransform::ProcessOutput contract.

Handle the result by payload, events, and status

For every METransformHaveOutput, call ProcessOutput exactly once. An asynchronous MFT requires this one-to-one event/call relationship and should not return MF_E_TRANSFORM_NEED_MORE_INPUT; calling at another time can return E_UNEXPECTED.^1^

On S_OK:

Process output.pEvents before the sample.
Inspect output.dwStatus and outputStatus.
Consume the sample only when its total/current length is nonzero.
If MFT_OUTPUT_DATA_BUFFER_INCOMPLETE is set, more output remains, but for an asynchronous MFT wait for the corresponding METransformHaveOutput rather than adding an unprompted synchronous-style call.
Log an anomaly when both payload and the event collection are empty.

Media Foundation explicitly permits S_OK with zero output samples when in-band events were returned. If no stream has data and no events exist, the normal synchronous result is MF_E_TRANSFORM_NEED_MORE_INPUT.^2^

Your current code releases pEvents without examining its contents. Record the collection count first:

DWORD eventCount = 0;
if (output.pEvents)
    output.pEvents->GetElementCount(&eventCount);

DWORD totalLength = 0;
if (output.pSample)
    output.pSample->GetTotalLength(&totalLength);

printf("hr=%08X bytes=%lu events=%lu stream=%08lX global=%08lX\n",
       static_cast<unsigned>(hr),
       static_cast<unsigned long>(totalLength),
       static_cast<unsigned long>(eventCount),
       static_cast<unsigned long>(output.dwStatus),
       static_cast<unsigned long>(outputStatus));

if (SUCCEEDED(hr) && totalLength == 0 && eventCount == 0)
    LogEncoderContractViolation();

Also decode dwStatus, particularly NO_SAMPLE, INCOMPLETE, FORMAT_CHANGE, and STREAM_END. NO_SAMPLE means that stream has no sample ready; INCOMPLETE means more output can be generated.^3^

It does not, by itself, prove a dropped frame

A zero-byte result alone does not distinguish:

an intentional duplicate-frame suppression;
a frame dropped because of encoder or buffer constraints;
an event-only output;
an implementation/driver defect.

The documented output attributes MFSampleExtension_VideoEncodePictureType and MFSampleExtension_VideoEncodeQP describe the picture type and QP of an encoded sample.^4^^5^ They are useful on non-empty samples but are not documented as drop-reason indicators. MFSampleExtension_Discontinuity identifies the first sample after a stream gap, not why an encoder omitted data.^6^

Microsoft documents zero-byte and dropped-frame statistics for codecs exposing the corresponding legacy property interfaces. Querying them is useful only if this Qualcomm MFT actually exposes those properties:

MFPKEY_ZEROBYTEFRAMES: duplicate frames skipped.
MFPKEY_CODEDFRAMES versus MFPKEY_TOTALFRAMES: difference indicates dropped frames.^7^

Do not assume those Windows Media codec properties or MFPKEY_PRODUCEDUMMYFRAMES are implemented by this Qualcomm H.264 MFT; no supplied Qualcomm documentation establishes that support.

Configuration and recovery

There is no documented Media Foundation operation that reconstructs the missing H.264 access unit after the MFT has emitted an empty result. Continue processing, but preserve the input/output timestamps in the diagnostic log.

If the MFT reports CODECAPI_AVLowLatencyMode as supported through ICodecAPI::IsSupported, test enabling it before streaming. Low-latency mode expects one input sample to produce one output sample without reordering delay, although it can reduce encoding quality.^8^ This is a diagnostic/configuration option, not evidence that empty outputs are expected or a guaranteed recovery mechanism.

Also compare the requested workload against the MFT’s read-only MF_VIDEO_MAX_MB_PER_SEC attribute, if present. It reports the hardware encoder’s maximum macroblock processing rate and is affected by level, quality-versus-speed, and B-frame settings.^9^ If the requested rate exceeds that capability, reduce frame rate/resolution or select faster supported settings before treating the result as a driver defect.

Capture evidence for the OEM

Run MFTrace around the process:

mftrace.exe -a YourEncoderApp.exe -o mftrace.txt

Search for:

CMFTransformDetours::ProcessInput
CMFTransformDetours::ProcessOutput
METransformNeedInput
ProcessOutputs available
0xc00d

MFTrace can attach to an existing process and records Media Foundation calls; ETW/WPP provider tracing requires a configuration file.^10^

If a trace shows one ProcessOutput per METransformHaveOutput, S_OK, zero bytes, zero in-band events, no format/stream status, and later normal output, collect the device model, Windows build, Qualcomm display/media driver version, negotiated media types, timestamps, and a minimal reproduction for the device OEM or Qualcomm. That evidence identifies a Qualcomm MFT/driver contract violation rather than an application-side empty-sample handling issue.

References