/*
 * Copyright (C) 2026 Linux Studio Plugins Project <https://lsp-plug.in/>
 *           (C) 2026 Vladimir Sadovnikov <sadko4u@gmail.com>
 *
 * This file is part of lsp-dsp-lib
 * Created on: 31 мар. 2020 г.
 *
 * lsp-dsp-lib is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * any later version.
 *
 * lsp-dsp-lib is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with lsp-dsp-lib. If not, see <https://www.gnu.org/licenses/>.
 */

#ifndef PRIVATE_DSP_ARCH_X86_SSE_FILTERS_STATIC_H_
#define PRIVATE_DSP_ARCH_X86_SSE_FILTERS_STATIC_H_

#ifndef PRIVATE_DSP_ARCH_X86_SSE_IMPL
    #error "This header should not be included directly"
#endif /* PRIVATE_DSP_ARCH_X86_SSE_IMPL */

#include <private/dsp/arch/x86/sse/filters/common.h>

namespace lsp
{
    namespace sse
    {
        void biquad_process_x1(float *dst, const float *src, float *d, size_t count, const dsp::biquad_x1_t *f)
        {
            IF_ARCH_X86(size_t off);

            ARCH_X86_ASM
            (
                // Check count
                __ASM_EMIT32("cmpl      $0, %[count]")
                __ASM_EMIT64("test      %[count], %[count]")
                __ASM_EMIT("jz          2f")

                // Load permanent data
                __ASM_EMIT("movss       0x00(%[d]), %%xmm6")                    // xmm6 = d0
                __ASM_EMIT("xor         %[off], %[off]")
                __ASM_EMIT("movss       0x04(%[d]), %%xmm7")                    // xmm7 = d1

                // Start loop
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src], %[off], 4), %%xmm0")           // xmm0 = s ? ? ?
                __ASM_EMIT("movss       0x00(%[f]), %%xmm1")                    // xmm1 = a0
                __ASM_EMIT("movss       0x04(%[f]), %%xmm2")                    // xmm2 = a1
                __ASM_EMIT("mulss       %%xmm0, %%xmm1")                        // xmm1 = a0*s
                __ASM_EMIT("movss       0x0c(%[f]), %%xmm3")                    // xmm3 = b1
                __ASM_EMIT("mulss       %%xmm0, %%xmm2")                        // xmm2 = a1*s
                __ASM_EMIT("addss       %%xmm6, %%xmm1")                        // xmm1 = s' = a0*s + d0
                __ASM_EMIT("mulss       0x08(%[f]), %%xmm0")                    // xmm0 = a2*s
                __ASM_EMIT("movss       %%xmm1, (%[dst], %[off], 4)")           // *dst = s'
                __ASM_EMIT("movaps      %%xmm7, %%xmm6")                        // xmm6 = d1
                __ASM_EMIT("mulss       %%xmm1, %%xmm3")                        // xmm3 = b1*s'
                __ASM_EMIT("add         $1, %[off]")
                __ASM_EMIT("mulss       0x10(%[f]), %%xmm1")                    // xmm1 = b2*s'
                __ASM_EMIT("addss       %%xmm3, %%xmm2")                        // xmm2 = a1*s + b1*s'
                __ASM_EMIT("addss       %%xmm0, %%xmm1")                        // xmm3 = d1' = a2*s + b2*s'
                __ASM_EMIT("cmp         %[count], %[off]")
                __ASM_EMIT("addss       %%xmm2, %%xmm6")                        // xmm6 = d0' = d1 + a1*s + b1*s'
                __ASM_EMIT("movaps      %%xmm1, %%xmm7")                        // xmm7 = d1'
                __ASM_EMIT("jb          1b")

                // Store the updated buffer state
                __ASM_EMIT("movss       %%xmm6, 0x00(%[d])")
                __ASM_EMIT("movss       %%xmm7, 0x04(%[d])")

                // Exit label
                __ASM_EMIT("2:")

                : [off] "=&r"(off)
                : [dst] "r" (dst), [src] "r" (src),
                  [count] X86_GREG (count),
                  [f] "r" (f), [d] "r" (d)
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm6", "%xmm7"
            );
        }


        void biquad_process_x2(float *dst, const float *src, float *d, size_t count, const dsp::biquad_x2_t *f)
        {
            ARCH_X86_ASM
            (
                // Check count
                __ASM_EMIT("test        %[count], %[count]")
                __ASM_EMIT("jz          4f")

                // Start loop
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0 = s ? ? ?
                __ASM_EMIT("movlps      0x00(%[f]), %%xmm1")                        // xmm1 = a0
                __ASM_EMIT("movlps      0x08(%[f]), %%xmm2")                        // xmm2 = a1
                __ASM_EMIT("mulss       %%xmm0, %%xmm1")                            // xmm1 = a0*s
                __ASM_EMIT("movlps      0x10(%[f]), %%xmm3")                        // xmm3 = a2
                __ASM_EMIT("mulss       %%xmm0, %%xmm2")                            // xmm2 = a1*s
                __ASM_EMIT("movlps      0x18(%[f]), %%xmm4")                        // xmm4 = b1
                __ASM_EMIT("addss       0x00(%[d]), %%xmm1")                        // xmm1 = s' = a0*s + d0
                __ASM_EMIT("mulss       %%xmm0, %%xmm3")                            // xmm3 = a2*s
                __ASM_EMIT("mulss       %%xmm1, %%xmm4")                            // xmm4 = b1*s'
                __ASM_EMIT("movaps      %%xmm1, %%xmm0")                            // xmm0 = s'
                __ASM_EMIT("mulss       0x20(%[f]), %%xmm1")                        // xmm1 = b2*s'
                __ASM_EMIT("shufps      $0xb1, %%xmm0, %%xmm0")                     // shift
                __ASM_EMIT("addss       %%xmm4, %%xmm2")                            // xmm2 = a1*s + b1*s'
                __ASM_EMIT("addss       %%xmm3, %%xmm1")                            // xmm1 = d1' = a2*s + b2*s'
                __ASM_EMIT("addss       0x08(%[d]), %%xmm2")                        // xmm6 = d0' = d1 + a1*s + b1*s'
                __ASM_EMIT("add         $0x04, %[src]")                             // src++
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("movss       %%xmm2, 0x00(%[d])")
                __ASM_EMIT("movss       %%xmm1, 0x08(%[d])")
                __ASM_EMIT("jz          2f")
                // x2 loop
                __ASM_EMIT("movups      0x00(%[d]), %%xmm6")                        // xmm6 = d0 e0 d1 e1
                __ASM_EMIT("xorps       %%xmm7, %%xmm7")                            // xmm7 = 0 0 0 0
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm4")                          // xmm4 = s0
                __ASM_EMIT("movlps      0x00(%[f]), %%xmm1")                        // xmm1 = a0 i0
                __ASM_EMIT("movss       %%xmm4, %%xmm0")                            // xmm0 = s0 s1
                __ASM_EMIT("movups      0x08(%[f]), %%xmm2")                        // xmm2 = a1 i1 a2 i2
                __ASM_EMIT("movlhps     %%xmm0, %%xmm0")                            // xmm0 = s0 s1 s0 s1
                __ASM_EMIT("movups      0x18(%[f]), %%xmm3")                        // xmm3 = b1 j1 b2 j2
                __ASM_EMIT("mulps       %%xmm0, %%xmm2")                            // xmm2 = a1*s0 i1*s1 a2*s0 i2*s1
                __ASM_EMIT("mulps       %%xmm1, %%xmm0")                            // xmm0 = a0*s0 i0*s1
                __ASM_EMIT("addps       %%xmm6, %%xmm0")                            // xmm0 = s0' s1' = d0+a0*s0 e0+i0*s1
                __ASM_EMIT("shufps      $0x0e, %%xmm7, %%xmm6")                     // xmm6 = d1 e1 0 0
                __ASM_EMIT("movlhps     %%xmm0, %%xmm0")                            // xmm0 = s0' s1' s0' s1'
                __ASM_EMIT("addps       %%xmm2, %%xmm6")                            // xmm6 = d1+a1*s0 e1+i1*s1 a2*s0 i2*s1
                __ASM_EMIT("mulps       %%xmm0, %%xmm3")                            // xmm3 = b1*s0' j1*s1' b2*s0' j2*s1'
                __ASM_EMIT("shufps      $0xb1, %%xmm0, %%xmm0")                     // shift
                __ASM_EMIT("addps       %%xmm3, %%xmm6")                            // xmm6 = d0' e0' d1' e1' = d1+a1*s0+b1*s0' e1+i1*s1+j1*s1' a2*s0+b2*s0' i2*s1+j2*s1'
                __ASM_EMIT("movss       %%xmm0, (%[dst])")
                __ASM_EMIT("add         $0x04, %[src]")
                __ASM_EMIT("add         $0x04, %[dst]")
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         1b")
                __ASM_EMIT("movups      %%xmm6, 0x00(%[d])")
                // Last step
                __ASM_EMIT("2:")
                __ASM_EMIT("shufps      $0xb1, %%xmm0, %%xmm0")                     // shift
                __ASM_EMIT("movss       0x04(%[f]), %%xmm1")                        // xmm1 = a0
                __ASM_EMIT("movss       0x0c(%[f]), %%xmm2")                        // xmm2 = a1
                __ASM_EMIT("mulss       %%xmm0, %%xmm1")                            // xmm1 = a0*s
                __ASM_EMIT("movss       0x14(%[f]), %%xmm3")                        // xmm3 = a2
                __ASM_EMIT("mulss       %%xmm0, %%xmm2")                            // xmm2 = a1*s
                __ASM_EMIT("movss       0x1c(%[f]), %%xmm4")                        // xmm4 = b1
                __ASM_EMIT("addss       0x04(%[d]), %%xmm1")                        // xmm1 = s' = a0*s + d0
                __ASM_EMIT("mulss       %%xmm0, %%xmm3")                            // xmm3 = a2*s
                __ASM_EMIT("mulss       %%xmm1, %%xmm4")                            // xmm4 = b1*s'
                __ASM_EMIT("movaps      %%xmm1, %%xmm0")                            // xmm0 = s'
                __ASM_EMIT("mulss       0x24(%[f]), %%xmm1")                        // xmm1 = b2*s'
                __ASM_EMIT("movss       %%xmm0, (%[dst])")
                __ASM_EMIT("addps       %%xmm4, %%xmm2")                            // xmm2 = a1*s + b1*s'
                __ASM_EMIT("addps       %%xmm3, %%xmm1")                            // xmm3 = d1' = a2*s + b2*s'
                __ASM_EMIT("addss       0x0c(%[d]), %%xmm2")                        // xmm6 = d0' = d1 + a1*s + b1*s'
                __ASM_EMIT("movss       %%xmm2, 0x04(%[d])")
                __ASM_EMIT("movss       %%xmm1, 0x0c(%[d])")

                // Exit label
                __ASM_EMIT("4:")

                : [dst] "+r" (dst), [src] "+r" (src)
                : [count] "r" (count),
                  [f] "r" (f), [d] "r" (d)
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        void biquad_process_x4(float *dst, const float *src, float *d, size_t count, const dsp::biquad_x4_t *f)
        {
            IF_ARCH_X86(
                float   MASK[4] __lsp_aligned16;
                size_t  mask;
            );

            ARCH_X86_ASM
            (
                // Check count
                __ASM_EMIT32("cmpl      $0, %[count]")
                __ASM_EMIT64("test      %[count], %[count]")
                __ASM_EMIT("jz          8f")

                // Load delay buffer
                __ASM_EMIT("movups      0x00(%[d]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT("movups      0x10(%[d]), %%xmm7")                        // xmm7     = d1

                // Initialize mask
                // xmm0=tmp, xmm1={s,s2[4]}, xmm2=p1[4], xmm3=p2[4], xmm6=d0[4], xmm7=d1[4]
                __ASM_EMIT("mov         $1, %[mask]")
                __ASM_EMIT("movaps      %[X_MASK], %%xmm0")
                __ASM_EMIT("xorps       %%xmm1, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")

                // Process first 3 steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X4_CORE
                // Shift buffer
                __ASM_EMIT("shufps      $0x90, %%xmm1, %%xmm1")                     // xmm1     = s2[0] s2[0] s2[1] s2[2]
                // Update delay only by mask
                FILTER_X4_PRE_MEMSYNC

                // Repeat loop
                __ASM_EMIT32("decl      %[count]")
                __ASM_EMIT64("dec       %[count]")
                __ASM_EMIT("jz          4f")                                        // jump to completion
                __ASM_EMIT("lea         0x01(,%[mask], 2), %[mask]")                // mask     = (mask << 1) | 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")                           // store mask
                __ASM_EMIT("cmp         $0x0f, %[mask]")
                __ASM_EMIT("jne         1b")

                // 4x filter processing without mask
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("3:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X4_CORE
                FILTER_X4_MEMSYNC

                // Shift buffer and repeat loop
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT32("decl      %[count]")
                __ASM_EMIT64("dec       %[count]")
                __ASM_EMIT("jnz         3b")

                // Prepare last loop
                __ASM_EMIT("4:")
                __ASM_EMIT("movaps      %[MASK], %%xmm0")                           // xmm0     = m[0] m[1] m[2] m[3]
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]

                // Process steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("5:")
                __ASM_EMIT("movaps      %%xmm1, %%xmm2")                            // xmm2     = s
                __ASM_EMIT("movaps      %%xmm1, %%xmm3")                            // xmm3     = s
                FILTER_X4_CORE

                // Shift buffer and store
                __ASM_EMIT("test        $0x8, %[mask]")
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("jz          7f")
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("7:")

                // Update delay only by mask
                FILTER_X4_POST_MEMSYNC

                // Repeat loop
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("jnz         5b")                                        // check that mask is not zero

                // Store delay buffer
                __ASM_EMIT("movups      %%xmm6, 0x00(%[d])")                        // xmm6     = d0
                __ASM_EMIT("movups      %%xmm7, 0x10(%[d])")                        // xmm7     = d1

                // Exit label
                __ASM_EMIT("8:")

                : [dst] "+r" (dst), [src] "+r" (src),
                  [mask] "=&r"(mask),
                  [count] X86_PGREG (count)
                : [f] "r" (f), [d] "r" (d),
                  [X_MASK] "m" (biquad_mask_const),
                  [MASK] "m" (MASK)
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        void biquad_process_x8(float *dst, const float *src, float *d, size_t count, const dsp::biquad_x8_t *f)
        {
            IF_ARCH_X86(
                float   MASK[4] __lsp_aligned16;
                float  *X_F, *X_D;
                size_t  X_COUNT;
                size_t  mask;
            )

            ARCH_X86_ASM
            (
                // Check count
                __ASM_EMIT("test        %[count], %[count]")
                __ASM_EMIT("jz          10f")

                //---------------------------------------------------------------------
                // Cycle 1
                __ASM_EMIT32("mov       %[f], %[X_F]")
                __ASM_EMIT("mov         %[dst], %[X_D]")
                __ASM_EMIT("mov         %[count], %[X_COUNT]")

                // Load delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    0x00(%[f]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT32("movups    0x20(%[f]), %%xmm7")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")
                __ASM_EMIT64("movups    0x00(%[d]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT64("movups    0x20(%[d]), %%xmm7")                        // xmm7     = d1

                // Initialize mask
                // xmm0=tmp, xmm1={s,s2[4]}, xmm2=p1[4], xmm3=p2[4], xmm6=d0[4], xmm7=d1[4]
                __ASM_EMIT("mov         $1, %[mask]")
                __ASM_EMIT("movaps      %[X_MASK], %%xmm0")
                __ASM_EMIT("xorps       %%xmm1, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")

                // Process first 3 steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X8P1_CORE
                // Shift buffer
                __ASM_EMIT("shufps      $0x90, %%xmm1, %%xmm1")                     // xmm1     = s2[0] s2[0] s2[1] s2[2]
                // Update delay only by mask
                FILTER_X4_PRE_MEMSYNC

                // Repeat loop
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jz          4f")                                        // jump to completion
                __ASM_EMIT("lea         0x01(,%[mask], 2), %[mask]")                // mask     = (mask << 1) | 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")                           // store mask
                __ASM_EMIT("cmp         $0x0f, %[mask]")
                __ASM_EMIT("jne         1b")

                // 4x filter processing without mask
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("3:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X8P1_CORE
                FILTER_X4_MEMSYNC

                // Shift buffer and repeat loop
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         3b")

                // Prepare last loop
                __ASM_EMIT("4:")
                __ASM_EMIT("movaps      %[MASK], %%xmm0")                           // xmm0     = m[0] m[1] m[2] m[3]
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]

                // Process steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("5:")
                FILTER_X8P1_CORE

                // Shift buffer and store
                __ASM_EMIT("test        $0x8, %[mask]")
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("jz          7f")
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("7:")

                // Update delay only by mask
                FILTER_X4_POST_MEMSYNC

                // Repeat loop
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]
                __ASM_EMIT("jnz         5b")                                        // check that mask is not zero

                // Store delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    %%xmm6, 0x00(%[f])")                        // xmm6     = d0
                __ASM_EMIT32("movups    %%xmm7, 0x20(%[f])")                        // xmm7     = d1
                __ASM_EMIT64("movups    %%xmm6, 0x00(%[d])")                        // xmm6     = d0
                __ASM_EMIT64("movups    %%xmm7, 0x20(%[d])")                        // xmm7     = d1

                //---------------------------------------------------------------------
                // Cycle 2
                __ASM_EMIT("mov         %[X_D], %[dst]")
                __ASM_EMIT("mov         %[X_COUNT], %[count]")
                __ASM_EMIT("mov         %[dst], %[src]")                            // Chaining filter groups

                // Load delay buffer
                __ASM_EMIT32("movups    0x10(%[f]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT32("movups    0x30(%[f]), %%xmm7")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")
                __ASM_EMIT64("movups    0x10(%[d]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT64("movups    0x30(%[d]), %%xmm7")                        // xmm7     = d1

                // Initialize mask
                // xmm0=tmp, xmm1={s,s2[4]}, xmm2=p1[4], xmm3=p2[4], xmm6=d0[4], xmm7=d1[4]
                __ASM_EMIT("mov         $1, %[mask]")
                __ASM_EMIT("movaps      %[X_MASK], %%xmm0")
                __ASM_EMIT("xorps       %%xmm1, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")

                // Process first 3 steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X8P2_CORE
                // Shift buffer
                __ASM_EMIT("shufps      $0x90, %%xmm1, %%xmm1")                     // xmm1     = s2[0] s2[0] s2[1] s2[2]
                // Update delay only by mask
                FILTER_X4_PRE_MEMSYNC

                // Repeat loop
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jz          4f")                                        // jump to completion
                __ASM_EMIT("lea         0x01(,%[mask], 2), %[mask]")                // mask     = (mask << 1) | 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")                           // store mask
                __ASM_EMIT("cmp         $0x0f, %[mask]")
                __ASM_EMIT("jne         1b")

                // 4x filter processing without mask
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("3:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X8P2_CORE
                FILTER_X4_MEMSYNC

                // Shift buffer and repeat loop
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         3b")

                // Prepare last loop
                __ASM_EMIT("4:")
                __ASM_EMIT("movaps      %[MASK], %%xmm0")                           // xmm0     = m[0] m[1] m[2] m[3]
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]

                // Process steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("5:")
                FILTER_X8P2_CORE

                // Shift buffer and store
                __ASM_EMIT("test        $0x8, %[mask]")
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("jz          7f")
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("7:")

                // Update delay only by mask
                FILTER_X4_POST_MEMSYNC

                // Repeat loop
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]
                __ASM_EMIT("jnz         5b")                                        // check that mask is not zero

                // Store delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    %%xmm6, 0x10(%[f])")                        // xmm6     = d0
                __ASM_EMIT32("movups    %%xmm7, 0x30(%[f])")                        // xmm7     = d1
                __ASM_EMIT64("movups    %%xmm6, 0x10(%[d])")                        // xmm6     = d0
                __ASM_EMIT64("movups    %%xmm7, 0x30(%[d])")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")

                // Exit label
                __ASM_EMIT("10:")

                : [dst] "+r" (dst), [src] "+r" (src),
                  [mask] "=&r" (mask), [count] "+r" (count)
                : [f] "r" (f), [d] X86_GREG (d),
                  [X_MASK] "m" (biquad_mask_const),
                  [MASK] "m" (MASK),
                  [X_F] "m" (X_F),
                  [X_D] "m" (X_D),
                  [X_COUNT] "m" (X_COUNT)
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }

        void biquad_process_x16(float *dst, const float *src, float *d, size_t count, const dsp::biquad_x16_t *f)
        {
            IF_ARCH_X86(
                float   MASK[4] __lsp_aligned16;
                float  *X_F, *X_D;
                size_t  X_COUNT;
                size_t  mask;
            )

            ARCH_X86_ASM
            (
                // Check count
                __ASM_EMIT("test        %[count], %[count]")
                __ASM_EMIT("jz          10f")

                //---------------------------------------------------------------------
                // Cycle 1
                __ASM_EMIT32("mov       %[f], %[X_F]")
                __ASM_EMIT("mov         %[dst], %[X_D]")
                __ASM_EMIT("mov         %[count], %[X_COUNT]")

                // Load delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    0x00(%[f]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT32("movups    0x40(%[f]), %%xmm7")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")
                __ASM_EMIT64("movups    0x00(%[d]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT64("movups    0x40(%[d]), %%xmm7")                        // xmm7     = d1

                // Initialize mask
                // xmm0=tmp, xmm1={s,s2[4]}, xmm2=p1[4], xmm3=p2[4], xmm6=d0[4], xmm7=d1[4]
                __ASM_EMIT("mov         $1, %[mask]")
                __ASM_EMIT("movaps      %[X_MASK], %%xmm0")
                __ASM_EMIT("xorps       %%xmm1, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")

                // Process first 3 steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P1_CORE
                // Shift buffer
                __ASM_EMIT("shufps      $0x90, %%xmm1, %%xmm1")                     // xmm1     = s2[0] s2[0] s2[1] s2[2]
                // Update delay only by mask
                FILTER_X4_PRE_MEMSYNC

                // Repeat loop
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jz          4f")                                        // jump to completion
                __ASM_EMIT("lea         0x01(,%[mask], 2), %[mask]")                // mask     = (mask << 1) | 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")                           // store mask
                __ASM_EMIT("cmp         $0x0f, %[mask]")
                __ASM_EMIT("jne         1b")

                // 4x filter processing without mask
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("3:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P1_CORE
                FILTER_X4_MEMSYNC

                // Shift buffer and repeat loop
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         3b")

                // Prepare last loop
                __ASM_EMIT("4:")
                __ASM_EMIT("movaps      %[MASK], %%xmm0")                           // xmm0     = m[0] m[1] m[2] m[3]
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]

                // Process steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("5:")
                FILTER_X16P1_CORE

                // Shift buffer and store
                __ASM_EMIT("test        $0x8, %[mask]")
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("jz          7f")
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("7:")

                // Update delay only by mask
                FILTER_X4_POST_MEMSYNC

                // Repeat loop
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]
                __ASM_EMIT("jnz         5b")                                        // check that mask is not zero

                // Store delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    %%xmm6, 0x00(%[f])")                        // xmm6     = d0
                __ASM_EMIT32("movups    %%xmm7, 0x40(%[f])")                        // xmm7     = d1
                __ASM_EMIT64("movups    %%xmm6, 0x00(%[d])")                        // xmm6     = d0
                __ASM_EMIT64("movups    %%xmm7, 0x40(%[d])")                        // xmm7     = d1

                //---------------------------------------------------------------------
                // Cycle 2
                __ASM_EMIT("mov         %[X_D], %[dst]")
                __ASM_EMIT("mov         %[X_COUNT], %[count]")
                __ASM_EMIT("mov         %[dst], %[src]")                            // Chaining filter groups

                // Load delay buffer
                __ASM_EMIT32("movups    0x10(%[f]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT32("movups    0x50(%[f]), %%xmm7")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")
                __ASM_EMIT64("movups    0x10(%[d]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT64("movups    0x50(%[d]), %%xmm7")                        // xmm7     = d1

                // Initialize mask
                // xmm0=tmp, xmm1={s,s2[4]}, xmm2=p1[4], xmm3=p2[4], xmm6=d0[4], xmm7=d1[4]
                __ASM_EMIT("mov         $1, %[mask]")
                __ASM_EMIT("movaps      %[X_MASK], %%xmm0")
                __ASM_EMIT("xorps       %%xmm1, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")

                // Process first 3 steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P2_CORE
                // Shift buffer
                __ASM_EMIT("shufps      $0x90, %%xmm1, %%xmm1")                     // xmm1     = s2[0] s2[0] s2[1] s2[2]
                // Update delay only by mask
                FILTER_X4_PRE_MEMSYNC

                // Repeat loop
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jz          4f")                                        // jump to completion
                __ASM_EMIT("lea         0x01(,%[mask], 2), %[mask]")                // mask     = (mask << 1) | 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")                           // store mask
                __ASM_EMIT("cmp         $0x0f, %[mask]")
                __ASM_EMIT("jne         1b")

                // 4x filter processing without mask
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("3:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P2_CORE
                FILTER_X4_MEMSYNC

                // Shift buffer and repeat loop
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         3b")

                // Prepare last loop
                __ASM_EMIT("4:")
                __ASM_EMIT("movaps      %[MASK], %%xmm0")                           // xmm0     = m[0] m[1] m[2] m[3]
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]

                // Process steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("5:")
                FILTER_X16P2_CORE

                // Shift buffer and store
                __ASM_EMIT("test        $0x8, %[mask]")
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("jz          7f")
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("7:")

                // Update delay only by mask
                FILTER_X4_POST_MEMSYNC

                // Repeat loop
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]
                __ASM_EMIT("jnz         5b")                                        // check that mask is not zero

                // Store delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    %%xmm6, 0x10(%[f])")                        // xmm6     = d0
                __ASM_EMIT32("movups    %%xmm7, 0x50(%[f])")                        // xmm7     = d1
                __ASM_EMIT64("movups    %%xmm6, 0x10(%[d])")                        // xmm6     = d0
                __ASM_EMIT64("movups    %%xmm7, 0x50(%[d])")                        // xmm7     = d1

                //---------------------------------------------------------------------
                // Cycle 3
                __ASM_EMIT("mov         %[X_D], %[dst]")
                __ASM_EMIT("mov         %[X_COUNT], %[count]")
                __ASM_EMIT("mov         %[dst], %[src]")                            // Chaining filter groups

                // Load delay buffer
                __ASM_EMIT32("movups    0x20(%[f]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT32("movups    0x60(%[f]), %%xmm7")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")
                __ASM_EMIT64("movups    0x20(%[d]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT64("movups    0x60(%[d]), %%xmm7")                        // xmm7     = d1

                // Initialize mask
                // xmm0=tmp, xmm1={s,s2[4]}, xmm2=p1[4], xmm3=p2[4], xmm6=d0[4], xmm7=d1[4]
                __ASM_EMIT("mov         $1, %[mask]")
                __ASM_EMIT("movaps      %[X_MASK], %%xmm0")
                __ASM_EMIT("xorps       %%xmm1, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")

                // Process first 3 steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P3_CORE
                // Shift buffer
                __ASM_EMIT("shufps      $0x90, %%xmm1, %%xmm1")                     // xmm1     = s2[0] s2[0] s2[1] s2[2]
                // Update delay only by mask
                FILTER_X4_PRE_MEMSYNC

                // Repeat loop
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jz          4f")                                        // jump to completion
                __ASM_EMIT("lea         0x01(,%[mask], 2), %[mask]")                // mask     = (mask << 1) | 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")                           // store mask
                __ASM_EMIT("cmp         $0x0f, %[mask]")
                __ASM_EMIT("jne         1b")

                // 4x filter processing without mask
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("3:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P3_CORE
                FILTER_X4_MEMSYNC

                // Shift buffer and repeat loop
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         3b")

                // Prepare last loop
                __ASM_EMIT("4:")
                __ASM_EMIT("movaps      %[MASK], %%xmm0")                           // xmm0     = m[0] m[1] m[2] m[3]
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]

                // Process steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("5:")
                FILTER_X16P3_CORE

                // Shift buffer and store
                __ASM_EMIT("test        $0x8, %[mask]")
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("jz          7f")
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("7:")

                // Update delay only by mask
                FILTER_X4_POST_MEMSYNC

                // Repeat loop
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]
                __ASM_EMIT("jnz         5b")                                        // check that mask is not zero

                // Store delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    %%xmm6, 0x20(%[f])")                        // xmm6     = d0
                __ASM_EMIT32("movups    %%xmm7, 0x60(%[f])")                        // xmm7     = d1
                __ASM_EMIT64("movups    %%xmm6, 0x20(%[d])")                        // xmm6     = d0
                __ASM_EMIT64("movups    %%xmm7, 0x60(%[d])")                        // xmm7     = d1

                //---------------------------------------------------------------------
                // Cycle 4
                __ASM_EMIT("mov         %[X_D], %[dst]")
                __ASM_EMIT("mov         %[X_COUNT], %[count]")
                __ASM_EMIT("mov         %[dst], %[src]")                            // Chaining filter groups

                // Load delay buffer
                __ASM_EMIT32("movups    0x30(%[f]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT32("movups    0x70(%[f]), %%xmm7")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")
                __ASM_EMIT64("movups    0x30(%[d]), %%xmm6")                        // xmm6     = d0
                __ASM_EMIT64("movups    0x70(%[d]), %%xmm7")                        // xmm7     = d1

                // Initialize mask
                // xmm0=tmp, xmm1={s,s2[4]}, xmm2=p1[4], xmm3=p2[4], xmm6=d0[4], xmm7=d1[4]
                __ASM_EMIT("mov         $1, %[mask]")
                __ASM_EMIT("movaps      %[X_MASK], %%xmm0")
                __ASM_EMIT("xorps       %%xmm1, %%xmm1")
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")

                // Process first 3 steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("1:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P4_CORE
                // Shift buffer
                __ASM_EMIT("shufps      $0x90, %%xmm1, %%xmm1")                     // xmm1     = s2[0] s2[0] s2[1] s2[2]
                // Update delay only by mask
                FILTER_X4_PRE_MEMSYNC

                // Repeat loop
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jz          4f")                                        // jump to completion
                __ASM_EMIT("lea         0x01(,%[mask], 2), %[mask]")                // mask     = (mask << 1) | 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("movaps      %%xmm0, %[MASK]")                           // store mask
                __ASM_EMIT("cmp         $0x0f, %[mask]")
                __ASM_EMIT("jne         1b")

                // 4x filter processing without mask
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("3:")
                __ASM_EMIT("movss       (%[src]), %%xmm0")                          // xmm0     = *src
                __ASM_EMIT("add         $4, %[src]")                                // src      ++
                __ASM_EMIT("movss       %%xmm0, %%xmm1")                            // xmm1     = s
                FILTER_X16P4_CORE
                FILTER_X4_MEMSYNC

                // Shift buffer and repeat loop
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("dec         %[count]")
                __ASM_EMIT("jnz         3b")

                // Prepare last loop
                __ASM_EMIT("4:")
                __ASM_EMIT("movaps      %[MASK], %%xmm0")                           // xmm0     = m[0] m[1] m[2] m[3]
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]

                // Process steps
                __ASM_EMIT(".p2align    4")
                __ASM_EMIT("5:")
                FILTER_X16P4_CORE

                // Shift buffer and store
                __ASM_EMIT("test        $0x8, %[mask]")
                __ASM_EMIT("shufps      $0x93, %%xmm1, %%xmm1")                     // xmm1     = s2[3] s2[0] s2[1] s2[2]
                __ASM_EMIT("jz          7f")
                __ASM_EMIT("movss       %%xmm1, (%[dst])")                          // *dst     = s2[3]
                __ASM_EMIT("add         $4, %[dst]")                                // dst      ++
                __ASM_EMIT("7:")

                // Update delay only by mask
                FILTER_X4_POST_MEMSYNC

                // Repeat loop
                __ASM_EMIT("xorps       %%xmm2, %%xmm2")                            // xmm2     = 0 0 0 0
                __ASM_EMIT("shl         $1, %[mask]")                               // mask     = mask << 1
                __ASM_EMIT("shufps      $0x90, %%xmm0, %%xmm0")                     // xmm0     = m[0] m[0] m[1] m[2]
                __ASM_EMIT("and         $0x0f, %[mask]")                            // mask     = (mask << 1) & 0x0f
                __ASM_EMIT("movss       %%xmm2, %%xmm0")                            // xmm0     = 0 m[0] m[1] m[2]
                __ASM_EMIT("jnz         5b")                                        // check that mask is not zero

                // Store delay buffer
                __ASM_EMIT32("mov       %[d], %[f]")
                __ASM_EMIT32("movups    %%xmm6, 0x30(%[f])")                        // xmm6     = d0
                __ASM_EMIT32("movups    %%xmm7, 0x70(%[f])")                        // xmm7     = d1
                __ASM_EMIT64("movups    %%xmm6, 0x30(%[d])")                        // xmm6     = d0
                __ASM_EMIT64("movups    %%xmm7, 0x70(%[d])")                        // xmm7     = d1
                __ASM_EMIT32("mov       %[X_F], %[f]")

                // Exit label
                __ASM_EMIT("10:")

                : [dst] "+r" (dst), [src] "+r" (src),
                  [mask] "=&r" (mask), [count] "+r" (count)
                : [f] "r" (f), [d] X86_GREG (d),
                  [X_MASK] "m" (biquad_mask_const),
                  [MASK] "m" (MASK),
                  [X_F] "m" (X_F),
                  [X_D] "m" (X_D),
                  [X_COUNT] "m" (X_COUNT)
                : "cc", "memory",
                  "%xmm0", "%xmm1", "%xmm2", "%xmm3",
                  "%xmm4", "%xmm5", "%xmm6", "%xmm7"
            );
        }
    } /* namespace sse */
} /* namespace lsp */

#endif /* PRIVATE_DSP_ARCH_X86_SSE_FILTERS_STATIC_H_ */
