class UpsampleConformerEncoder(Module):
  __parameters__ = []
  __buffers__ = []
  embed : __torch__.cosyvoice.transformer.subsampling.___torch_mangle_33.LinearNoSubsampling
  up_embed : __torch__.cosyvoice.transformer.subsampling.___torch_mangle_33.LinearNoSubsampling
  def forward(self: __torch__.cosyvoice.transformer.upsample_encoder.___torch_mangle_32.UpsampleConformerEncoder,
    xs: Tensor,
    xs_lens: Tensor,
    context: Tensor=CONSTANTS.c0,
    decoding_chunk_size: int=0,
    num_decoding_left_chunks: int=-1,
    streaming: bool=False) -> Tuple[Tensor, Tensor]:
    _0 = "get chunk_masks all false at some timestep, force set to true, make sure they are masked in futuer computation!"
    _1 = "Input Error: Only 3D, 4D and 5D input Tensors supported (got {}D) for the modes: nearest | linear | bilinear | bicubic | trilinear | area | nearest-exact (got {})"
    T = torch.size(xs, 1)
    batch_size = torch.size(xs_lens, 0)
    if torch.gt(T, 0):
      max_len = T
    else:
      max_len = torch.item(torch.max(xs_lens))
    seq_range = torch.arange(0, max_len, dtype=4, layout=None, device=ops.prim.device(xs_lens))
    seq_range_expand = torch.expand(torch.unsqueeze(seq_range, 0), [batch_size, int(max_len)])
    seq_length_expand = torch.unsqueeze(xs_lens, -1)
    mask = torch.ge(seq_range_expand, seq_length_expand)
    masks = torch.bitwise_not(torch.unsqueeze(mask, 1))
    embed = self.embed
    _2 = torch.add(torch.matmul(xs, CONSTANTS.c1), CONSTANTS.c2)
    input = torch.layer_norm(_2, [512], CONSTANTS.c3, CONSTANTS.c4)
    pos_enc = embed.pos_enc
    pe = pos_enc.pe
    _3 = torch.size(pe, 1)
    _4 = torch.size(input, 1)
    _5 = torch.ge(_3, torch.sub(torch.mul(_4, 2), 1))
    if _5:
      pe0 = pos_enc.pe
      _6 = ops.prim.dtype(pe0)
      _7 = ops.prim.dtype(input)
      if torch.ne(_6, _7):
        _8 = True
      else:
        pe1 = pos_enc.pe
        _9 = torch.ne(ops.prim.device(pe1), ops.prim.device(input))
        _8 = _9
      if _8:
        pe2 = pos_enc.pe
        _10 = torch.to(pe2, ops.prim.device(input), _7)
        pos_enc.pe = _10
      else:
        pass
    else:
      _11 = [_4, 512]
      pe_positive = torch.zeros(_11)
      pe_negative = torch.zeros(_11)
      position = torch.unsqueeze(torch.arange(0, _4, dtype=6), 1)
      _12 = torch.mul(position, CONSTANTS.c5)
      _13 = torch.sin(_12)
      _14 = torch.slice(torch.slice(pe_positive), 1, 0, None, 2)
      _15 = torch.copy_(_14, _13)
      _16 = torch.cos(_12)
      _17 = torch.slice(torch.slice(pe_positive), 1, 1, None, 2)
      _18 = torch.copy_(_17, _16)
      _19 = torch.mul(torch.mul(position, -1), CONSTANTS.c5)
      _20 = torch.sin(_19)
      _21 = torch.slice(torch.slice(pe_negative), 1, 0, None, 2)
      _22 = torch.copy_(_21, _20)
      _23 = torch.cos(_19)
      _24 = torch.slice(torch.slice(pe_negative), 1, 1, None, 2)
      _25 = torch.copy_(_24, _23)
      pe_positive0 = torch.unsqueeze(torch.flip(pe_positive, [0]), 0)
      pe_negative0 = torch.unsqueeze(torch.slice(pe_negative, 0, 1), 0)
      pe3 = torch.cat([pe_positive0, pe_negative0], 1)
      _26 = torch.to(pe3, ops.prim.device(input), ops.prim.dtype(input))
      pos_enc.pe = _26
    x = torch.mul(input, 22.627416997969522)
    _27 = torch.size(x, 1)
    pe4 = pos_enc.pe
    _28 = torch.slice(pe4)
    pe5 = pos_enc.pe
    _29 = torch.floordiv(torch.size(pe5, 1), 2)
    _30 = torch.add(torch.sub(_29, _27), 1)
    pe6 = pos_enc.pe
    _31 = torch.floordiv(torch.size(pe6, 1), 2)
    pos_emb = torch.slice(_28, 1, _30, torch.add(_31, _27))
    if torch.ne(torch.size(context, 1), 0):
      embed0 = self.embed
      _32 = torch.matmul(context, CONSTANTS.c1)
      input0 = torch.layer_norm(torch.add(_32, CONSTANTS.c2), [512], CONSTANTS.c3, CONSTANTS.c4)
      pos_enc0 = embed0.pos_enc
      pe7 = pos_enc0.pe
      _33 = torch.size(pe7, 1)
      _34 = torch.size(input0, 1)
      _35 = torch.ge(_33, torch.sub(torch.mul(_34, 2), 1))
      if _35:
        pe8 = pos_enc0.pe
        _36 = ops.prim.dtype(pe8)
        _37 = ops.prim.dtype(input0)
        if torch.ne(_36, _37):
          _38 = True
        else:
          pe9 = pos_enc0.pe
          _39 = torch.ne(ops.prim.device(pe9), ops.prim.device(input0))
          _38 = _39
        if _38:
          pe10 = pos_enc0.pe
          _40 = torch.to(pe10, ops.prim.device(input0), _37)
          pos_enc0.pe = _40
        else:
          pass
      else:
        _41 = [_34, 512]
        pe_positive1 = torch.zeros(_41)
        pe_negative1 = torch.zeros(_41)
        position0 = torch.unsqueeze(torch.arange(0, _34, dtype=6), 1)
        _42 = torch.mul(position0, CONSTANTS.c5)
        _43 = torch.sin(_42)
        _44 = torch.slice(torch.slice(pe_positive1), 1, 0, None, 2)
        _45 = torch.copy_(_44, _43)
        _46 = torch.cos(_42)
        _47 = torch.slice(torch.slice(pe_positive1), 1, 1, None, 2)
        _48 = torch.copy_(_47, _46)
        _49 = torch.mul(torch.mul(position0, -1), CONSTANTS.c5)
        _50 = torch.sin(_49)
        _51 = torch.slice(torch.slice(pe_negative1), 1, 0, None, 2)
        _52 = torch.copy_(_51, _50)
        _53 = torch.cos(_49)
        _54 = torch.slice(torch.slice(pe_negative1), 1, 1, None, 2)
        _55 = torch.copy_(_54, _53)
        pe_positive2 = torch.unsqueeze(torch.flip(pe_positive1, [0]), 0)
        pe_negative2 = torch.unsqueeze(torch.slice(pe_negative1, 0, 1), 0)
        pe11 = torch.cat([pe_positive2, pe_negative2], 1)
        _56 = torch.to(pe11, ops.prim.device(input0), ops.prim.dtype(input0))
        pos_enc0.pe = _56
      x0 = torch.mul(input0, 22.627416997969522)
      context0 = x0
    else:
      context0 = context
    _57 = torch.eq(streaming, True)
    if _57:
      _58 = 25
    else:
      _58 = 0
    if torch.gt(_58, 0):
      pos_idx = torch.arange(torch.size(x, 1), dtype=None, layout=None, device=ops.prim.device(x))
      _59 = torch.div(pos_idx, _58, rounding_mode="trunc")
      block_value = torch.mul(torch.add(_59, 1), _58)
      chunk_masks0 = torch.lt(torch.unsqueeze(pos_idx, 0), torch.unsqueeze(block_value, 1))
      chunk_masks1 = torch.unsqueeze(chunk_masks0, 0)
      chunk_masks2 = torch.__and__(masks, chunk_masks1)
      chunk_masks = chunk_masks2
    else:
      chunk_masks = masks
    _60 = torch.eq(ops.prim.dtype(chunk_masks), 11)
    if _60:
      pass
    else:
      ops.prim.RaiseException("AssertionError: ")
    _61 = torch.eq(torch.sum(chunk_masks, [-1]), 0)
    _62 = torch.ne(torch.item(torch.sum(_61)), 0)
    if _62:
      print(_0)
      _63 = torch.eq(torch.sum(chunk_masks, [-1]), 0)
      _64 = torch.tensor(True, dtype=ops.prim.dtype(chunk_masks), device=ops.prim.device(chunk_masks))
      _65 = annotate(List[Optional[Tensor]], [_63])
      _66 = torch.index_put_(chunk_masks, _65, _64)
    else:
      pass
    outputs = torch.contiguous(torch.transpose(x, 1, 2))
    context1 = torch.contiguous(torch.transpose(context0, 1, 2))
    _67 = torch.eq(torch.size(context1, 2), 0)
    if _67:
      outputs1 = torch.pad(outputs, [0, 3], "constant", 0.)
      outputs0 = outputs1
    else:
      _68 = torch.eq(torch.size(context1, 2), 3)
      if _68:
        pass
      else:
        ops.prim.RaiseException("AssertionError: ")
      _69 = torch.cat([outputs, context1], 2)
      _70 = torch.sub(3, torch.size(context1, 2))
      outputs2 = torch.pad(_69, [0, _70], "constant", 0.)
      outputs0 = outputs2
    _71 = torch.conv1d(outputs0, CONSTANTS.c6, CONSTANTS.c7)
    result = torch.leaky_relu(_71)
    outputs3 = torch.pad(result, [2, 0], "constant", 0.)
    outputs4 = torch.conv1d(outputs3, CONSTANTS.c8, CONSTANTS.c9)
    outputs5 = torch.contiguous(torch.transpose(outputs4, 1, 2))
    xs0 = torch.add(outputs5, x)
    x1 = torch.layer_norm(xs0, [512], CONSTANTS.c10, CONSTANTS.c11, 9.9999999999999998e-13)
    n_batch = torch.size(x1, 0)
    _72 = torch.add(torch.matmul(x1, CONSTANTS.c12), CONSTANTS.c13)
    _73 = torch.slice(_72, -1, 1024, 1536)
    _74 = torch.slice(_72, -1, 512, 1024)
    _75 = torch.slice(_72, -1, 0, 512)
    _76 = [n_batch, -1, 8, 64]
    q = torch.view(_75, _76)
    k = torch.view(_74, _76)
    v = torch.view(_73, _76)
    q0 = torch.transpose(q, 1, 2)
    k0 = torch.transpose(k, 1, 2)
    v0 = torch.transpose(v, 1, 2)
    q1 = torch.transpose(q0, 1, 2)
    n_batch_pos = torch.size(pos_emb, 0)
    _77 = torch.matmul(pos_emb, CONSTANTS.c14)
    p = torch.view(_77, [n_batch_pos, -1, 8, 64])
    p0 = torch.transpose(p, 1, 2)
    q_with_bias_u = torch.transpose(torch.add(q1, CONSTANTS.c15), 1, 2)
    q_with_bias_v = torch.transpose(torch.add(q1, CONSTANTS.c16), 1, 2)
    matrix_ac = torch.matmul(q_with_bias_u, torch.transpose(k0, -2, -1))
    matrix_bd = torch.matmul(q_with_bias_v, torch.transpose(p0, -2, -1))
    _78 = torch.size(matrix_ac)
    _79 = torch.size(matrix_bd)
    if torch.ne(_78, _79):
      _80 = _79[0]
      _81 = _79[1]
      _82 = _79[2]
      _83 = ops.prim.device(matrix_bd)
      _84 = ops.prim.dtype(matrix_bd)
      zero_pad = torch.zeros([_80, _81, _82, 1], dtype=_84, layout=None, device=_83)
      x_padded = torch.cat([zero_pad, matrix_bd], -1)
      _85 = torch.add(torch.size(matrix_bd, 3), 1)
      _86 = [_80, _81, _85, torch.size(matrix_bd, 2)]
      x_padded0 = torch.view(x_padded, _86)
      _87 = torch.slice(torch.slice(x_padded0), 1)
      _88 = torch.view_as(torch.slice(_87, 2, 1), matrix_bd)
      _89 = torch.slice(torch.slice(torch.slice(_88), 1), 2)
      _90 = torch.floordiv(torch.size(matrix_bd, -1), 2)
      matrix_bd1 = torch.slice(_89, 3, None, torch.add(_90, 1))
      matrix_bd0 = matrix_bd1
    else:
      matrix_bd0 = matrix_bd
    scores = torch.div(torch.add(matrix_ac, matrix_bd0), 8.)
    n_batch0 = torch.size(v0, 0)
    _91 = torch.gt(torch.size(chunk_masks, 2), 0)
    if _91:
      mask0 = torch.eq(torch.unsqueeze(chunk_masks, 1), 0)
      _92 = torch.slice(torch.slice(mask0), 1)
      mask1 = torch.slice(torch.slice(_92, 2), 3, None, torch.size(scores, -1))
      scores0 = torch.masked_fill(scores, mask1, -inf)
      attn0 = torch.masked_fill(torch.softmax(scores0, -1), mask1, 0.)
      attn = attn0
    else:
      attn = torch.softmax(scores, -1)
    x2 = torch.matmul(attn, v0)
    _93 = torch.contiguous(torch.transpose(x2, 1, 2))
    x3 = torch.view(_93, [n_batch0, -1, 512])
    _94 = torch.add(torch.matmul(x3, CONSTANTS.c17), CONSTANTS.c18)
    x4 = torch.add(xs0, _94)
    x5 = torch.layer_norm(x4, [512], CONSTANTS.c19, CONSTANTS.c20, 9.9999999999999998e-13)
    _95 = torch.add(torch.matmul(x5, CONSTANTS.c21), CONSTANTS.c22)
    _96 = torch.matmul(torch.silu(_95), CONSTANTS.c23)
    _97 = torch.mul(torch.add(_96, CONSTANTS.c24), 1.)
    x6 = torch.add(x4, _97)
    x7 = torch.layer_norm(x6, [512], CONSTANTS.c25, CONSTANTS.c26, 9.9999999999999998e-13)
    n_batch1 = torch.size(x7, 0)
    _98 = torch.add(torch.matmul(x7, CONSTANTS.c27), CONSTANTS.c28)
    _99 = torch.slice(_98, -1, 1024, 1536)
    _100 = torch.slice(_98, -1, 512, 1024)
    _101 = torch.slice(_98, -1, 0, 512)
    _102 = [n_batch1, -1, 8, 64]
    q2 = torch.view(_101, _102)
    k1 = torch.view(_100, _102)
    v1 = torch.view(_99, _102)
    q3 = torch.transpose(q2, 1, 2)
    k2 = torch.transpose(k1, 1, 2)
    v2 = torch.transpose(v1, 1, 2)
    q4 = torch.transpose(q3, 1, 2)
    n_batch_pos0 = torch.size(pos_emb, 0)
    _103 = torch.matmul(pos_emb, CONSTANTS.c29)
    p1 = torch.view(_103, [n_batch_pos0, -1, 8, 64])
    p2 = torch.transpose(p1, 1, 2)
    q_with_bias_u0 = torch.transpose(torch.add(q4, CONSTANTS.c30), 1, 2)
    q_with_bias_v0 = torch.transpose(torch.add(q4, CONSTANTS.c31), 1, 2)
    matrix_ac0 = torch.matmul(q_with_bias_u0, torch.transpose(k2, -2, -1))
    matrix_bd2 = torch.matmul(q_with_bias_v0, torch.transpose(p2, -2, -1))
    _104 = torch.size(matrix_ac0)
    _105 = torch.size(matrix_bd2)
    if torch.ne(_104, _105):
      _106 = _105[0]
      _107 = _105[1]
      _108 = _105[2]
      _109 = ops.prim.device(matrix_bd2)
      _110 = ops.prim.dtype(matrix_bd2)
      zero_pad0 = torch.zeros([_106, _107, _108, 1], dtype=_110, layout=None, device=_109)
      x_padded1 = torch.cat([zero_pad0, matrix_bd2], -1)
      _111 = torch.add(torch.size(matrix_bd2, 3), 1)
      _112 = [_106, _107, _111, torch.size(matrix_bd2, 2)]
      x_padded2 = torch.view(x_padded1, _112)
      _113 = torch.slice(torch.slice(x_padded2), 1)
      _114 = torch.view_as(torch.slice(_113, 2, 1), matrix_bd2)
      _115 = torch.slice(torch.slice(torch.slice(_114), 1), 2)
      _116 = torch.floordiv(torch.size(matrix_bd2, -1), 2)
      matrix_bd4 = torch.slice(_115, 3, None, torch.add(_116, 1))
      matrix_bd3 = matrix_bd4
    else:
      matrix_bd3 = matrix_bd2
    scores1 = torch.div(torch.add(matrix_ac0, matrix_bd3), 8.)
    n_batch2 = torch.size(v2, 0)
    _117 = torch.gt(torch.size(chunk_masks, 2), 0)
    if _117:
      mask2 = torch.eq(torch.unsqueeze(chunk_masks, 1), 0)
      _118 = torch.slice(torch.slice(mask2), 1)
      mask3 = torch.slice(torch.slice(_118, 2), 3, None, torch.size(scores1, -1))
      scores2 = torch.masked_fill(scores1, mask3, -inf)
      attn2 = torch.masked_fill(torch.softmax(scores2, -1), mask3, 0.)
      attn1 = attn2
    else:
      attn1 = torch.softmax(scores1, -1)
    x8 = torch.matmul(attn1, v2)
    _119 = torch.contiguous(torch.transpose(x8, 1, 2))
    x9 = torch.view(_119, [n_batch2, -1, 512])
    _120 = torch.add(torch.matmul(x9, CONSTANTS.c32), CONSTANTS.c33)
    x10 = torch.add(x6, _120)
    x11 = torch.layer_norm(x10, [512], CONSTANTS.c34, CONSTANTS.c35, 9.9999999999999998e-13)
    _121 = torch.add(torch.matmul(x11, CONSTANTS.c36), CONSTANTS.c37)
    _122 = torch.matmul(torch.silu(_121), CONSTANTS.c38)
    _123 = torch.mul(torch.add(_122, CONSTANTS.c39), 1.)
    x12 = torch.add(x10, _123)
    x13 = torch.layer_norm(x12, [512], CONSTANTS.c40, CONSTANTS.c41, 9.9999999999999998e-13)
    n_batch3 = torch.size(x13, 0)
    _124 = torch.add(torch.matmul(x13, CONSTANTS.c42), CONSTANTS.c43)
    _125 = torch.slice(_124, -1, 1024, 1536)
    _126 = torch.slice(_124, -1, 512, 1024)
    _127 = torch.slice(_124, -1, 0, 512)
    _128 = [n_batch3, -1, 8, 64]
    q5 = torch.view(_127, _128)
    k3 = torch.view(_126, _128)
    v3 = torch.view(_125, _128)
    q6 = torch.transpose(q5, 1, 2)
    k4 = torch.transpose(k3, 1, 2)
    v4 = torch.transpose(v3, 1, 2)
    q7 = torch.transpose(q6, 1, 2)
    n_batch_pos1 = torch.size(pos_emb, 0)
    _129 = torch.matmul(pos_emb, CONSTANTS.c44)
    p3 = torch.view(_129, [n_batch_pos1, -1, 8, 64])
    p4 = torch.transpose(p3, 1, 2)
    q_with_bias_u1 = torch.transpose(torch.add(q7, CONSTANTS.c45), 1, 2)
    q_with_bias_v1 = torch.transpose(torch.add(q7, CONSTANTS.c46), 1, 2)
    matrix_ac1 = torch.matmul(q_with_bias_u1, torch.transpose(k4, -2, -1))
    matrix_bd5 = torch.matmul(q_with_bias_v1, torch.transpose(p4, -2, -1))
    _130 = torch.size(matrix_ac1)
    _131 = torch.size(matrix_bd5)
    if torch.ne(_130, _131):
      _132 = _131[0]
      _133 = _131[1]
      _134 = _131[2]
      _135 = ops.prim.device(matrix_bd5)
      _136 = ops.prim.dtype(matrix_bd5)
      zero_pad1 = torch.zeros([_132, _133, _134, 1], dtype=_136, layout=None, device=_135)
      x_padded3 = torch.cat([zero_pad1, matrix_bd5], -1)
      _137 = torch.add(torch.size(matrix_bd5, 3), 1)
      _138 = [_132, _133, _137, torch.size(matrix_bd5, 2)]
      x_padded4 = torch.view(x_padded3, _138)
      _139 = torch.slice(torch.slice(x_padded4), 1)
      _140 = torch.view_as(torch.slice(_139, 2, 1), matrix_bd5)
      _141 = torch.slice(torch.slice(torch.slice(_140), 1), 2)
      _142 = torch.floordiv(torch.size(matrix_bd5, -1), 2)
      matrix_bd7 = torch.slice(_141, 3, None, torch.add(_142, 1))
      matrix_bd6 = matrix_bd7
    else:
      matrix_bd6 = matrix_bd5
    scores3 = torch.div(torch.add(matrix_ac1, matrix_bd6), 8.)
    n_batch4 = torch.size(v4, 0)
    _143 = torch.gt(torch.size(chunk_masks, 2), 0)
    if _143:
      mask4 = torch.eq(torch.unsqueeze(chunk_masks, 1), 0)
      _144 = torch.slice(torch.slice(mask4), 1)
      mask5 = torch.slice(torch.slice(_144, 2), 3, None, torch.size(scores3, -1))
      scores4 = torch.masked_fill(scores3, mask5, -inf)
      attn4 = torch.masked_fill(torch.softmax(scores4, -1), mask5, 0.)
      attn3 = attn4
    else:
      attn3 = torch.softmax(scores3, -1)
    x14 = torch.matmul(attn3, v4)
    _145 = torch.contiguous(torch.transpose(x14, 1, 2))
    x15 = torch.view(_145, [n_batch4, -1, 512])
    _146 = torch.add(torch.matmul(x15, CONSTANTS.c47), CONSTANTS.c48)
    x16 = torch.add(x12, _146)
    x17 = torch.layer_norm(x16, [512], CONSTANTS.c49, CONSTANTS.c50, 9.9999999999999998e-13)
    _147 = torch.add(torch.matmul(x17, CONSTANTS.c51), CONSTANTS.c52)
    _148 = torch.matmul(torch.silu(_147), CONSTANTS.c53)
    _149 = torch.mul(torch.add(_148, CONSTANTS.c54), 1.)
    x18 = torch.add(x16, _149)
    x19 = torch.layer_norm(x18, [512], CONSTANTS.c55, CONSTANTS.c56, 9.9999999999999998e-13)
    n_batch5 = torch.size(x19, 0)
    _150 = torch.add(torch.matmul(x19, CONSTANTS.c57), CONSTANTS.c58)
    _151 = torch.slice(_150, -1, 1024, 1536)
    _152 = torch.slice(_150, -1, 512, 1024)
    _153 = torch.slice(_150, -1, 0, 512)
    _154 = [n_batch5, -1, 8, 64]
    q8 = torch.view(_153, _154)
    k5 = torch.view(_152, _154)
    v5 = torch.view(_151, _154)
    q9 = torch.transpose(q8, 1, 2)
    k6 = torch.transpose(k5, 1, 2)
    v6 = torch.transpose(v5, 1, 2)
    q10 = torch.transpose(q9, 1, 2)
    n_batch_pos2 = torch.size(pos_emb, 0)
    _155 = torch.matmul(pos_emb, CONSTANTS.c59)
    p5 = torch.view(_155, [n_batch_pos2, -1, 8, 64])
    p6 = torch.transpose(p5, 1, 2)
    q_with_bias_u2 = torch.transpose(torch.add(q10, CONSTANTS.c60), 1, 2)
    q_with_bias_v2 = torch.transpose(torch.add(q10, CONSTANTS.c61), 1, 2)
    matrix_ac2 = torch.matmul(q_with_bias_u2, torch.transpose(k6, -2, -1))
    matrix_bd8 = torch.matmul(q_with_bias_v2, torch.transpose(p6, -2, -1))
    _156 = torch.size(matrix_ac2)
    _157 = torch.size(matrix_bd8)
    if torch.ne(_156, _157):
      _158 = _157[0]
      _159 = _157[1]
      _160 = _157[2]
      _161 = ops.prim.device(matrix_bd8)
      _162 = ops.prim.dtype(matrix_bd8)
      zero_pad2 = torch.zeros([_158, _159, _160, 1], dtype=_162, layout=None, device=_161)
      x_padded5 = torch.cat([zero_pad2, matrix_bd8], -1)
      _163 = torch.add(torch.size(matrix_bd8, 3), 1)
      _164 = [_158, _159, _163, torch.size(matrix_bd8, 2)]
      x_padded6 = torch.view(x_padded5, _164)
      _165 = torch.slice(torch.slice(x_padded6), 1)
      _166 = torch.view_as(torch.slice(_165, 2, 1), matrix_bd8)
      _167 = torch.slice(torch.slice(torch.slice(_166), 1), 2)
      _168 = torch.floordiv(torch.size(matrix_bd8, -1), 2)
      matrix_bd10 = torch.slice(_167, 3, None, torch.add(_168, 1))
      matrix_bd9 = matrix_bd10
    else:
      matrix_bd9 = matrix_bd8
    scores5 = torch.div(torch.add(matrix_ac2, matrix_bd9), 8.)
    n_batch6 = torch.size(v6, 0)
    _169 = torch.gt(torch.size(chunk_masks, 2), 0)
    if _169:
      mask6 = torch.eq(torch.unsqueeze(chunk_masks, 1), 0)
      _170 = torch.slice(torch.slice(mask6), 1)
      mask7 = torch.slice(torch.slice(_170, 2), 3, None, torch.size(scores5, -1))
      scores6 = torch.masked_fill(scores5, mask7, -inf)
      attn6 = torch.masked_fill(torch.softmax(scores6, -1), mask7, 0.)
      attn5 = attn6
    else:
      attn5 = torch.softmax(scores5, -1)
    x20 = torch.matmul(attn5, v6)
    _171 = torch.contiguous(torch.transpose(x20, 1, 2))
    x21 = torch.view(_171, [n_batch6, -1, 512])
    _172 = torch.add(torch.matmul(x21, CONSTANTS.c62), CONSTANTS.c63)
    x22 = torch.add(x18, _172)
    x23 = torch.layer_norm(x22, [512], CONSTANTS.c64, CONSTANTS.c65, 9.9999999999999998e-13)
    _173 = torch.add(torch.matmul(x23, CONSTANTS.c66), CONSTANTS.c67)
    _174 = torch.matmul(torch.silu(_173), CONSTANTS.c68)
    _175 = torch.mul(torch.add(_174, CONSTANTS.c69), 1.)
    x24 = torch.add(x22, _175)
    x25 = torch.layer_norm(x24, [512], CONSTANTS.c70, CONSTANTS.c71, 9.9999999999999998e-13)
    n_batch7 = torch.size(x25, 0)
    _176 = torch.add(torch.matmul(x25, CONSTANTS.c72), CONSTANTS.c73)
    _177 = torch.slice(_176, -1, 1024, 1536)
    _178 = torch.slice(_176, -1, 512, 1024)
    _179 = torch.slice(_176, -1, 0, 512)
    _180 = [n_batch7, -1, 8, 64]
    q11 = torch.view(_179, _180)
    k7 = torch.view(_178, _180)
    v7 = torch.view(_177, _180)
    q12 = torch.transpose(q11, 1, 2)
    k8 = torch.transpose(k7, 1, 2)
    v8 = torch.transpose(v7, 1, 2)
    q13 = torch.transpose(q12, 1, 2)
    n_batch_pos3 = torch.size(pos_emb, 0)
    _181 = torch.matmul(pos_emb, CONSTANTS.c74)
    p7 = torch.view(_181, [n_batch_pos3, -1, 8, 64])
    p8 = torch.transpose(p7, 1, 2)
    q_with_bias_u3 = torch.transpose(torch.add(q13, CONSTANTS.c75), 1, 2)
    q_with_bias_v3 = torch.transpose(torch.add(q13, CONSTANTS.c76), 1, 2)
    matrix_ac3 = torch.matmul(q_with_bias_u3, torch.transpose(k8, -2, -1))
    matrix_bd11 = torch.matmul(q_with_bias_v3, torch.transpose(p8, -2, -1))
    _182 = torch.size(matrix_ac3)
    _183 = torch.size(matrix_bd11)
    if torch.ne(_182, _183):
      _184 = _183[0]
      _185 = _183[1]
      _186 = _183[2]
      _187 = ops.prim.device(matrix_bd11)
      _188 = ops.prim.dtype(matrix_bd11)
      zero_pad3 = torch.zeros([_184, _185, _186, 1], dtype=_188, layout=None, device=_187)
      x_padded7 = torch.cat([zero_pad3, matrix_bd11], -1)
      _189 = torch.add(torch.size(matrix_bd11, 3), 1)
      _190 = [_184, _185, _189, torch.size(matrix_bd11, 2)]
      x_padded8 = torch.view(x_padded7, _190)
      _191 = torch.slice(torch.slice(x_padded8), 1)
      _192 = torch.view_as(torch.slice(_191, 2, 1), matrix_bd11)
      _193 = torch.slice(torch.slice(torch.slice(_192), 1), 2)
      _194 = torch.floordiv(torch.size(matrix_bd11, -1), 2)
      matrix_bd13 = torch.slice(_193, 3, None, torch.add(_194, 1))
      matrix_bd12 = matrix_bd13
    else:
      matrix_bd12 = matrix_bd11
    scores7 = torch.div(torch.add(matrix_ac3, matrix_bd12), 8.)
    n_batch8 = torch.size(v8, 0)
    _195 = torch.gt(torch.size(chunk_masks, 2), 0)
    if _195:
      mask8 = torch.eq(torch.unsqueeze(chunk_masks, 1), 0)
      _196 = torch.slice(torch.slice(mask8), 1)
      mask9 = torch.slice(torch.slice(_196, 2), 3, None, torch.size(scores7, -1))
      scores8 = torch.masked_fill(scores7, mask9, -inf)
      attn8 = torch.masked_fill(torch.softmax(scores8, -1), mask9, 0.)
      attn7 = attn8
    else:
      attn7 = torch.softmax(scores7, -1)
    x26 = torch.matmul(attn7, v8)
    _197 = torch.contiguous(torch.transpose(x26, 1, 2))
    x27 = torch.view(_197, [n_batch8, -1, 512])
    _198 = torch.add(torch.matmul(x27, CONSTANTS.c77), CONSTANTS.c78)
    x28 = torch.add(x24, _198)
    x29 = torch.layer_norm(x28, [512], CONSTANTS.c79, CONSTANTS.c80, 9.9999999999999998e-13)
    _199 = torch.add(torch.matmul(x29, CONSTANTS.c81), CONSTANTS.c82)
    _200 = torch.matmul(torch.silu(_199), CONSTANTS.c83)
    _201 = torch.mul(torch.add(_200, CONSTANTS.c84), 1.)
    x30 = torch.add(x28, _201)
    x31 = torch.layer_norm(x30, [512], CONSTANTS.c85, CONSTANTS.c86, 9.9999999999999998e-13)
    n_batch9 = torch.size(x31, 0)
    _202 = torch.add(torch.matmul(x31, CONSTANTS.c87), CONSTANTS.c88)
    _203 = torch.slice(_202, -1, 1024, 1536)
    _204 = torch.slice(_202, -1, 512, 1024)
    _205 = torch.slice(_202, -1, 0, 512)
    _206 = [n_batch9, -1, 8, 64]
    q14 = torch.view(_205, _206)
    k9 = torch.view(_204, _206)
    v9 = torch.view(_203, _206)
    q15 = torch.transpose(q14, 1, 2)
    k10 = torch.transpose(k9, 1, 2)
    v10 = torch.transpose(v9, 1, 2)
    q16 = torch.transpose(q15, 1, 2)
    n_batch_pos4 = torch.size(pos_emb, 0)
    _207 = torch.matmul(pos_emb, CONSTANTS.c89)
    p9 = torch.view(_207, [n_batch_pos4, -1, 8, 64])
    p10 = torch.transpose(p9, 1, 2)
    q_with_bias_u4 = torch.transpose(torch.add(q16, CONSTANTS.c90), 1, 2)
    q_with_bias_v4 = torch.transpose(torch.add(q16, CONSTANTS.c91), 1, 2)
    matrix_ac4 = torch.matmul(q_with_bias_u4, torch.transpose(k10, -2, -1))
    matrix_bd14 = torch.matmul(q_with_bias_v4, torch.transpose(p10, -2, -1))
    _208 = torch.size(matrix_ac4)
    _209 = torch.size(matrix_bd14)
    if torch.ne(_208, _209):
      _210 = _209[0]
      _211 = _209[1]
      _212 = _209[2]
      _213 = ops.prim.device(matrix_bd14)
      _214 = ops.prim.dtype(matrix_bd14)
      zero_pad4 = torch.zeros([_210, _211, _212, 1], dtype=_214, layout=None, device=_213)
      x_padded9 = torch.cat([zero_pad4, matrix_bd14], -1)
      _215 = torch.add(torch.size(matrix_bd14, 3), 1)
      _216 = [_210, _211, _215, torch.size(matrix_bd14, 2)]
      x_padded10 = torch.view(x_padded9, _216)
      _217 = torch.slice(torch.slice(x_padded10), 1)
      _218 = torch.view_as(torch.slice(_217, 2, 1), matrix_bd14)
      _219 = torch.slice(torch.slice(torch.slice(_218), 1), 2)
      _220 = torch.floordiv(torch.size(matrix_bd14, -1), 2)
      matrix_bd16 = torch.slice(_219, 3, None, torch.add(_220, 1))
      matrix_bd15 = matrix_bd16
    else:
      matrix_bd15 = matrix_bd14
    scores9 = torch.div(torch.add(matrix_ac4, matrix_bd15), 8.)
    n_batch10 = torch.size(v10, 0)
    _221 = torch.gt(torch.size(chunk_masks, 2), 0)
    if _221:
      mask10 = torch.eq(torch.unsqueeze(chunk_masks, 1), 0)
      _222 = torch.slice(torch.slice(mask10), 1)
      mask11 = torch.slice(torch.slice(_222, 2), 3, None, torch.size(scores9, -1))
      scores10 = torch.masked_fill(scores9, mask11, -inf)
      attn10 = torch.masked_fill(torch.softmax(scores10, -1), mask11, 0.)
      attn9 = attn10
    else:
      attn9 = torch.softmax(scores9, -1)
    x32 = torch.matmul(attn9, v10)
    _223 = torch.contiguous(torch.transpose(x32, 1, 2))
    x33 = torch.view(_223, [n_batch10, -1, 512])
    _224 = torch.add(torch.matmul(x33, CONSTANTS.c92), CONSTANTS.c93)
    x34 = torch.add(x30, _224)
    x35 = torch.layer_norm(x34, [512], CONSTANTS.c94, CONSTANTS.c95, 9.9999999999999998e-13)
    _225 = torch.add(torch.matmul(x35, CONSTANTS.c96), CONSTANTS.c97)
    _226 = torch.matmul(torch.silu(_225), CONSTANTS.c98)
    _227 = torch.mul(torch.add(_226, CONSTANTS.c99), 1.)
    x36 = torch.add(x34, _227)
    xs1 = torch.contiguous(torch.transpose(x36, 1, 2))
    _228 = uninitialized(Tensor)
    dim = torch.sub(torch.dim(xs1), 2)
    scale_factors = annotate(List[float], [])
    for _229 in range(dim):
      _230 = torch.append(scale_factors, 2.)
    if torch.eq(torch.dim(xs1), 3):
      _231 = torch.upsample_nearest1d(xs1, None, scale_factors)
      outputs6 = _231
    else:
      if torch.eq(torch.dim(xs1), 4):
        _233 = torch.upsample_nearest2d(xs1, None, scale_factors)
        _232 = _233
      else:
        if torch.eq(torch.dim(xs1), 5):
          _235 = torch.upsample_nearest3d(xs1, None, scale_factors)
          _234 = _235
        else:
          _236 = torch.format(_1, torch.dim(xs1), "nearest")
          ops.prim.RaiseException(_236, "builtins.NotImplementedError")
          _234 = _228
        _232 = _234
      outputs6 = _232
    outputs7 = torch.pad(outputs6, [4, 0], "constant", 0.)
    outputs8 = torch.conv1d(outputs7, CONSTANTS.c100, CONSTANTS.c101)
    _237 = torch.mul(xs_lens, 2)
    xs2 = torch.contiguous(torch.transpose(outputs8, 1, 2))
    T0 = torch.size(xs2, 1)
    batch_size0 = torch.size(_237, 0)
    if torch.gt(T0, 0):
      max_len0 = T0
    else:
      max_len0 = torch.item(torch.max(_237))
    seq_range0 = torch.arange(0, max_len0, dtype=4, layout=None, device=ops.prim.device(_237))
    seq_range_expand0 = torch.expand(torch.unsqueeze(seq_range0, 0), [batch_size0, int(max_len0)])
    seq_length_expand0 = torch.unsqueeze(_237, -1)
    mask12 = torch.ge(seq_range_expand0, seq_length_expand0)
    masks0 = torch.bitwise_not(torch.unsqueeze(mask12, 1))
    up_embed = self.up_embed
    _238 = torch.add(torch.matmul(xs2, CONSTANTS.c102), CONSTANTS.c103)
    input1 = torch.layer_norm(_238, [512], CONSTANTS.c104, CONSTANTS.c105)
    pos_enc1 = up_embed.pos_enc
    pe12 = pos_enc1.pe
    _239 = torch.size(pe12, 1)
    _240 = torch.size(input1, 1)
    _241 = torch.ge(_239, torch.sub(torch.mul(_240, 2), 1))
    if _241:
      pe13 = pos_enc1.pe
      _242 = ops.prim.dtype(pe13)
      _243 = ops.prim.dtype(input1)
      if torch.ne(_242, _243):
        _244 = True
      else:
        pe14 = pos_enc1.pe
        _245 = torch.ne(ops.prim.device(pe14), ops.prim.device(input1))
        _244 = _245
      if _244:
        pe15 = pos_enc1.pe
        _246 = torch.to(pe15, ops.prim.device(input1), _243)
        pos_enc1.pe = _246
      else:
        pass
    else:
      _247 = [_240, 512]
      pe_positive3 = torch.zeros(_247)
      pe_negative3 = torch.zeros(_247)
      position1 = torch.unsqueeze(torch.arange(0, _240, dtype=6), 1)
      _248 = torch.mul(position1, CONSTANTS.c5)
      _249 = torch.sin(_248)
      _250 = torch.slice(torch.slice(pe_positive3), 1, 0, None, 2)
      _251 = torch.copy_(_250, _249)
      _252 = torch.cos(_248)
      _253 = torch.slice(torch.slice(pe_positive3), 1, 1, None, 2)
      _254 = torch.copy_(_253, _252)
      _255 = torch.mul(torch.mul(position1, -1), CONSTANTS.c5)
      _256 = torch.sin(_255)
      _257 = torch.slice(torch.slice(pe_negative3), 1, 0, None, 2)
      _258 = torch.copy_(_257, _256)
      _259 = torch.cos(_255)
      _260 = torch.slice(torch.slice(pe_negative3), 1, 1, None, 2)
      _261 = torch.copy_(_260, _259)
      pe_positive4 = torch.unsqueeze(torch.flip(pe_positive3, [0]), 0)
      pe_negative4 = torch.unsqueeze(torch.slice(pe_negative3, 0, 1), 0)
      pe16 = torch.cat([pe_positive4, pe_negative4], 1)
      _262 = torch.to(pe16, ops.prim.device(input1), ops.prim.dtype(input1))
      pos_enc1.pe = _262
    x37 = torch.mul(input1, 22.627416997969522)
    _263 = torch.size(x37, 1)
    pe17 = pos_enc1.pe
    _264 = torch.slice(pe17)
    pe18 = pos_enc1.pe
    _265 = torch.floordiv(torch.size(pe18, 1), 2)
    _266 = torch.add(torch.sub(_265, _263), 1)
    pe19 = pos_enc1.pe
    _267 = torch.floordiv(torch.size(pe19, 1), 2)
    pos_emb0 = torch.slice(_264, 1, _266, torch.add(_267, _263))
    if _57:
      _268 = 50
    else:
      _268 = 0
    if torch.gt(_268, 0):
      pos_idx0 = torch.arange(torch.size(x37, 1), dtype=None, layout=None, device=ops.prim.device(x37))
      _269 = torch.div(pos_idx0, _268, rounding_mode="trunc")
      block_value0 = torch.mul(torch.add(_269, 1), _268)
      chunk_masks4 = torch.lt(torch.unsqueeze(pos_idx0, 0), torch.unsqueeze(block_value0, 1))
      chunk_masks5 = torch.unsqueeze(chunk_masks4, 0)
      chunk_masks6 = torch.__and__(masks0, chunk_masks5)
      chunk_masks3 = chunk_masks6
    else:
      chunk_masks3 = masks0
    _270 = torch.eq(ops.prim.dtype(chunk_masks3), 11)
    if _270:
      pass
    else:
      ops.prim.RaiseException("AssertionError: ")
    _271 = torch.eq(torch.sum(chunk_masks3, [-1]), 0)
    _272 = torch.ne(torch.item(torch.sum(_271)), 0)
    if _272:
      print(_0)
      _273 = torch.eq(torch.sum(chunk_masks3, [-1]), 0)
      _274 = torch.tensor(True, dtype=ops.prim.dtype(chunk_masks3), device=ops.prim.device(chunk_masks3))
      _275 = annotate(List[Optional[Tensor]], [_273])
      _276 = torch.index_put_(chunk_masks3, _275, _274)
    else:
      pass
    x38 = torch.layer_norm(x37, [512], CONSTANTS.c106, CONSTANTS.c107, 9.9999999999999998e-13)
    n_batch11 = torch.size(x38, 0)
    _277 = torch.add(torch.matmul(x38, CONSTANTS.c108), CONSTANTS.c109)
    _278 = torch.slice(_277, -1, 1024, 1536)
    _279 = torch.slice(_277, -1, 512, 1024)
    _280 = torch.slice(_277, -1, 0, 512)
    _281 = [n_batch11, -1, 8, 64]
    q17 = torch.view(_280, _281)
    k11 = torch.view(_279, _281)
    v11 = torch.view(_278, _281)
    q18 = torch.transpose(q17, 1, 2)
    k12 = torch.transpose(k11, 1, 2)
    v12 = torch.transpose(v11, 1, 2)
    q19 = torch.transpose(q18, 1, 2)
    n_batch_pos5 = torch.size(pos_emb0, 0)
    _282 = torch.matmul(pos_emb0, CONSTANTS.c110)
    p11 = torch.view(_282, [n_batch_pos5, -1, 8, 64])
    p12 = torch.transpose(p11, 1, 2)
    q_with_bias_u5 = torch.transpose(torch.add(q19, CONSTANTS.c111), 1, 2)
    q_with_bias_v5 = torch.transpose(torch.add(q19, CONSTANTS.c112), 1, 2)
    matrix_ac5 = torch.matmul(q_with_bias_u5, torch.transpose(k12, -2, -1))
    matrix_bd17 = torch.matmul(q_with_bias_v5, torch.transpose(p12, -2, -1))
    _283 = torch.size(matrix_ac5)
    _284 = torch.size(matrix_bd17)
    if torch.ne(_283, _284):
      _285 = _284[0]
      _286 = _284[1]
      _287 = _284[2]
      _288 = ops.prim.device(matrix_bd17)
      _289 = ops.prim.dtype(matrix_bd17)
      zero_pad5 = torch.zeros([_285, _286, _287, 1], dtype=_289, layout=None, device=_288)
      x_padded11 = torch.cat([zero_pad5, matrix_bd17], -1)
      _290 = torch.add(torch.size(matrix_bd17, 3), 1)
      _291 = [_285, _286, _290, torch.size(matrix_bd17, 2)]
      x_padded12 = torch.view(x_padded11, _291)
      _292 = torch.slice(torch.slice(x_padded12), 1)
      _293 = torch.view_as(torch.slice(_292, 2, 1), matrix_bd17)
      _294 = torch.slice(torch.slice(torch.slice(_293), 1), 2)
      _295 = torch.floordiv(torch.size(matrix_bd17, -1), 2)
      matrix_bd19 = torch.slice(_294, 3, None, torch.add(_295, 1))
      matrix_bd18 = matrix_bd19
    else:
      matrix_bd18 = matrix_bd17
    scores11 = torch.div(torch.add(matrix_ac5, matrix_bd18), 8.)
    n_batch12 = torch.size(v12, 0)
    _296 = torch.gt(torch.size(chunk_masks3, 2), 0)
    if _296:
      mask13 = torch.eq(torch.unsqueeze(chunk_masks3, 1), 0)
      _297 = torch.slice(torch.slice(mask13), 1)
      mask14 = torch.slice(torch.slice(_297, 2), 3, None, torch.size(scores11, -1))
      scores12 = torch.masked_fill(scores11, mask14, -inf)
      attn12 = torch.masked_fill(torch.softmax(scores12, -1), mask14, 0.)
      attn11 = attn12
    else:
      attn11 = torch.softmax(scores11, -1)
    x39 = torch.matmul(attn11, v12)
    _298 = torch.contiguous(torch.transpose(x39, 1, 2))
    x40 = torch.view(_298, [n_batch12, -1, 512])
    _299 = torch.add(torch.matmul(x40, CONSTANTS.c113), CONSTANTS.c114)
    x41 = torch.add(x37, _299)
    x42 = torch.layer_norm(x41, [512], CONSTANTS.c115, CONSTANTS.c116, 9.9999999999999998e-13)
    _300 = torch.add(torch.matmul(x42, CONSTANTS.c117), CONSTANTS.c118)
    _301 = torch.matmul(torch.silu(_300), CONSTANTS.c119)
    _302 = torch.mul(torch.add(_301, CONSTANTS.c120), 1.)
    x43 = torch.add(x41, _302)
    x44 = torch.layer_norm(x43, [512], CONSTANTS.c121, CONSTANTS.c122, 9.9999999999999998e-13)
    n_batch13 = torch.size(x44, 0)
    _303 = torch.add(torch.matmul(x44, CONSTANTS.c123), CONSTANTS.c124)
    _304 = torch.slice(_303, -1, 1024, 1536)
    _305 = torch.slice(_303, -1, 512, 1024)
    _306 = torch.slice(_303, -1, 0, 512)
    _307 = [n_batch13, -1, 8, 64]
    q20 = torch.view(_306, _307)
    k13 = torch.view(_305, _307)
    v13 = torch.view(_304, _307)
    q21 = torch.transpose(q20, 1, 2)
    k14 = torch.transpose(k13, 1, 2)
    v14 = torch.transpose(v13, 1, 2)
    q22 = torch.transpose(q21, 1, 2)
    n_batch_pos6 = torch.size(pos_emb0, 0)
    _308 = torch.matmul(pos_emb0, CONSTANTS.c125)
    p13 = torch.view(_308, [n_batch_pos6, -1, 8, 64])
    p14 = torch.transpose(p13, 1, 2)
    q_with_bias_u6 = torch.transpose(torch.add(q22, CONSTANTS.c126), 1, 2)
    q_with_bias_v6 = torch.transpose(torch.add(q22, CONSTANTS.c127), 1, 2)
    matrix_ac6 = torch.matmul(q_with_bias_u6, torch.transpose(k14, -2, -1))
    matrix_bd20 = torch.matmul(q_with_bias_v6, torch.transpose(p14, -2, -1))
    _309 = torch.size(matrix_ac6)
    _310 = torch.size(matrix_bd20)
    if torch.ne(_309, _310):
      _311 = _310[0]
      _312 = _310[1]
      _313 = _310[2]
      _314 = ops.prim.device(matrix_bd20)
      _315 = ops.prim.dtype(matrix_bd20)
      zero_pad6 = torch.zeros([_311, _312, _313, 1], dtype=_315, layout=None, device=_314)
      x_padded13 = torch.cat([zero_pad6, matrix_bd20], -1)
      _316 = torch.add(torch.size(matrix_bd20, 3), 1)
      _317 = [_311, _312, _316, torch.size(matrix_bd20, 2)]
      x_padded14 = torch.view(x_padded13, _317)
      _318 = torch.slice(torch.slice(x_padded14), 1)
      _319 = torch.view_as(torch.slice(_318, 2, 1), matrix_bd20)
      _320 = torch.slice(torch.slice(torch.slice(_319), 1), 2)
      _321 = torch.floordiv(torch.size(matrix_bd20, -1), 2)
      matrix_bd22 = torch.slice(_320, 3, None, torch.add(_321, 1))
      matrix_bd21 = matrix_bd22
    else:
      matrix_bd21 = matrix_bd20
    scores13 = torch.div(torch.add(matrix_ac6, matrix_bd21), 8.)
    n_batch14 = torch.size(v14, 0)
    _322 = torch.gt(torch.size(chunk_masks3, 2), 0)
    if _322:
      mask15 = torch.eq(torch.unsqueeze(chunk_masks3, 1), 0)
      _323 = torch.slice(torch.slice(mask15), 1)
      mask16 = torch.slice(torch.slice(_323, 2), 3, None, torch.size(scores13, -1))
      scores14 = torch.masked_fill(scores13, mask16, -inf)
      attn14 = torch.masked_fill(torch.softmax(scores14, -1), mask16, 0.)
      attn13 = attn14
    else:
      attn13 = torch.softmax(scores13, -1)
    x45 = torch.matmul(attn13, v14)
    _324 = torch.contiguous(torch.transpose(x45, 1, 2))
    x46 = torch.view(_324, [n_batch14, -1, 512])
    _325 = torch.add(torch.matmul(x46, CONSTANTS.c128), CONSTANTS.c129)
    x47 = torch.add(x43, _325)
    x48 = torch.layer_norm(x47, [512], CONSTANTS.c130, CONSTANTS.c131, 9.9999999999999998e-13)
    _326 = torch.add(torch.matmul(x48, CONSTANTS.c132), CONSTANTS.c133)
    _327 = torch.matmul(torch.silu(_326), CONSTANTS.c134)
    _328 = torch.mul(torch.add(_327, CONSTANTS.c135), 1.)
    x49 = torch.add(x47, _328)
    x50 = torch.layer_norm(x49, [512], CONSTANTS.c136, CONSTANTS.c137, 9.9999999999999998e-13)
    n_batch15 = torch.size(x50, 0)
    _329 = torch.add(torch.matmul(x50, CONSTANTS.c138), CONSTANTS.c139)
    _330 = torch.slice(_329, -1, 1024, 1536)
    _331 = torch.slice(_329, -1, 512, 1024)
    _332 = torch.slice(_329, -1, 0, 512)
    _333 = [n_batch15, -1, 8, 64]
    q23 = torch.view(_332, _333)
    k15 = torch.view(_331, _333)
    v15 = torch.view(_330, _333)
    q24 = torch.transpose(q23, 1, 2)
    k16 = torch.transpose(k15, 1, 2)
    v16 = torch.transpose(v15, 1, 2)
    q25 = torch.transpose(q24, 1, 2)
    n_batch_pos7 = torch.size(pos_emb0, 0)
    _334 = torch.matmul(pos_emb0, CONSTANTS.c140)
    p15 = torch.view(_334, [n_batch_pos7, -1, 8, 64])
    p16 = torch.transpose(p15, 1, 2)
    q_with_bias_u7 = torch.transpose(torch.add(q25, CONSTANTS.c141), 1, 2)
    q_with_bias_v7 = torch.transpose(torch.add(q25, CONSTANTS.c142), 1, 2)
    matrix_ac7 = torch.matmul(q_with_bias_u7, torch.transpose(k16, -2, -1))
    matrix_bd23 = torch.matmul(q_with_bias_v7, torch.transpose(p16, -2, -1))
    _335 = torch.size(matrix_ac7)
    _336 = torch.size(matrix_bd23)
    if torch.ne(_335, _336):
      _337 = _336[0]
      _338 = _336[1]
      _339 = _336[2]
      _340 = ops.prim.device(matrix_bd23)
      _341 = ops.prim.dtype(matrix_bd23)
      zero_pad7 = torch.zeros([_337, _338, _339, 1], dtype=_341, layout=None, device=_340)
      x_padded15 = torch.cat([zero_pad7, matrix_bd23], -1)
      _342 = torch.add(torch.size(matrix_bd23, 3), 1)
      _343 = [_337, _338, _342, torch.size(matrix_bd23, 2)]
      x_padded16 = torch.view(x_padded15, _343)
      _344 = torch.slice(torch.slice(x_padded16), 1)
      _345 = torch.view_as(torch.slice(_344, 2, 1), matrix_bd23)
      _346 = torch.slice(torch.slice(torch.slice(_345), 1), 2)
      _347 = torch.floordiv(torch.size(matrix_bd23, -1), 2)
      matrix_bd25 = torch.slice(_346, 3, None, torch.add(_347, 1))
      matrix_bd24 = matrix_bd25
    else:
      matrix_bd24 = matrix_bd23
    scores15 = torch.div(torch.add(matrix_ac7, matrix_bd24), 8.)
    n_batch16 = torch.size(v16, 0)
    _348 = torch.gt(torch.size(chunk_masks3, 2), 0)
    if _348:
      mask17 = torch.eq(torch.unsqueeze(chunk_masks3, 1), 0)
      _349 = torch.slice(torch.slice(mask17), 1)
      mask18 = torch.slice(torch.slice(_349, 2), 3, None, torch.size(scores15, -1))
      scores16 = torch.masked_fill(scores15, mask18, -inf)
      attn16 = torch.masked_fill(torch.softmax(scores16, -1), mask18, 0.)
      attn15 = attn16
    else:
      attn15 = torch.softmax(scores15, -1)
    x51 = torch.matmul(attn15, v16)
    _350 = torch.contiguous(torch.transpose(x51, 1, 2))
    x52 = torch.view(_350, [n_batch16, -1, 512])
    _351 = torch.add(torch.matmul(x52, CONSTANTS.c143), CONSTANTS.c144)
    x53 = torch.add(x49, _351)
    x54 = torch.layer_norm(x53, [512], CONSTANTS.c145, CONSTANTS.c146, 9.9999999999999998e-13)
    _352 = torch.add(torch.matmul(x54, CONSTANTS.c147), CONSTANTS.c148)
    _353 = torch.matmul(torch.silu(_352), CONSTANTS.c149)
    _354 = torch.mul(torch.add(_353, CONSTANTS.c150), 1.)
    x55 = torch.add(x53, _354)
    x56 = torch.layer_norm(x55, [512], CONSTANTS.c151, CONSTANTS.c152, 9.9999999999999998e-13)
    n_batch17 = torch.size(x56, 0)
    _355 = torch.add(torch.matmul(x56, CONSTANTS.c153), CONSTANTS.c154)
    _356 = torch.slice(_355, -1, 1024, 1536)
    _357 = torch.slice(_355, -1, 512, 1024)
    _358 = torch.slice(_355, -1, 0, 512)
    _359 = [n_batch17, -1, 8, 64]
    q26 = torch.view(_358, _359)
    k17 = torch.view(_357, _359)
    v17 = torch.view(_356, _359)
    q27 = torch.transpose(q26, 1, 2)
    k18 = torch.transpose(k17, 1, 2)
    v18 = torch.transpose(v17, 1, 2)
    q28 = torch.transpose(q27, 1, 2)
    n_batch_pos8 = torch.size(pos_emb0, 0)
    _360 = torch.matmul(pos_emb0, CONSTANTS.c155)
    p17 = torch.view(_360, [n_batch_pos8, -1, 8, 64])
    p18 = torch.transpose(p17, 1, 2)
    q_with_bias_u8 = torch.transpose(torch.add(q28, CONSTANTS.c156), 1, 2)
    q_with_bias_v8 = torch.transpose(torch.add(q28, CONSTANTS.c157), 1, 2)
    matrix_ac8 = torch.matmul(q_with_bias_u8, torch.transpose(k18, -2, -1))
    matrix_bd26 = torch.matmul(q_with_bias_v8, torch.transpose(p18, -2, -1))
    _361 = torch.size(matrix_ac8)
    _362 = torch.size(matrix_bd26)
    if torch.ne(_361, _362):
      _363 = _362[0]
      _364 = _362[1]
      _365 = _362[2]
      _366 = ops.prim.device(matrix_bd26)
      _367 = ops.prim.dtype(matrix_bd26)
      zero_pad8 = torch.zeros([_363, _364, _365, 1], dtype=_367, layout=None, device=_366)
      x_padded17 = torch.cat([zero_pad8, matrix_bd26], -1)
      _368 = torch.add(torch.size(matrix_bd26, 3), 1)
      _369 = [_363, _364, _368, torch.size(matrix_bd26, 2)]
      x_padded18 = torch.view(x_padded17, _369)
      _370 = torch.slice(torch.slice(x_padded18), 1)
      _371 = torch.view_as(torch.slice(_370, 2, 1), matrix_bd26)
      _372 = torch.slice(torch.slice(torch.slice(_371), 1), 2)
      _373 = torch.floordiv(torch.size(matrix_bd26, -1), 2)
      matrix_bd28 = torch.slice(_372, 3, None, torch.add(_373, 1))
      matrix_bd27 = matrix_bd28
    else:
      matrix_bd27 = matrix_bd26
    scores17 = torch.div(torch.add(matrix_ac8, matrix_bd27), 8.)
    n_batch18 = torch.size(v18, 0)
    _374 = torch.gt(torch.size(chunk_masks3, 2), 0)
    if _374:
      mask19 = torch.eq(torch.unsqueeze(chunk_masks3, 1), 0)
      _375 = torch.slice(torch.slice(mask19), 1)
      mask20 = torch.slice(torch.slice(_375, 2), 3, None, torch.size(scores17, -1))
      scores18 = torch.masked_fill(scores17, mask20, -inf)
      attn18 = torch.masked_fill(torch.softmax(scores18, -1), mask20, 0.)
      attn17 = attn18
    else:
      attn17 = torch.softmax(scores17, -1)
    x57 = torch.matmul(attn17, v18)
    _376 = torch.contiguous(torch.transpose(x57, 1, 2))
    x58 = torch.view(_376, [n_batch18, -1, 512])
    _377 = torch.add(torch.matmul(x58, CONSTANTS.c158), CONSTANTS.c159)
    x59 = torch.add(x55, _377)
    x60 = torch.layer_norm(x59, [512], CONSTANTS.c160, CONSTANTS.c161, 9.9999999999999998e-13)
    _378 = torch.add(torch.matmul(x60, CONSTANTS.c162), CONSTANTS.c163)
    _379 = torch.matmul(torch.silu(_378), CONSTANTS.c164)
    _380 = torch.mul(torch.add(_379, CONSTANTS.c165), 1.)
    x61 = torch.add(x59, _380)
    xs3 = torch.layer_norm(x61, [512], CONSTANTS.c166, CONSTANTS.c167)
    return (xs3, masks0)
