Upload 10 files

attention_mlp.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:079123a0377f9c2d9c3efe00032dedf38fe5bbcc88e72bbf6a16f78d8eac2636
+size 4204478

classifier.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09e06f21026255bb11042e450e59e0a4ed8058ba6982a9cc83305a285de65cf9
+size 931371

custom_interface.py

@@ -0,0 +1,157 @@
+import torch
+from speechbrain.inference.interfaces import Pretrained
+
+
+class CustomEncoderClassifier(Pretrained):
+    """A ready-to-use class for utterance-level classification (e.g, speaker-id,
+    language-id, emotion recognition, keyword spotting, etc).
+    The class assumes that an self-supervised encoder like wav2vec2/hubert and a classifier model
+    are defined in the yaml file. If you want to
+    convert the predicted index into a corresponding text label, please
+    provide the path of the label_encoder in a variable called 'lab_encoder_file'
+    within the yaml.
+    The class can be used either to run only the encoder (encode_batch()) to
+    extract embeddings or to run a classification step (classify_batch()).
+    ```
+    Example
+    -------
+    >>> import torchaudio
+    >>> from speechbrain.pretrained import EncoderClassifier
+    >>> # Model is downloaded from the speechbrain HuggingFace repo
+    >>> tmpdir = getfixture("tmpdir")
+    >>> classifier = EncoderClassifier.from_hparams(
+    ...     source="speechbrain/spkrec-ecapa-voxceleb",
+    ...     savedir=tmpdir,
+    ... )
+    >>> # Compute embeddings
+    >>> signal, fs = torchaudio.load("samples/audio_samples/example1.wav")
+    >>> embeddings =  classifier.encode_batch(signal)
+    >>> # Classification
+    >>> prediction =  classifier .classify_batch(signal)
+    """
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.similarity = torch.nn.CosineSimilarity(dim=-1, eps=1e-6)
+
+    def encode_batch(self, wavs, wav_lens=None, normalize=False):
+        """Encodes the input audio into a single vector embedding.
+        The waveforms should already be in the model's desired format.
+        You can call:
+        ``normalized = <this>.normalizer(signal, sample_rate)``
+        to get a correctly converted signal in most cases.
+        Arguments
+        ---------
+        wavs : torch.tensor
+            Batch of waveforms [batch, time, channels] or [batch, time]
+            depending on the model. Make sure the sample rate is fs=16000 Hz.
+        wav_lens : torch.tensor
+            Lengths of the waveforms relative to the longest one in the
+            batch, tensor of shape [batch]. The longest one should have
+            relative length 1.0 and others len(waveform) / max_length.
+            Used for ignoring padding.
+        normalize : bool
+            If True, it normalizes the embeddings with the statistics
+            contained in mean_var_norm_emb.
+        Returns
+        -------
+        torch.tensor
+            The encoded batch
+        """
+        # Manage single waveforms in input
+        if len(wavs.shape) == 1:
+            wavs = wavs.unsqueeze(0)
+
+        # Assign full length if wav_lens is not assigned
+        if wav_lens is None:
+            wav_lens = torch.ones(wavs.shape[0], device=self.device)
+
+        # Storing waveform in the specified device
+        wavs, wav_lens = wavs.to(self.device), wav_lens.to(self.device)
+        wavs = wavs.float()
+
+        with torch.no_grad():
+            self.hparams.codec.to(self.device).eval()
+            tokens, _, _ = self.hparams.codec(
+                wavs, wav_lens, **self.hparams.tokenizer_config
+            )
+            embeddings = self.mods.discrete_embedding_layer(tokens)
+            att_w = self.mods.attention_mlp(embeddings)
+            feats = torch.matmul(att_w.transpose(2, -1), embeddings).squeeze(-2)
+            embeddings = self.mods.embedding_model(feats, wav_lens)
+        return embeddings.squeeze(1)
+
+
+    def verify_batch(
+        self, wavs1, wavs2, wav1_lens=None, wav2_lens=None, threshold=0.25
+    ):
+        """Performs speaker verification with cosine distance.
+
+        It returns the score and the decision (0 different speakers,
+        1 same speakers).
+
+        Arguments
+        ---------
+        wavs1 : Torch.Tensor
+            torch.Tensor containing the speech waveform1 (batch, time).
+            Make sure the sample rate is fs=16000 Hz.
+        wavs2 : Torch.Tensor
+            torch.Tensor containing the speech waveform2 (batch, time).
+            Make sure the sample rate is fs=16000 Hz.
+        wav1_lens : Torch.Tensor
+            torch.Tensor containing the relative length for each sentence
+            in the length (e.g., [0.8 0.6 1.0])
+        wav2_lens : Torch.Tensor
+            torch.Tensor containing the relative length for each sentence
+            in the length (e.g., [0.8 0.6 1.0])
+        threshold : Float
+            Threshold applied to the cosine distance to decide if the
+            speaker is different (0) or the same (1).
+
+        Returns
+        -------
+        score
+            The score associated to the binary verification output
+            (cosine distance).
+        prediction
+            The prediction is 1 if the two signals in input are from the same
+            speaker and 0 otherwise.
+        """
+        emb1 = self.encode_batch(wavs1, wav1_lens, normalize=False)
+        emb2 = self.encode_batch(wavs2, wav2_lens, normalize=False)
+        score = self.similarity(emb1, emb2)
+        return score, score > threshold
+
+    def verify_files(self, path_x, path_y, **kwargs):
+        """Speaker verification with cosine distance
+
+        Returns the score and the decision (0 different speakers,
+        1 same speakers).
+
+        Arguments
+        ---------
+        path_x : str
+            Path to file x
+        path_y : str
+            Path to file y
+        **kwargs : dict
+            Arguments to ``load_audio``
+
+        Returns
+        -------
+        score
+            The score associated to the binary verification output
+            (cosine distance).
+        prediction
+            The prediction is 1 if the two signals in input are from the same
+            speaker and 0 otherwise.
+        """
+        waveform_x = self.load_audio(path_x, **kwargs)
+        waveform_y = self.load_audio(path_y, **kwargs)
+        # Fake batches:
+        batch_x = waveform_x.unsqueeze(0)
+        batch_y = waveform_y.unsqueeze(0)
+        # Verify:
+        score, decision = self.verify_batch(batch_x, batch_y)
+        # Squeeze:
+        return score[0], decision[0]

custom_model.py

@@ -0,0 +1,100 @@
+import torch
+
+
+class AttentionMLP(torch.nn.Module):
+    def __init__(self, input_dim, hidden_dim):
+        super(AttentionMLP, self).__init__()
+        self.layers = torch.nn.Sequential(
+            torch.nn.Linear(input_dim, hidden_dim),
+            torch.nn.ReLU(),
+            torch.nn.Linear(hidden_dim, 1, bias=False),
+        )
+
+    def forward(self, x):
+        x = self.layers(x)
+        att_w = torch.nn.functional.softmax(x, dim=2)
+        return att_w
+
+
+class Discrete_EmbeddingLayer(torch.nn.Module):
+    """This class handles embedding layers  for discrete tokens.
+
+    Arguments
+    ---------
+    num_codebooks: int ,
+        number of codebooks of the tokenizer.
+    vocab_size : int,
+        size of the dictionary of embeddings
+    emb_dim: int ,
+        the size of each embedding vector
+    pad_index: int (default: 0),
+        If specified, the entries at padding_idx do not contribute to the gradient.
+    init: boolean (default: False):
+        If set to True, init the embedding with the tokenizer embedding otherwise init randomly.
+    freeze: boolean (default: False)
+       If True, the embedding is frozen. If False, the model will be trained
+        alongside with the rest of the pipeline.
+
+    Example
+    -------
+    >>> from speechbrain.lobes.models.huggingface_transformers.encodec import Encodec
+    >>> model_hub = "facebook/encodec_24khz"
+    >>> save_path = "savedir"
+    >>> model = Encodec(model_hub, save_path)
+    >>> audio = torch.randn(4, 1000)
+    >>> length = torch.tensor([1.0, .5, .75, 1.0])
+    >>> tokens, emb = model.encode(audio, length)
+    >>> print(tokens.shape)
+    torch.Size([4, 4, 2])
+    >>> emb= Discrete_EmbeddingLayer(2, 1024, 1024)
+    >>> in_emb = emb(tokens)
+    >>> print(in_emb.shape)
+    torch.Size([4, 4, 2, 1024])
+    """
+
+    def __init__(
+        self,
+        num_codebooks,
+        vocab_size,
+        emb_dim,
+        pad_index=0,
+        init=False,
+        freeze=False,
+    ):
+        super(Discrete_EmbeddingLayer, self).__init__()
+        self.vocab_size = vocab_size
+        self.num_codebooks = num_codebooks
+        self.freeze = freeze
+        self.embedding = torch.nn.Embedding(
+            num_codebooks * vocab_size, emb_dim
+        ).requires_grad_(not self.freeze)
+        self.init = init
+
+    def init_embedding(self, weights):
+        with torch.no_grad():
+            self.embedding.weight = torch.nn.Parameter(weights)
+
+    def forward(self, in_tokens):
+        """Computes the embedding for discrete tokens.
+        a sample.
+
+        Arguments
+        ---------
+        in_tokens : torch.Tensor
+            A (Batch x Time x num_codebooks)
+            audio sample
+        Returns
+        -------
+        in_embs : torch.Tensor
+        """
+        with torch.set_grad_enabled(not self.freeze):
+            #  Add unique token IDs across diffrent codebooks by adding num_codebooks * vocab_size
+            in_tokens += torch.arange(
+                0,
+                self.num_codebooks * self.vocab_size,
+                self.vocab_size,
+                device=in_tokens.device,
+            )
+            # Forward Pass to embedding and
+            in_embs = self.embedding(in_tokens)
+            return in_embs

discrete_embedding_layer.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:87fbbc6ab429b57493baba626f7c26584ef559b7608be849f2d09ef53f7077ef
+size 24577457

embedding_model.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:99fff92b110bc038dc81ceee1bab1a61ebbb2e69d10de13929259516ba7f66da
+size 102646844

hyperparams.yaml

@@ -0,0 +1,129 @@
+# ############################################################################
+# Model: ECAPA big for Speaker verification
+# ############################################################################
+
+# Feature parameters
+n_mels: 80
+
+# Pretrain folder (HuggingFace)
+# pretrained_path: poonehmousavi/discrete_wavlm_spk_rec_ecapatdn
+pretrained_path: benchmarks/DASB/VoiceCeleb1/speaker_ver/temp
+# Output parameters
+out_n_neurons: 1211
+save_folder: tmp
+
+### Configuration for  discrete SSL model
+# ssl_model_type: hubert, wavlm, wav2vec2
+# ssl_hub: facebook/hubert-large-ll60k, microsoft/wavlm-large,  facebook/wav2vec2-large
+ssl_model_type: wavlm # hubert, wavml or wav2vec2
+ssl_hub: microsoft/wavlm-large
+ssl_folder: !ref <save_folder>/ssl_checkpoint
+kmeans_repo_id: speechbrain/SSL_Quantization
+kmeans_cache_dir: !ref <save_folder>/kmeans_checkpoint
+kmeans_dataset: LibriSpeech-100-360-500
+freeze_ssl: True
+freeze_feature_extractor: True
+num_clusters: 1000
+
+### Config for Tokenizer
+# Layer number should be among the supported layers for discrete SSL models(kmenas  model should be available for that layer)
+# ssl_layer_num: [3, 7, 12, 23]
+# deduplicate: [False, False, False, False]
+# bpe_tokenizer_path: [null , null,  null, null]
+ssl_layer_num: [1, 3, 7, 12, 18, 23]
+num_codebooks: 6
+deduplicate: [False, False, False, False, False, False]
+bpe_tokenizer_path: [null, null, null, null, null, null]
+sample_rate: 16000
+
+# Feature parameters
+encoder_dim: 1024
+# Modules
+tokenizer_config:
+    SSL_layers: !ref <ssl_layer_num>
+    deduplicates: !ref <deduplicate>
+    bpe_tokenizers: !ref <bpe_tokenizer_path>
+
+ssl_model: !apply:speechbrain.utils.hparams.choice
+    value: !ref <ssl_model_type>
+    choices:
+        wavlm: !new:speechbrain.lobes.models.huggingface_transformers.wavlm.WavLM
+            source: !ref <ssl_hub>
+            output_norm: False
+            freeze: !ref <freeze_ssl>
+            freeze_feature_extractor: !ref <freeze_feature_extractor>
+            output_all_hiddens: True
+            save_path: !ref <ssl_folder>
+        hubert: !new:speechbrain.lobes.models.huggingface_transformers.hubert.HuBERT
+            source: !ref <ssl_hub>
+            output_norm: False
+            freeze: !ref <freeze_ssl>
+            freeze_feature_extractor: !ref <freeze_feature_extractor>
+            output_all_hiddens: True
+            save_path: !ref <ssl_folder>
+        wav2vec2: !new:speechbrain.lobes.models.huggingface_transformers.wav2vec2.Wav2Vec2
+            source: !ref <ssl_hub>
+            output_norm: False
+            freeze: !ref <freeze_ssl>
+            freeze_feature_extractor: !ref <freeze_feature_extractor>
+            output_all_hiddens: True
+            save_path: !ref <ssl_folder>
+
+codec: !new:speechbrain.lobes.models.huggingface_transformers.discrete_ssl.DiscreteSSL
+    save_path: !ref <kmeans_cache_dir>
+    ssl_model: !ref <ssl_model>
+    kmeans_dataset: !ref <kmeans_dataset>
+    kmeans_repo_id: !ref <kmeans_repo_id>
+    num_clusters: !ref <num_clusters>
+
+discrete_embedding_layer: !new:custom_model.Discrete_EmbeddingLayer
+    num_codebooks: !ref <num_codebooks>
+    vocab_size: !ref <num_clusters>
+    emb_dim: !ref <encoder_dim>
+
+attention_mlp: !new:custom_model.AttentionMLP
+    input_dim: !ref <encoder_dim>
+    hidden_dim: !ref <encoder_dim>
+
+embedding_model: !new:speechbrain.lobes.models.ECAPA_TDNN.ECAPA_TDNN
+    input_size: !ref <encoder_dim>
+    channels: [1024, 1024, 1024, 1024, 3072]
+    kernel_sizes: [5, 3, 3, 3, 1]
+    dilations: [1, 2, 3, 4, 1]
+    groups: [1, 1, 1, 1, 1]
+    attention_channels: 128
+    lin_neurons: 192
+
+classifier: !new:speechbrain.lobes.models.ECAPA_TDNN.Classifier
+    input_size: 192
+    out_neurons: !ref <out_n_neurons>
+
+
+
+modules:
+    embedding_model: !ref <embedding_model>
+    classifier: !ref <classifier>
+    attention_mlp: !ref <attention_mlp>
+    codec: !ref <codec>
+    discrete_embedding_layer: !ref <discrete_embedding_layer>
+
+        
+label_encoder: !new:speechbrain.dataio.encoder.CategoricalEncoder
+
+        
+pretrainer: !new:speechbrain.utils.parameter_transfer.Pretrainer
+    loadables:
+        embedding_model: !ref <embedding_model>
+        classifier: !ref <classifier>
+        attention_mlp: !ref <attention_mlp>
+        discrete_embedding_layer: !ref <discrete_embedding_layer>
+        label_encoder: !ref <label_encoder>
+
+    paths:
+        embedding_model: !ref <pretrained_path>/embedding_model.ckpt
+        classifier: !ref <pretrained_path>/classifier.ckpt
+        attention_mlp: !ref <pretrained_path>/attention_mlp.ckpt
+        label_encoder: !ref <pretrained_path>/label_encoder.txt
+        discrete_embedding_layer: !ref <pretrained_path>/discrete_embedding_layer.ckpt
+
+

label_encoder.txt

@@ -0,0 +1,1213 @@
+'id11129' => 0
+'id10248' => 1
+'id11225' => 2
+'id10977' => 3
+'id11142' => 4
+'id10715' => 5
+'id11250' => 6
+'id10892' => 7
+'id11239' => 8
+'id11168' => 9
+'id10711' => 10
+'id10571' => 11
+'id10197' => 12
+'id10129' => 13
+'id11008' => 14
+'id10783' => 15
+'id10944' => 16
+'id10639' => 17
+'id11195' => 18
+'id10790' => 19
+'id10780' => 20
+'id10805' => 21
+'id10145' => 22
+'id10716' => 23
+'id10664' => 24
+'id10086' => 25
+'id10840' => 26
+'id10928' => 27
+'id10447' => 28
+'id10510' => 29
+'id11105' => 30
+'id10569' => 31
+'id10398' => 32
+'id10047' => 33
+'id10184' => 34
+'id10706' => 35
+'id10959' => 36
+'id10480' => 37
+'id10631' => 38
+'id10617' => 39
+'id10543' => 40
+'id11027' => 41
+'id10225' => 42
+'id10786' => 43
+'id10611' => 44
+'id10608' => 45
+'id10035' => 46
+'id10488' => 47
+'id10428' => 48
+'id11204' => 49
+'id10491' => 50
+'id10730' => 51
+'id10979' => 52
+'id11183' => 53
+'id10784' => 54
+'id10090' => 55
+'id11041' => 56
+'id10449' => 57
+'id10419' => 58
+'id10536' => 59
+'id11188' => 60
+'id10036' => 61
+'id11057' => 62
+'id10921' => 63
+'id11148' => 64
+'id11056' => 65
+'id10573' => 66
+'id10472' => 67
+'id10125' => 68
+'id10610' => 69
+'id10534' => 70
+'id10943' => 71
+'id10367' => 72
+'id11076' => 73
+'id10883' => 74
+'id10853' => 75
+'id10260' => 76
+'id10549' => 77
+'id10919' => 78
+'id10994' => 79
+'id10756' => 80
+'id10325' => 81
+'id11112' => 82
+'id10396' => 83
+'id10471' => 84
+'id10329' => 85
+'id10144' => 86
+'id10850' => 87
+'id10640' => 88
+'id11165' => 89
+'id10430' => 90
+'id10792' => 91
+'id10798' => 92
+'id10317' => 93
+'id10103' => 94
+'id10149' => 95
+'id10981' => 96
+'id11182' => 97
+'id10732' => 98
+'id10425' => 99
+'id10411' => 100
+'id10520' => 101
+'id11152' => 102
+'id10930' => 103
+'id10636' => 104
+'id10986' => 105
+'id10130' => 106
+'id10063' => 107
+'id11224' => 108
+'id10113' => 109
+'id10477' => 110
+'id10925' => 111
+'id10383' => 112
+'id10517' => 113
+'id10237' => 114
+'id10957' => 115
+'id10576' => 116
+'id11130' => 117
+'id10956' => 118
+'id10831' => 119
+'id11144' => 120
+'id10604' => 121
+'id10799' => 122
+'id10508' => 123
+'id10148' => 124
+'id10583' => 125
+'id10906' => 126
+'id11121' => 127
+'id10938' => 128
+'id10243' => 129
+'id10949' => 130
+'id10426' => 131
+'id11004' => 132
+'id11223' => 133
+'id10466' => 134
+'id10006' => 135
+'id10537' => 136
+'id10975' => 137
+'id11158' => 138
+'id10423' => 139
+'id10659' => 140
+'id11015' => 141
+'id10627' => 142
+'id10931' => 143
+'id10031' => 144
+'id10817' => 145
+'id11101' => 146
+'id10879' => 147
+'id10913' => 148
+'id10399' => 149
+'id10239' => 150
+'id10708' => 151
+'id10025' => 152
+'id10912' => 153
+'id11210' => 154
+'id10891' => 155
+'id10267' => 156
+'id10584' => 157
+'id10450' => 158
+'id10755' => 159
+'id10193' => 160
+'id10993' => 161
+'id11149' => 162
+'id10045' => 163
+'id10055' => 164
+'id10062' => 165
+'id10095' => 166
+'id10686' => 167
+'id10712' => 168
+'id10002' => 169
+'id10106' => 170
+'id10094' => 171
+'id10505' => 172
+'id10421' => 173
+'id10587' => 174
+'id10231' => 175
+'id10072' => 176
+'id11154' => 177
+'id10689' => 178
+'id10665' => 179
+'id10950' => 180
+'id10586' => 181
+'id10108' => 182
+'id10690' => 183
+'id10335' => 184
+'id10704' => 185
+'id11181' => 186
+'id10860' => 187
+'id10465' => 188
+'id10199' => 189
+'id11039' => 190
+'id10539' => 191
+'id10245' => 192
+'id10191' => 193
+'id10168' => 194
+'id11002' => 195
+'id10679' => 196
+'id10165' => 197
+'id10945' => 198
+'id10454' => 199
+'id11044' => 200
+'id10380' => 201
+'id11177' => 202
+'id10996' => 203
+'id11169' => 204
+'id10155' => 205
+'id10691' => 206
+'id11184' => 207
+'id10810' => 208
+'id10941' => 209
+'id10482' => 210
+'id11089' => 211
+'id10579' => 212
+'id11113' => 213
+'id10462' => 214
+'id10186' => 215
+'id11136' => 216
+'id10628' => 217
+'id10102' => 218
+'id11199' => 219
+'id10707' => 220
+'id10980' => 221
+'id10633' => 222
+'id10328' => 223
+'id11035' => 224
+'id10397' => 225
+'id10175' => 226
+'id11011' => 227
+'id10356' => 228
+'id10886' => 229
+'id10416' => 230
+'id11058' => 231
+'id10779' => 232
+'id10240' => 233
+'id11030' => 234
+'id10180' => 235
+'id10096' => 236
+'id10061' => 237
+'id10116' => 238
+'id11123' => 239
+'id10811' => 240
+'id10092' => 241
+'id10354' => 242
+'id10601' => 243
+'id10709' => 244
+'id11193' => 245
+'id10042' => 246
+'id11176' => 247
+'id10578' => 248
+'id10032' => 249
+'id10439' => 250
+'id10988' => 251
+'id10143' => 252
+'id10147' => 253
+'id11111' => 254
+'id10258' => 255
+'id11059' => 256
+'id11221' => 257
+'id10935' => 258
+'id10929' => 259
+'id11095' => 260
+'id10166' => 261
+'id11003' => 262
+'id10825' => 263
+'id10405' => 264
+'id10226' => 265
+'id10839' => 266
+'id10768' => 267
+'id10794' => 268
+'id10512' => 269
+'id10259' => 270
+'id11205' => 271
+'id10652' => 272
+'id10552' => 273
+'id10351' => 274
+'id10997' => 275
+'id10268' => 276
+'id10448' => 277
+'id10353' => 278
+'id11093' => 279
+'id10942' => 280
+'id10672' => 281
+'id10519' => 282
+'id10662' => 283
+'id11097' => 284
+'id10334' => 285
+'id10843' => 286
+'id11220' => 287
+'id10645' => 288
+'id10410' => 289
+'id10342' => 290
+'id10582' => 291
+'id10720' => 292
+'id10341' => 293
+'id10503' => 294
+'id10650' => 295
+'id11166' => 296
+'id11019' => 297
+'id10597' => 298
+'id11010' => 299
+'id10596' => 300
+'id10053' => 301
+'id10753' => 302
+'id10360' => 303
+'id11202' => 304
+'id10207' => 305
+'id10394' => 306
+'id10738' => 307
+'id11235' => 308
+'id10775' => 309
+'id10209' => 310
+'id10098' => 311
+'id10431' => 312
+'id10079' => 313
+'id11190' => 314
+'id10826' => 315
+'id10812' => 316
+'id10995' => 317
+'id10922' => 318
+'id10084' => 319
+'id10638' => 320
+'id10829' => 321
+'id10513' => 322
+'id10479' => 323
+'id11024' => 324
+'id10702' => 325
+'id11107' => 326
+'id10402' => 327
+'id10206' => 328
+'id10404' => 329
+'id10761' => 330
+'id11094' => 331
+'id10787' => 332
+'id10412' => 333
+'id10171' => 334
+'id11020' => 335
+'id10244' => 336
+'id10012' => 337
+'id11209' => 338
+'id10141' => 339
+'id10764' => 340
+'id10769' => 341
+'id10157' => 342
+'id10983' => 343
+'id10310' => 344
+'id11203' => 345
+'id11174' => 346
+'id10313' => 347
+'id10971' => 348
+'id11077' => 349
+'id10355' => 350
+'id10043' => 351
+'id10970' => 352
+'id11236' => 353
+'id11122' => 354
+'id10218' => 355
+'id10326' => 356
+'id10156' => 357
+'id10845' => 358
+'id11206' => 359
+'id10311' => 360
+'id10851' => 361
+'id10744' => 362
+'id10040' => 363
+'id10444' => 364
+'id10976' => 365
+'id10859' => 366
+'id10663' => 367
+'id10588' => 368
+'id11173' => 369
+'id10065' => 370
+'id10343' => 371
+'id10417' => 372
+'id10345' => 373
+'id10403' => 374
+'id10347' => 375
+'id10414' => 376
+'id10333' => 377
+'id10377' => 378
+'id10160' => 379
+'id10630' => 380
+'id10782' => 381
+'id10801' => 382
+'id10257' => 383
+'id10642' => 384
+'id11099' => 385
+'id10954' => 386
+'id10557' => 387
+'id10881' => 388
+'id10685' => 389
+'id10100' => 390
+'id10727' => 391
+'id10776' => 392
+'id10797' => 393
+'id10624' => 394
+'id10562' => 395
+'id10188' => 396
+'id10214' => 397
+'id10232' => 398
+'id10901' => 399
+'id10701' => 400
+'id10349' => 401
+'id11062' => 402
+'id10135' => 403
+'id10110' => 404
+'id11048' => 405
+'id10201' => 406
+'id10140' => 407
+'id10682' => 408
+'id10122' => 409
+'id10391' => 410
+'id10612' => 411
+'id11201' => 412
+'id10312' => 413
+'id10870' => 414
+'id11009' => 415
+'id10039' => 416
+'id10131' => 417
+'id10330' => 418
+'id10082' => 419
+'id10884' => 420
+'id11064' => 421
+'id10252' => 422
+'id10365' => 423
+'id10729' => 424
+'id10064' => 425
+'id11109' => 426
+'id10803' => 427
+'id10736' => 428
+'id10069' => 429
+'id11014' => 430
+'id10241' => 431
+'id10909' => 432
+'id10902' => 433
+'id11090' => 434
+'id10568' => 435
+'id10777' => 436
+'id10265' => 437
+'id10920' => 438
+'id10731' => 439
+'id10757' => 440
+'id10083' => 441
+'id11140' => 442
+'id10564' => 443
+'id11189' => 444
+'id11248' => 445
+'id10813' => 446
+'id10392' => 447
+'id10337' => 448
+'id10492' => 449
+'id10680' => 450
+'id10933' => 451
+'id10540' => 452
+'id10443' => 453
+'id10915' => 454
+'id10874' => 455
+'id11081' => 456
+'id10560' => 457
+'id11157' => 458
+'id10229' => 459
+'id11139' => 460
+'id10992' => 461
+'id10256' => 462
+'id10542' => 463
+'id10644' => 464
+'id10233' => 465
+'id10484' => 466
+'id11246' => 467
+'id10204' => 468
+'id10895' => 469
+'id10442' => 470
+'id10434' => 471
+'id10725' => 472
+'id10139' => 473
+'id10828' => 474
+'id10676' => 475
+'id10570' => 476
+'id11171' => 477
+'id10220' => 478
+'id10649' => 479
+'id11047' => 480
+'id11180' => 481
+'id10170' => 482
+'id10060' => 483
+'id10771' => 484
+'id11117' => 485
+'id11211' => 486
+'id10152' => 487
+'id11025' => 488
+'id10580' => 489
+'id11150' => 490
+'id11043' => 491
+'id10619' => 492
+'id11147' => 493
+'id10019' => 494
+'id10500' => 495
+'id11218' => 496
+'id11249' => 497
+'id10528' => 498
+'id10743' => 499
+'id11185' => 500
+'id10044' => 501
+'id10246' => 502
+'id10497' => 503
+'id10594' => 504
+'id11082' => 505
+'id10618' => 506
+'id10236' => 507
+'id10982' => 508
+'id10017' => 509
+'id10408' => 510
+'id10056' => 511
+'id10028' => 512
+'id11120' => 513
+'id10531' => 514
+'id10393' => 515
+'id10059' => 516
+'id10374' => 517
+'id10016' => 518
+'id10172' => 519
+'id10376' => 520
+'id10616' => 521
+'id10363' => 522
+'id10473' => 523
+'id11016' => 524
+'id11066' => 525
+'id10200' => 526
+'id10718' => 527
+'id10710' => 528
+'id10388' => 529
+'id10934' => 530
+'id10614' => 531
+'id10014' => 532
+'id10752' => 533
+'id10415' => 534
+'id11108' => 535
+'id10625' => 536
+'id10020' => 537
+'id10362' => 538
+'id11013' => 539
+'id11127' => 540
+'id11038' => 541
+'id10858' => 542
+'id10952' => 543
+'id10483' => 544
+'id10386' => 545
+'id10456' => 546
+'id10767' => 547
+'id11069' => 548
+'id11073' => 549
+'id10085' => 550
+'id10486' => 551
+'id11212' => 552
+'id11000' => 553
+'id11159' => 554
+'id11234' => 555
+'id10759' => 556
+'id10834' => 557
+'id10511' => 558
+'id10847' => 559
+'id11119' => 560
+'id10893' => 561
+'id10550' => 562
+'id11012' => 563
+'id10556' => 564
+'id10071' => 565
+'id10849' => 566
+'id10607' => 567
+'id11233' => 568
+'id10077' => 569
+'id10058' => 570
+'id10089' => 571
+'id10008' => 572
+'id10118' => 573
+'id10501' => 574
+'id10504' => 575
+'id10250' => 576
+'id10190' => 577
+'id10238' => 578
+'id10198' => 579
+'id10899' => 580
+'id11106' => 581
+'id11178' => 582
+'id10348' => 583
+'id10331' => 584
+'id10490' => 585
+'id10762' => 586
+'id10489' => 587
+'id10809' => 588
+'id11032' => 589
+'id10951' => 590
+'id10936' => 591
+'id11018' => 592
+'id10529' => 593
+'id10262' => 594
+'id10474' => 595
+'id10460' => 596
+'id10852' => 597
+'id11128' => 598
+'id10223' => 599
+'id10872' => 600
+'id11021' => 601
+'id10592' => 602
+'id10822' => 603
+'id11046' => 604
+'id10748' => 605
+'id10321' => 606
+'id10808' => 607
+'id11049' => 608
+'id10029' => 609
+'id10873' => 610
+'id10048' => 611
+'id10499' => 612
+'id10916' => 613
+'id10074' => 614
+'id10269' => 615
+'id10469' => 616
+'id10076' => 617
+'id10918' => 618
+'id10478' => 619
+'id10567' => 620
+'id10506' => 621
+'id10989' => 622
+'id10544' => 623
+'id10352' => 624
+'id10158' => 625
+'id10216' => 626
+'id11072' => 627
+'id10136' => 628
+'id10509' => 629
+'id10816' => 630
+'id10203' => 631
+'id10559' => 632
+'id10862' => 633
+'id11240' => 634
+'id10179' => 635
+'id10889' => 636
+'id10527' => 637
+'id10195' => 638
+'id11060' => 639
+'id10661' => 640
+'id10683' => 641
+'id10835' => 642
+'id10955' => 643
+'id10751' => 644
+'id10115' => 645
+'id10024' => 646
+'id10137' => 647
+'id10338' => 648
+'id10038' => 649
+'id10485' => 650
+'id10795' => 651
+'id11054' => 652
+'id10390' => 653
+'id10327' => 654
+'id10760' => 655
+'id10009' => 656
+'id11087' => 657
+'id10609' => 658
+'id10553' => 659
+'id10361' => 660
+'id10263' => 661
+'id10167' => 662
+'id10177' => 663
+'id10742' => 664
+'id10457' => 665
+'id10178' => 666
+'id10364' => 667
+'id11230' => 668
+'id10463' => 669
+'id10900' => 670
+'id10885' => 671
+'id11229' => 672
+'id10242' => 673
+'id10904' => 674
+'id10694' => 675
+'id10819' => 676
+'id11194' => 677
+'id10120' => 678
+'id11098' => 679
+'id10739' => 680
+'id10590' => 681
+'id11238' => 682
+'id10138' => 683
+'id10961' => 684
+'id10081' => 685
+'id10162' => 686
+'id10734' => 687
+'id11084' => 688
+'id10948' => 689
+'id10078' => 690
+'id10344' => 691
+'id10635' => 692
+'id10524' => 693
+'id10440' => 694
+'id10818' => 695
+'id11161' => 696
+'id11017' => 697
+'id10545' => 698
+'id10964' => 699
+'id10535' => 700
+'id10001' => 701
+'id11170' => 702
+'id10646' => 703
+'id11198' => 704
+'id10806' => 705
+'id10495' => 706
+'id10427' => 707
+'id10379' => 708
+'id10114' => 709
+'id10577' => 710
+'id10453' => 711
+'id10796' => 712
+'id10656' => 713
+'id11065' => 714
+'id10123' => 715
+'id11100' => 716
+'id10888' => 717
+'id10525' => 718
+'id11080' => 719
+'id10395' => 720
+'id10212' => 721
+'id11086' => 722
+'id10437' => 723
+'id10677' => 724
+'id10668' => 725
+'id10820' => 726
+'id10643' => 727
+'id10714' => 728
+'id11231' => 729
+'id10107' => 730
+'id10173' => 731
+'id11075' => 732
+'id10903' => 733
+'id10251' => 734
+'id11052' => 735
+'id11088' => 736
+'id10224' => 737
+'id10073' => 738
+'id10823' => 739
+'id10978' => 740
+'id10905' => 741
+'id10692' => 742
+'id10530' => 743
+'id10595' => 744
+'id10400' => 745
+'id10432' => 746
+'id10634' => 747
+'id11071' => 748
+'id10932' => 749
+'id10856' => 750
+'id10066' => 751
+'id10523' => 752
+'id10104' => 753
+'id10318' => 754
+'id11151' => 755
+'id10371' => 756
+'id10022' => 757
+'id10254' => 758
+'id10004' => 759
+'id10667' => 760
+'id10041' => 761
+'id10464' => 762
+'id10211' => 763
+'id10522' => 764
+'id10161' => 765
+'id10908' => 766
+'id10368' => 767
+'id10696' => 768
+'id11132' => 769
+'id10515' => 770
+'id10003' => 771
+'id10261' => 772
+'id11131' => 773
+'id10433' => 774
+'id10966' => 775
+'id11007' => 776
+'id10541' => 777
+'id11242' => 778
+'id10183' => 779
+'id11232' => 780
+'id10037' => 781
+'id11026' => 782
+'id10773' => 783
+'id10868' => 784
+'id11110' => 785
+'id10735' => 786
+'id11116' => 787
+'id10322' => 788
+'id10678' => 789
+'id10623' => 790
+'id10336' => 791
+'id10435' => 792
+'id10937' => 793
+'id10747' => 794
+'id10346' => 795
+'id10629' => 796
+'id10196' => 797
+'id11167' => 798
+'id10369' => 799
+'id10654' => 800
+'id10698' => 801
+'id10632' => 802
+'id10867' => 803
+'id11022' => 804
+'id10227' => 805
+'id10067' => 806
+'id10723' => 807
+'id11197' => 808
+'id10605' => 809
+'id10210' => 810
+'id10965' => 811
+'id11079' => 812
+'id10791' => 813
+'id10724' => 814
+'id10222' => 815
+'id11244' => 816
+'id10051' => 817
+'id10070' => 818
+'id10985' => 819
+'id10555' => 820
+'id10615' => 821
+'id10458' => 822
+'id10132' => 823
+'id11063' => 824
+'id10424' => 825
+'id10758' => 826
+'id10846' => 827
+'id10827' => 828
+'id10861' => 829
+'id10866' => 830
+'id10027' => 831
+'id10159' => 832
+'id10647' => 833
+'id10194' => 834
+'id10185' => 835
+'id11137' => 836
+'id10091' => 837
+'id10726' => 838
+'id11200' => 839
+'id10153' => 840
+'id10088' => 841
+'id10378' => 842
+'id10498' => 843
+'id10461' => 844
+'id11092' => 845
+'id10911' => 846
+'id10719' => 847
+'id11133' => 848
+'id10963' => 849
+'id10991' => 850
+'id10476' => 851
+'id10021' => 852
+'id10101' => 853
+'id10713' => 854
+'id11006' => 855
+'id11164' => 856
+'id10054' => 857
+'id10688' => 858
+'id11023' => 859
+'id11160' => 860
+'id10745' => 861
+'id10842' => 862
+'id10804' => 863
+'id10007' => 864
+'id11216' => 865
+'id10968' => 866
+'id10566' => 867
+'id11196' => 868
+'id10316' => 869
+'id10722' => 870
+'id10880' => 871
+'id10973' => 872
+'id11163' => 873
+'id11138' => 874
+'id11104' => 875
+'id10532' => 876
+'id10967' => 877
+'id10593' => 878
+'id10192' => 879
+'id10109' => 880
+'id11186' => 881
+'id10324' => 882
+'id10603' => 883
+'id10740' => 884
+'id10574' => 885
+'id10359' => 886
+'id10551' => 887
+'id10516' => 888
+'id10917' => 889
+'id10057' => 890
+'id10841' => 891
+'id10887' => 892
+'id11068' => 893
+'id10898' => 894
+'id10687' => 895
+'id10674' => 896
+'id10781' => 897
+'id10864' => 898
+'id11228' => 899
+'id10452' => 900
+'id10785' => 901
+'id10622' => 902
+'id10018' => 903
+'id10766' => 904
+'id10789' => 905
+'id10660' => 906
+'id11050' => 907
+'id11114' => 908
+'id11208' => 909
+'id10481' => 910
+'id10572' => 911
+'id10824' => 912
+'id11055' => 913
+'id10637' => 914
+'id10370' => 915
+'id10833' => 916
+'id10350' => 917
+'id10589' => 918
+'id10387' => 919
+'id10445' => 920
+'id10052' => 921
+'id10878' => 922
+'id10854' => 923
+'id10697' => 924
+'id10521' => 925
+'id10247' => 926
+'id10671' => 927
+'id11045' => 928
+'id10011' => 929
+'id10877' => 930
+'id10438' => 931
+'id10733' => 932
+'id10585' => 933
+'id10750' => 934
+'id10418' => 935
+'id10939' => 936
+'id10670' => 937
+'id10844' => 938
+'id10548' => 939
+'id11187' => 940
+'id10163' => 941
+'id11226' => 942
+'id10010' => 943
+'id11214' => 944
+'id11028' => 945
+'id10234' => 946
+'id10049' => 947
+'id10914' => 948
+'id10863' => 949
+'id10182' => 950
+'id10737' => 951
+'id11213' => 952
+'id10754' => 953
+'id11245' => 954
+'id10857' => 955
+'id10599' => 956
+'id10215' => 957
+'id11179' => 958
+'id10728' => 959
+'id10940' => 960
+'id10778' => 961
+'id10406' => 962
+'id10999' => 963
+'id10897' => 964
+'id10372' => 965
+'id10451' => 966
+'id10653' => 967
+'id11155' => 968
+'id10127' => 969
+'id11103' => 970
+'id10765' => 971
+'id10332' => 972
+'id10871' => 973
+'id11070' => 974
+'id10974' => 975
+'id10969' => 976
+'id10230' => 977
+'id10800' => 978
+'id10357' => 979
+'id11215' => 980
+'id10561' => 981
+'id10705' => 982
+'id10436' => 983
+'id10838' => 984
+'id10146' => 985
+'id10323' => 986
+'id11078' => 987
+'id10358' => 988
+'id11172' => 989
+'id10927' => 990
+'id10314' => 991
+'id11227' => 992
+'id10987' => 993
+'id10124' => 994
+'id10187' => 995
+'id10600' => 996
+'id10496' => 997
+'id11067' => 998
+'id10249' => 999
+'id10023' => 1000
+'id10673' => 1001
+'id10126' => 1002
+'id10563' => 1003
+'id10621' => 1004
+'id10401' => 1005
+'id10869' => 1006
+'id10319' => 1007
+'id11126' => 1008
+'id10097' => 1009
+'id11134' => 1010
+'id10373' => 1011
+'id10422' => 1012
+'id10554' => 1013
+'id11033' => 1014
+'id11125' => 1015
+'id10960' => 1016
+'id11085' => 1017
+'id11061' => 1018
+'id10033' => 1019
+'id10703' => 1020
+'id10907' => 1021
+'id11091' => 1022
+'id10441' => 1023
+'id10830' => 1024
+'id10151' => 1025
+'id11237' => 1026
+'id10005' => 1027
+'id10641' => 1028
+'id10315' => 1029
+'id10875' => 1030
+'id10189' => 1031
+'id10121' => 1032
+'id10112' => 1033
+'id11053' => 1034
+'id11096' => 1035
+'id10420' => 1036
+'id10080' => 1037
+'id10266' => 1038
+'id10150' => 1039
+'id10235' => 1040
+'id10546' => 1041
+'id10575' => 1042
+'id11143' => 1043
+'id10407' => 1044
+'id10793' => 1045
+'id10446' => 1046
+'id10111' => 1047
+'id10581' => 1048
+'id10099' => 1049
+'id11146' => 1050
+'id10695' => 1051
+'id10802' => 1052
+'id10684' => 1053
+'id10890' => 1054
+'id10774' => 1055
+'id10105' => 1056
+'id10026' => 1057
+'id10865' => 1058
+'id11074' => 1059
+'id10117' => 1060
+'id10602' => 1061
+'id10693' => 1062
+'id10923' => 1063
+'id10669' => 1064
+'id11102' => 1065
+'id10821' => 1066
+'id10547' => 1067
+'id10087' => 1068
+'id10651' => 1069
+'id10046' => 1070
+'id10514' => 1071
+'id11219' => 1072
+'id10763' => 1073
+'id10205' => 1074
+'id11124' => 1075
+'id10613' => 1076
+'id11031' => 1077
+'id11145' => 1078
+'id10366' => 1079
+'id10494' => 1080
+'id10699' => 1081
+'id11162' => 1082
+'id10217' => 1083
+'id10493' => 1084
+'id10876' => 1085
+'id11156' => 1086
+'id10648' => 1087
+'id11042' => 1088
+'id10264' => 1089
+'id10533' => 1090
+'id10389' => 1091
+'id10681' => 1092
+'id10788' => 1093
+'id10470' => 1094
+'id10174' => 1095
+'id10340' => 1096
+'id11192' => 1097
+'id10181' => 1098
+'id10606' => 1099
+'id10208' => 1100
+'id10176' => 1101
+'id11083' => 1102
+'id10924' => 1103
+'id10591' => 1104
+'id10894' => 1105
+'id10119' => 1106
+'id10721' => 1107
+'id11029' => 1108
+'id11241' => 1109
+'id10598' => 1110
+'id10998' => 1111
+'id10133' => 1112
+'id11247' => 1113
+'id10526' => 1114
+'id10221' => 1115
+'id10429' => 1116
+'id10030' => 1117
+'id10034' => 1118
+'id10620' => 1119
+'id11005' => 1120
+'id11036' => 1121
+'id10675' => 1122
+'id10518' => 1123
+'id11051' => 1124
+'id11141' => 1125
+'id10666' => 1126
+'id10990' => 1127
+'id10962' => 1128
+'id11118' => 1129
+'id11037' => 1130
+'id10075' => 1131
+'id10154' => 1132
+'id10972' => 1133
+'id10855' => 1134
+'id10487' => 1135
+'id10565' => 1136
+'id10409' => 1137
+'id10741' => 1138
+'id10142' => 1139
+'id10626' => 1140
+'id10700' => 1141
+'id10339' => 1142
+'id10202' => 1143
+'id11222' => 1144
+'id10749' => 1145
+'id10896' => 1146
+'id11243' => 1147
+'id10655' => 1148
+'id10837' => 1149
+'id10717' => 1150
+'id10013' => 1151
+'id10814' => 1152
+'id10169' => 1153
+'id10015' => 1154
+'id10467' => 1155
+'id10384' => 1156
+'id11040' => 1157
+'id10093' => 1158
+'id10836' => 1159
+'id10219' => 1160
+'id10050' => 1161
+'id11175' => 1162
+'id10382' => 1163
+'id11001' => 1164
+'id11034' => 1165
+'id10413' => 1166
+'id10958' => 1167
+'id11153' => 1168
+'id10984' => 1169
+'id11207' => 1170
+'id10910' => 1171
+'id10459' => 1172
+'id10213' => 1173
+'id10558' => 1174
+'id10253' => 1175
+'id10502' => 1176
+'id10947' => 1177
+'id10381' => 1178
+'id11251' => 1179
+'id10815' => 1180
+'id11115' => 1181
+'id10385' => 1182
+'id11217' => 1183
+'id10832' => 1184
+'id10770' => 1185
+'id10134' => 1186
+'id10455' => 1187
+'id10848' => 1188
+'id10807' => 1189
+'id10228' => 1190
+'id10882' => 1191
+'id11191' => 1192
+'id10164' => 1193
+'id10772' => 1194
+'id10538' => 1195
+'id10068' => 1196
+'id10375' => 1197
+'id11135' => 1198
+'id10255' => 1199
+'id10746' => 1200
+'id10657' => 1201
+'id10658' => 1202
+'id10946' => 1203
+'id10926' => 1204
+'id10507' => 1205
+'id10468' => 1206
+'id10128' => 1207
+'id10475' => 1208
+'id10953' => 1209
+'id10320' => 1210
+================
+'starting_index' => 0