How to find the which file is making loss inf

[JRMeyer]

>>> vigneshgig
[August 20, 2019, 5:29am]

I want to print or save it as a text log , which files is training on
which batch. slash
because i have huge data in that in some audio file making loss has inf
. slash
any suggestion

if any give me a hint about where i code print statement in the
Deepspeech.py slash
thanks,

[This is an archived TTS discussion thread from discourse.mozilla.org/t/how-to-find-the-which-file-is-making-loss-inf]

[JRMeyer]

>>> reuben
[August 20, 2019, 7:32am]

Code for doing this has been merged on master:
mozilla/DeepSpeech#2289

[Archived Post]

[JRMeyer]

>>> vigneshgig
[August 20, 2019, 10:27am]

hey it is alpha version it does not support both the checkpoint 0.5.1
and 0.5.0 slash
it showing error. slash
thanks

[Archived Post]

[JRMeyer]

>>> vigneshgig
[August 20, 2019, 10:30am]

and it taking so much time for initial transcript and alphabet process
compare to 0.5.1 version

[Archived Post]

[JRMeyer]

>>> vigneshgig
[August 20, 2019, 11:35am]

how can I add this code in 0.5.1 I just copied the alpha version
Deepspeech.py and pasted in 0.5.1 folder and i tried to run the train
but shown a error slash
alpha version

# Obtain the next batch of data

_, (batch_x, batch_seq_len), batch_y = iterator.get_next()
batch_filenames, (batch_x, batch_seq_len), batch_y = iterator.get_next()

0.5.1 slash
(batch_x, batch_seq_len), batch_y = iterator.get_next()

so where you coded to get the batch_filenames

any suggestion slash
thanks

[Archived Post]

[JRMeyer]

>>> reuben
[August 20, 2019, 11:55am]

Using this with 0.5.1 will be harder, you'll have to rebase that PR as
well as the previous one that added the filenames to the dataset. I
wouldn't recommend it if you're not familiar with git and our codebase.

Another alternative is to do a binary search on the data by splitting
the CSV in half and checking which half produces inf loss.

[Archived Post]

[JRMeyer]

>>> vigneshgig
[August 20, 2019, 2:03pm]

thanks but I trying to merge the alpha code of loss:inf check and 0.5.1
if succeed i let you know.

[Archived Post]

[JRMeyer]

>>> vigneshgig
[August 20, 2019, 2:50pm]

hey I tried it out it's working I modified the 0.5.1 code according to
the alpha code and I copied the alpha feeding.py and text.py to the
0.5.1 util folder .Now it's working fine. slash
but I tried to change the 0.5.1 feeding.py and text.py by refering the
alpha code feeding.py and text.py .but I got error which is batch_fn
expected three but get four argument. which is wav_filenames.

alpha code

# -- coding: utf-8 --

from future import absolute_import, division, print_function

import os

from functools import partial

import numpy as np slash
import pandas slash
import tensorflow as tf slash
import datetime

from tensorflow.contrib.framework.python.ops import audio_ops as
contrib_audio

from util.config import Config slash
from util.logging import log_error slash
from util.text import text_to_char_array

def read_csvs(csv_files): slash
source_data = None slash
for csv in csv_files: slash
file = pandas.read_csv(csv, encoding='utf-8', na_filter=False) slash
[ slash #FIXME:] not cross-platform slash
csv_dir = os.path.dirname(os.path.abspath(csv)) slash
file slash ['wav_filename' slash ] =
file slash ['wav_filename' slash ].str.replace(r'( slash ^ slash [ slash ^/ slash ])', lambda m:
os.path.join(csv_dir, m.group(1))) slash # pylint:
disable=cell-var-from-loop slash
if source_data is None: slash
source_data = file slash
else: slash
source_data = source_data.append(file) slash
return source_data

def samples_to_mfccs(samples, sample_rate): slash
spectrogram = contrib_audio.audio_spectrogram(samples, slash
window_size=Config.audio_window_samples, slash
stride=Config.audio_step_samples, slash
magnitude_squared=True) slash
mfccs = contrib_audio.mfcc(spectrogram, sample_rate,
dct_coefficient_count=Config.n_input) slash
mfccs = tf.reshape(mfccs, slash [-1, Config.n_input slash ])

return mfccs, tf.shape(mfccs)[0]

def audiofile_to_features(wav_filename): slash
samples = tf.io.read_file(wav_filename) slash
decoded = contrib_audio.decode_wav(samples, desired_channels=1) slash
features, features_len = samples_to_mfccs(decoded.audio,
decoded.sample_rate)

return features, features_len

def entry_to_features(wav_filename, transcript): slash
slash # https://bugs.python.org/issue32117 slash
features, features_len = audiofile_to_features(wav_filename) slash
return wav_filename, features, features_len,
tf.SparseTensor( slash *transcript)

def to_sparse_tuple(sequence): slash
r slash ' slash ''Creates a sparse representention of sequence. slash
Returns a tuple with (indices, values, shape) slash
''' slash
indices = np.asarray(list(zip( slash [0 slash ] slash len(sequence),
range(len(sequence)))), dtype=np.int64) slash
shape = np.asarray( slash [1, len(sequence) slash ], dtype=np.int64) slash
return indices, sequence, shape

def create_dataset(csvs, batch_size, cache_path=''): slash
df = read_csvs(csvs) slash
df.sort_values(by='wav_filesize', inplace=True)

try:
# Convert to character index arrays
df = df.apply(partial(text_to_char_array, alphabet=Config.alphabet), result_type='broadcast', axis=1)
except ValueError as e:
error_message, series, _ = e.args
log_error('While processing {}: slash n {}'.format(series['wav_filename'], error_message))
exit(1)

def generate_values():
for _, row in df.iterrows():
yield row.wav_filename, to_sparse_tuple(row.transcript)

# Batching a dataset of 2D SparseTensors creates 3D batches, which fail
# when passed to tf.nn.ctc_loss, so we reshape them to remove the extra
# dimension here.
def sparse_reshape(sparse):
shape = sparse.dense_shape
return tf.sparse.reshape(sparse, [shape[0], shape[2]])

def batch_fn(wav_filenames, features, features_len, transcripts):
features = tf.data.Dataset.zip((features, features_len))
features = features.padded_batch(batch_size,
padded_shapes=([None, Config.n_input], []))
transcripts = transcripts.batch(batch_size).map(sparse_reshape)
wav_filenames = wav_filenames.batch(batch_size)
return tf.data.Dataset.zip((wav_filenames, features, transcripts))

num_gpus = len(Config.available_devices)

dataset = (tf.data.Dataset.from_generator(generate_values,
output_types=(tf.string, (tf.int64, tf.int32, tf.int64)))
.map(entry_to_features, num_parallel_calls=tf.data.experimental.AUTOTUNE)
.cache(cache_path)
.window(batch_size, drop_remainder=True).flat_map(batch_fn)
.prefetch(num_gpus))

return dataset

def secs_to_hours(secs): slash
hours, remainder = divmod(secs, 3600) slash
minutes, seconds = divmod(remainder, 60) slash
return '%d:%02d:%02d' % (hours, minutes, seconds)

0.5.1

# -- coding: utf-8 --

from future import absolute_import, division, print_function

import os

from functools import partial

import numpy as np slash
import pandas slash
import tensorflow as tf slash
import datetime

from tensorflow.contrib.framework.python.ops import audio_ops as
contrib_audio

from util.config import Config slash
from util.logging import log_error slash
from util.text import text_to_char_array

def read_csvs(csv_files): slash
source_data = None slash
for csv in csv_files: slash
file = pandas.read_csv(csv, encoding='utf-8', na_filter=False) slash
[ slash #FIXME:] not cross-platform slash
csv_dir = os.path.dirname(os.path.abspath(csv)) slash
file slash ['wav_filename' slash ] =
file slash ['wav_filename' slash ].str.replace(r'( slash ^ slash [ slash ^/ slash ])', lambda m:
os.path.join(csv_dir, m.group(1))) slash # pylint:
disable=cell-var-from-loop slash
if source_data is None: slash
source_data = file slash
else: slash
source_data = source_data.append(file) slash
return source_data

def samples_to_mfccs(samples, sample_rate): slash
spectrogram = contrib_audio.audio_spectrogram(samples, slash
window_size=Config.audio_window_samples, slash
stride=Config.audio_step_samples, slash
magnitude_squared=True) slash
mfccs = contrib_audio.mfcc(spectrogram, sample_rate,
dct_coefficient_count=Config.n_input) slash
mfccs = tf.reshape(mfccs, slash [-1, Config.n_input slash ])

return mfccs, tf.shape(mfccs)[0]

def audiofile_to_features(wav_filename): slash
samples = tf.read_file(wav_filename) slash
decoded = contrib_audio.decode_wav(samples, desired_channels=1) slash
features, features_len = samples_to_mfccs(decoded.audio,
decoded.sample_rate)

return features, features_len

def entry_to_features(wav_filename, transcript): slash
slash # https://bugs.python.org/issue32117 slash
features, features_len = audiofile_to_features(wav_filename) slash
return wav_filename, features, features_len,
tf.SparseTensor( slash transcript)

def to_sparse_tuple(sequence): slash
r slash ' slash ''Creates a sparse representention of sequence. slash
Returns a tuple with (indices, values, shape) slash
''' slash
indices = np.asarray(list(zip( slash [0 slash ] slash len(sequence),
range(len(sequence)))), dtype=np.int64) slash
shape = np.asarray( slash [1, len(sequence) slash ], dtype=np.int64) slash
return indices, sequence, shape

def create_dataset(csvs, batch_size, cache_path=''): slash
df = read_csvs(csvs) slash
df.sort_values(by='wav_filesize', inplace=True) slash
print(';;;;;;;;;;;;;;;;;;;;;;;;;')

# Convert to character index arrays
df['transcript'] = df['transcript'].apply(partial(text_to_char_array, alphabet=Config.alphabet))

def generate_values():
for _, row in df.iterrows():
yield row.wav_filename, to_sparse_tuple(row.transcript)

# Batching a dataset of 2D SparseTensors creates 3D batches, which fail
# when passed to tf.nn.ctc_loss, so we reshape them to remove the extra
# dimension here.
def sparse_reshape(sparse):
shape = sparse.dense_shape
return tf.sparse.reshape(sparse, [shape[0], shape[2]])

def batch_fn(wav_filenames,features, features_len, transcripts):
print('.....................................')
features = tf.data.Dataset.zip((features, features_len))
features = features.padded_batch(batch_size,
padded_shapes=([None, Config.n_input], []))
transcripts = transcripts.batch(batch_size).map(sparse_reshape)
wav_filenames = wav_filenames.batch(batch_size)
return tf.data.Dataset.zip((wav_filenames,features, transcripts))

num_gpus = len(Config.available_devices)

dataset = (tf.data.Dataset.from_generator(generate_values,
output_types=(tf.string, (tf.int64, tf.int32, tf.int64)))
.map(entry_to_features, num_parallel_calls=tf.data.experimental.AUTOTUNE)
.cache(cache_path)
.window(batch_size, drop_remainder=True).flat_map(batch_fn)
.prefetch(num_gpus))

return dataset

def secs_to_hours(secs): slash
hours, remainder = divmod(secs, 3600) slash
minutes, seconds = divmod(remainder, 60) slash
return '%d:%02d:%02d' % (hours, minutes, seconds)

0.5.1 original code

# -- coding: utf-8 --

from future import absolute_import, division, print_function

import os

from functools import partial

import numpy as np slash
import pandas slash
import tensorflow as tf slash
import datetime

from tensorflow.contrib.framework.python.ops import audio_ops as
contrib_audio

from util.config import Config slash
from util.text import text_to_char_array

def read_csvs(csv_files): slash
source_data = None slash
for csv in csv_files: slash
file = pandas.read_csv(csv, encoding='utf-8', na_filter=False) slash
[ slash #FIXME:] not cross-platform slash
csv_dir = os.path.dirname(os.path.abspath(csv)) slash
file slash ['wav_filename' slash ] =
file slash ['wav_filename' slash ].str.replace(r'( slash ^ slash [ slash ^/ slash ])', lambda m:
os.path.join(csv_dir, m.group(1))) slash # pylint:
disable=cell-var-from-loop slash
if source_data is None: slash
source_data = file slash
else: slash
source_data = source_data.append(file) slash
return source_data

def samples_to_mfccs(samples, sample_rate): slash
spectrogram = contrib_audio.audio_spectrogram(samples, slash
window_size=Config.audio_window_samples, slash
stride=Config.audio_step_samples, slash
magnitude_squared=True) slash
mfccs = contrib_audio.mfcc(spectrogram, sample_rate,
dct_coefficient_count=Config.n_input) slash
mfccs = tf.reshape(mfccs, slash [-1, Config.n_input slash ])

return mfccs, tf.shape(mfccs)[0]

def audiofile_to_features(wav_filename): slash
samples = tf.read_file(wav_filename) slash
decoded = contrib_audio.decode_wav(samples, desired_channels=1) slash
features, features_len = samples_to_mfccs(decoded.audio,
decoded.sample_rate)

return features, features_len

def entry_to_features(wav_filename, transcript): slash
slash # https://bugs.python.org/issue32117 slash
features, features_len = audiofile_to_features(wav_filename) slash
return features, features_len, tf.SparseTensor( slash *transcript)

def to_sparse_tuple(sequence): slash
r slash ' slash ''Creates a sparse representention of sequence. slash
Returns a tuple with (indices, values, shape) slash
''' slash
indices = np.asarray(list(zip( slash [0 slash ] slash *len(sequence),
range(len(sequence)))), dtype=np.int64) slash
shape = np.asarray( slash [1, len(sequence) slash ], dtype=np.int64) slash
return indices, sequence, shape

def create_dataset(csvs, batch_size, cache_path=''): slash
df = read_csvs(csvs) slash
df.sort_values(by='wav_filesize', inplace=True) slash
print(';;;;;;;;;;;;;;;;;;;;;;;;;')

# Convert to character index arrays
df['transcript'] = df['transcript'].apply(partial(text_to_char_array, alphabet=Config.alphabet))

def generate_values():
for _, row in df.iterrows():
yield row.wav_filename, to_sparse_tuple(row.transcript)

# Batching a dataset of 2D SparseTensors creates 3D batches, which fail
# when passed to tf.nn.ctc_loss, so we reshape them to remove the extra
# dimension here.
def sparse_reshape(sparse):
shape = sparse.dense_shape
return tf.sparse.reshape(sparse, [shape[0], shape[2]])

def batch_fn(features, features_len, transcripts):
print('.....................................')
features = tf.data.Dataset.zip((features, features_len))
features = features.padded_batch(batch_size,
padded_shapes=([None, Config.n_input], []))
transcripts = transcripts.batch(batch_size).map(sparse_reshape)
return tf.data.Dataset.zip((features, transcripts))

num_gpus = len(Config.available_devices)

dataset = (tf.data.Dataset.from_generator(generate_values,
output_types=(tf.string, (tf.int64, tf.int32, tf.int64)))
.map(entry_to_features, num_parallel_calls=tf.data.experimental.AUTOTUNE)
.cache(cache_path)
.window(batch_size, drop_remainder=True).flat_map(batch_fn)
.prefetch(num_gpus))

return dataset

def secs_to_hours(secs): slash
hours, remainder = divmod(secs, 3600) slash
minutes, seconds = divmod(remainder, 60) slash
return '%d:%02d:%02d' % (hours, minutes, seconds)

So I overwrited the feeding.py and text.py. slash
Now Its working

thanks

If possible please tell me why I m getting the error.

Thanks



[My RTX 4000 GPU is not used fully while i pre-trained the deepspeech
0.5.1 models

[Archived Post]

[JRMeyer]

>>> SamahZaro
[August 24, 2019, 7:00am]

On v.0.5.1, do:

git cherry-pick 007e512

then, update DeepSpeech.py as attached:
DeepSpeech.py.zip
(9.8 KB)

This should work, if I didn't miss anything.



[I am getting Validation Loss: inf



[Pre-trained model become worse when i trained common voice
data

[Archived Post]