up

config.json

@@ -65,7 +65,7 @@
   "mask_time_length": 10,
   "mask_time_min_space": 1,
   "mask_time_other": 0.0,
-  "mask_time_prob": 0.65,
+  "mask_time_prob": 0.18,
   "mask_time_selection": "static",
   "model_type": "wav2vec2",
   "num_attention_heads": 16,

run_main_flax.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+# JAX_PLATFORM_NAME="cpu" ./run_wav2vec2_pretrain_flax.py \
+./run_wav2vec2_pretrain_flax.py \
+    --output_dir="./" \
+    --num_train_epochs="1000" \
+    --warmup_steps="32000" \
+		--dataset_name="patrickvonplaten/librispeech_local" \
+		--dataset_config_name="clean" \
+		--train_split_name="train" \
+    --per_device_train_batch_size="8" \
+    --per_device_eval_batch_size="8" \
+    --learning_rate="0.0005" \
+    --weight_decay="0.01" \
+    --model_name_or_path="./" \
+		--logging_steps="1" \
+		--max_duration_in_seconds="16.0" \
+		--adam_beta1="0.9" \
+	  --adam_beta2="0.98" \
+	  --adam_epsilon="1e-06" \
+#		--logging_steps="10" \

wandb/debug-internal.log

@@ -0,0 +1 @@
+run-20210907_094306-1gii9ass/logs/debug-internal.log

wandb/debug.log

@@ -0,0 +1 @@
+run-20210907_094306-1gii9ass/logs/debug.log

wandb/latest-run

@@ -0,0 +1 @@
+run-20210907_094306-1gii9ass

wandb/run-20210907_084857-bmxot28l/files/config.yaml

@@ -0,0 +1,30 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.1
+    framework: huggingface
+    huggingface_version: 4.10.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.6.9
+    start_time: 1631004537
+    t:
+      1:
+      - 1
+      - 3
+      - 5
+      - 11
+      - 12
+      2:
+      - 1
+      - 3
+      - 5
+      - 11
+      - 12
+      4: 3.6.9
+      5: 0.12.1
+      6: 4.10.0.dev0
+      8:
+      - 5

wandb/run-20210907_084857-bmxot28l/files/output.log

@@ -0,0 +1,86 @@
+09/07/2021 08:49:00 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 08:49:00 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+09/07/2021 08:49:02 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 08:49:02 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+09/07/2021 08:49:03 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 08:49:03 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+09/07/2021 08:49:03 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-db644ef2f80e3eb0.arrow
+09/07/2021 08:49:03 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-28ff0a758509a518.arrow
+09/07/2021 08:49:03 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-ebd4bfab3f9a52e1.arrow
+09/07/2021 08:49:03 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-22013ad6e1e490a3.arrow
+09/07/2021 08:49:03 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-76e336d8ca423b69.arrow
+09/07/2021 08:49:03 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-4434cc61a19cb118.arrow
+09/07/2021 08:49:03 - INFO - absl - Starting the local TPU driver.
+09/07/2021 08:49:03 - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://
+09/07/2021 08:49:03 - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: TPU Interpreter Host
+2021-09-07 08:49:16.475557: W tensorflow/stream_executor/platform/default/dso_loader.cc:60] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/home/patrick/hf_flax_tpu/lib:/home/patrick/hf_flax_tpu/lib
+2021-09-07 08:49:16.475608: W tensorflow/stream_executor/cuda/cuda_driver.cc:326] failed call to cuInit: UNKNOWN ERROR (303)
+/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/lib/xla_bridge.py:387: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  "jax.host_count has been renamed to jax.process_count. This alias "
+/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/lib/xla_bridge.py:374: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  "jax.host_id has been renamed to jax.process_index. This alias "
+09/07/2021 08:49:16 - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
+Epoch ... (1/1000):   0%|                                                        | 0/1000 [00:00<?, ?it/s]
+Epoch ... (1/1000):   0%|                                                        | 0/1000 [00:16<?, ?it/s]
+Traceback (most recent call last):
+  File "./run_wav2vec2_pretrain_flax.py", line 677, in <module>
+    main()
+  File "./run_wav2vec2_pretrain_flax.py", line 615, in main
+    state, model_inputs, dropout_rngs, gumbel_rngs
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback
+    return fun(*args, **kwargs)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/_src/api.py", line 1676, in f_pmapped
+    global_arg_shapes=tuple(global_arg_shapes_flat))
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/core.py", line 1620, in bind
+    return call_bind(self, fun, *args, **params)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/core.py", line 1551, in call_bind
+    outs = primitive.process(top_trace, fun, tracers, params)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/core.py", line 1623, in process
+    return trace.process_map(self, fun, tracers, params)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/core.py", line 606, in process_call
+    return primitive.impl(f, *tracers, **params)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/interpreters/pxla.py", line 628, in xla_pmap_impl
+    *abstract_args)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/linear_util.py", line 262, in memoized_fun
+    ans = call(fun, *args)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/interpreters/pxla.py", line 713, in parallel_callable
+    fun, global_sharded_avals, pe.debug_info_final(fun, "pmap"))
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/interpreters/partial_eval.py", line 1284, in trace_to_jaxpr_final
+    jaxpr, out_avals, consts = trace_to_subjaxpr_dynamic(fun, main, in_avals)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/interpreters/partial_eval.py", line 1262, in trace_to_subjaxpr_dynamic
+    ans = fun.call_wrapped(*in_tracers)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/linear_util.py", line 166, in call_wrapped
+    ans = self.f(*args, **dict(self.params, **kwargs))
+  File "./run_wav2vec2_pretrain_flax.py", line 529, in train_step
+    (loss, logs), grad = grad_fn(state.params)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/_src/traceback_util.py", line 183, in reraise_with_filtered_traceback
+    return fun(*args, **kwargs)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/_src/api.py", line 904, in value_and_grad_f
+    f_partial, *dyn_args, has_aux=True, reduce_axes=reduce_axes)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/_src/api.py", line 2002, in _vjp
+    flat_fun, primals_flat, has_aux=True, reduce_axes=reduce_axes)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/interpreters/ad.py", line 117, in vjp
+    out_primals, pvals, jaxpr, consts, aux = linearize(traceable, *primals, has_aux=True)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/interpreters/ad.py", line 102, in linearize
+    jaxpr, out_pvals, consts = pe.trace_to_jaxpr(jvpfun_flat, in_pvals)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/interpreters/partial_eval.py", line 505, in trace_to_jaxpr
+    jaxpr, (out_pvals, consts, env) = fun.call_wrapped(pvals)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/linear_util.py", line 166, in call_wrapped
+    ans = self.f(*args, **dict(self.params, **kwargs))
+  File "./run_wav2vec2_pretrain_flax.py", line 522, in loss_fn
+    logs["contrast_loss"] = jax.lax.psum(contrastive_loss) / global_sample_size
+jax._src.traceback_util.UnfilteredStackTrace: TypeError: psum() missing 1 required positional argument: 'axis_name'
+The stack trace below excludes JAX-internal frames.
+The preceding is the original exception that occurred, unmodified.
+--------------------
+The above exception was the direct cause of the following exception:
+Traceback (most recent call last):
+  File "./run_wav2vec2_pretrain_flax.py", line 677, in <module>
+    main()
+  File "./run_wav2vec2_pretrain_flax.py", line 615, in main
+    state, model_inputs, dropout_rngs, gumbel_rngs
+  File "./run_wav2vec2_pretrain_flax.py", line 529, in train_step
+    (loss, logs), grad = grad_fn(state.params)
+  File "./run_wav2vec2_pretrain_flax.py", line 522, in loss_fn
+    logs["contrast_loss"] = jax.lax.psum(contrastive_loss) / global_sample_size
+TypeError: psum() missing 1 required positional argument: 'axis_name'

wandb/run-20210907_084857-bmxot28l/files/requirements.txt

@@ -0,0 +1,293 @@
+absl-py==0.12.0
+aiobotocore==1.2.2
+aiohttp==3.7.4.post0
+aioitertools==0.7.1
+alabaster==0.7.12
+antlr4-python3-runtime==4.8
+apache-beam==2.28.0
+apipkg==1.5
+appdirs==1.4.4
+argcomplete==1.12.3
+arrow==1.0.3
+astunparse==1.6.3
+async-timeout==3.0.1
+attrs==20.3.0
+audioread==2.1.9
+avro-python3==1.9.2.1
+aws-sam-translator==1.35.0
+aws-xray-sdk==2.7.0
+babel==2.9.0
+backcall==0.2.0
+bcj-cffi==0.5.1
+beautifulsoup4==4.9.3
+bert-score==0.3.9
+binaryornot==0.4.4
+black==21.4b0
+boto3==1.16.43
+boto==2.49.0
+botocore==1.19.52
+bs4==0.0.1
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+cfn-lint==0.48.3
+chardet==4.0.0
+chex==0.0.6
+click==7.1.2
+cloud-tpu-client==0.10
+cloud-tpu-profiler==2.4.0
+commonmark==0.9.1
+configparser==5.0.2
+conllu==4.4
+cookiecutter==1.7.2
+crcmod==1.7
+cryptography==3.4.7
+cycler==0.10.0
+cython==0.29.23
+dataclasses==0.8
+datasets==1.9.0
+decorator==4.4.2
+dill==0.3.1.1
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+docker==5.0.0
+docopt==0.6.2
+docutils==0.16
+ecdsa==0.14.1
+elasticsearch==7.12.0
+entrypoints==0.3
+et-xmlfile==1.1.0
+execnet==1.8.0
+fairseq==1.0.0a0+9825786
+faiss-cpu==1.7.0
+fastavro==1.4.0
+fastbpe==0.1.0
+fasteners==0.16
+filelock==3.0.12
+fire==0.4.0
+flake8==3.7.9
+flask-cors==3.0.10
+flask==1.1.2
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.6.1
+fugashi==1.1.0
+future==0.18.2
+gast==0.3.3
+gcs-oauth2-boto-plugin==2.7
+gitdb==4.0.7
+gitpython==3.1.18
+google-api-core==1.26.3
+google-api-python-client==1.8.0
+google-apitools==0.5.32
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.4
+google-auth==1.30.0
+google-pasta==0.2.0
+google-reauth==0.1.1
+googleapis-common-protos==1.53.0
+grpcio==1.32.0
+gsutil==4.62
+gviz-api==1.9.0
+h5py==2.10.0
+hdfs==2.6.0
+httplib2==0.19.1
+huggingface-hub==0.0.8
+hydra-core==1.0.7
+idna-ssl==1.1.0
+idna==2.10
+imagesize==1.2.0
+importlib-metadata==4.0.1
+importlib-resources==3.3.1
+iniconfig==1.1.1
+intel-openmp==2021.2.0
+ipadic==1.0.0
+ipdb==0.13.7
+ipython-genutils==0.2.0
+ipython==7.16.1
+isort==5.8.0
+itsdangerous==1.1.0
+jax==0.2.17
+jaxlib==0.1.68
+jedi==0.17.2
+jinja2-time==0.2.0
+jinja2==2.11.3
+jiwer==2.2.0
+jmespath==0.10.0
+joblib==1.0.1
+jsondiff==1.3.0
+jsonpatch==1.32
+jsonpointer==2.1
+jsonschema==3.2.0
+junit-xml==1.9
+keras-preprocessing==1.1.2
+keras2onnx==1.7.0
+kiwisolver==1.3.1
+langdetect==1.0.8
+librosa==0.8.1
+libtpu-nightly==0.1.dev20210615
+llvmlite==0.36.0
+lxml==4.6.3
+markdown==3.3.4
+markupsafe==1.1.1
+matplotlib==3.3.4
+mccabe==0.6.1
+mkl==2021.2.0
+mock==2.0.0
+monotonic==1.6
+more-itertools==8.7.0
+moto==2.0.4
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.9
+multivolumefile==0.2.2
+mwparserfromhell==0.6
+mypy-extensions==0.4.3
+networkx==2.5.1
+nltk==3.6.2
+numba==0.53.1
+numpy==1.19.5
+oauth2client==4.1.3
+oauthlib==3.1.0
+omegaconf==2.0.6
+onnx==1.9.0
+onnxconverter-common==1.8.1
+openpyxl==3.0.7
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==20.9
+pandas==1.1.5
+parameterized==0.8.1
+parso==0.7.1
+pathspec==0.8.1
+pathtools==0.1.2
+pbr==5.6.0
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.2.0
+pip==21.1
+pkg-resources==0.0.0
+plac==1.3.3
+pluggy==0.13.1
+pooch==1.4.0
+portalocker==2.0.0
+poyo==0.5.0
+ppmd-cffi==0.4.2
+promise==2.3
+prompt-toolkit==3.0.18
+protobuf==3.15.8
+psutil==5.8.0
+ptyprocess==0.7.0
+py7zr==0.15.1
+py==1.10.0
+pyarrow==2.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pycodestyle==2.5.0
+pycparser==2.20
+pycryptodome==3.10.1
+pydot==1.4.2
+pyflakes==2.1.1
+pygments==2.8.1
+pymongo==3.11.3
+pyopenssl==20.0.1
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pytest-forked==1.3.0
+pytest-sugar==0.9.4
+pytest-xdist==2.2.1
+pytest==6.2.3
+python-dateutil==2.8.1
+python-jose==3.2.0
+python-levenshtein==0.12.2
+python-slugify==4.0.1
+pytorch-lightning==1.1.2
+pytorch-nlp==0.5.0
+pytz==2021.1
+pyu2f==0.1.5
+pyyaml==5.4.1
+pyzstd==0.14.4
+rarfile==4.0
+recommonmark==0.7.1
+regex==2021.4.4
+requests-file==1.5.1
+requests-oauthlib==1.3.0
+requests==2.25.1
+resampy==0.2.2
+responses==0.13.2
+retry-decorator==1.1.1
+rouge-score==0.0.4
+rsa==4.7.2
+s3fs==0.4.2
+s3transfer==0.3.7
+sacrebleu==1.5.1
+sacremoses==0.0.45
+scikit-learn==0.24.1
+scipy==1.5.4
+sentencepiece==0.1.91
+sentry-sdk==1.3.1
+seqeval==1.2.2
+setuptools==56.0.0
+shortuuid==1.0.1
+six==1.15.0
+sklearn==0.0
+smmap==4.0.0
+snowballstemmer==2.1.0
+soundfile==0.10.3.post1
+soupsieve==2.2.1
+sphinx-copybutton==0.3.1
+sphinx-markdown-tables==0.0.15
+sphinx-rtd-theme==0.4.3
+sphinx==3.2.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxext-opengraph==0.4.1
+sshpubkeys==3.3.1
+style==1.1.0
+subprocess32==3.5.4
+tbb==2021.2.0
+tbp-nightly==2.5.0a20210511
+tensorboard-data-server==0.6.0
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.4.0
+tensorflow==2.4.1
+termcolor==1.1.0
+text-unidecode==1.3
+texttable==1.6.3
+threadpoolctl==2.1.0
+timeout-decorator==0.5.0
+tldextract==3.1.0
+tokenizers==0.10.2
+toml==0.10.2
+toolz==0.11.1
+torch==1.9.0
+torchaudio==0.8.1
+torchvision==0.7.0a0+a6073f0
+tqdm==4.49.0
+traitlets==4.3.3
+transformers==4.9.0.dev0
+typed-ast==1.4.3
+typing-extensions==3.7.4.3
+unidic-lite==1.0.8
+unidic==1.0.3
+update==0.0.1
+uritemplate==3.0.1
+urllib3==1.26.4
+wandb==0.12.1
+wasabi==0.8.2
+wcwidth==0.2.5
+websocket-client==0.58.0
+werkzeug==1.0.1
+wget==3.2
+wheel==0.36.2
+wrapt==1.12.1
+xmltodict==0.12.0
+xxhash==2.0.2
+yarl==1.6.3
+zipp==3.4.1
+zstandard==0.15.2

wandb/run-20210907_084857-bmxot28l/files/wandb-metadata.json

@@ -0,0 +1,39 @@
+{
+    "os": "Linux-5.4.0-1042-gcp-x86_64-with-Ubuntu-18.04-bionic",
+    "python": "3.6.9",
+    "heartbeatAt": "2021-09-07T08:48:59.818731",
+    "startedAt": "2021-09-07T08:48:57.134297",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--num_train_epochs=1000",
+        "--warmup_steps=32000",
+        "--dataset_name=patrickvonplaten/librispeech_local",
+        "--dataset_config_name=clean",
+        "--train_split_name=train",
+        "--per_device_train_batch_size=16",
+        "--per_device_eval_batch_size=16",
+        "--learning_rate=0.0005",
+        "--weight_decay=0.01",
+        "--model_name_or_path=./",
+        "--logging_steps=1",
+        "--max_duration_in_seconds=8.0",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--adam_epsilon=1e-06"
+    ],
+    "state": "running",
+    "program": "./run_wav2vec2_pretrain_flax.py",
+    "codePath": "run_wav2vec2_pretrain_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/patrickvonplaten/wav2vec2_libri",
+        "commit": "365d5efbbfdc7c14d0dfcf152e1a1f1de74aa15d"
+    },
+    "email": "patrick@huggingface.co",
+    "root": "/home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri",
+    "host": "t1v-n-71556209-w-0",
+    "username": "patrick",
+    "executable": "/home/patrick/hf_flax_tpu/bin/python3"
+}

wandb/run-20210907_084857-bmxot28l/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb": {"runtime": 39}}

wandb/run-20210907_084857-bmxot28l/logs/debug-internal.log

@@ -0,0 +1,148 @@
+2021-09-07 08:48:57,831 INFO    MainThread:85206 [internal.py:wandb_internal():89] W&B internal server running at pid: 85206, started at: 2021-09-07 08:48:57.831699
+2021-09-07 08:48:57,834 INFO    WriterThread:85206 [datastore.py:open_for_write():77] open: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/run-bmxot28l.wandb
+2021-09-07 08:48:57,835 DEBUG   SenderThread:85206 [sender.py:send():182] send: header
+2021-09-07 08:48:57,835 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: check_version
+2021-09-07 08:48:57,835 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: check_version
+2021-09-07 08:48:57,875 DEBUG   SenderThread:85206 [sender.py:send():182] send: run
+2021-09-07 08:48:58,075 INFO    SenderThread:85206 [dir_watcher.py:__init__():169] watching files in: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files
+2021-09-07 08:48:58,075 INFO    SenderThread:85206 [sender.py:_start_run_threads():743] run started: bmxot28l with start time 1631004537
+2021-09-07 08:48:58,075 DEBUG   SenderThread:85206 [sender.py:send():182] send: summary
+2021-09-07 08:48:58,076 INFO    SenderThread:85206 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2021-09-07 08:48:58,077 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: run_start
+2021-09-07 08:48:59,079 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/wandb-summary.json
+2021-09-07 08:48:59,818 DEBUG   HandlerThread:85206 [meta.py:__init__():40] meta init
+2021-09-07 08:48:59,818 DEBUG   HandlerThread:85206 [meta.py:__init__():54] meta init done
+2021-09-07 08:48:59,818 DEBUG   HandlerThread:85206 [meta.py:probe():211] probe
+2021-09-07 08:48:59,820 DEBUG   HandlerThread:85206 [meta.py:_setup_git():201] setup git
+2021-09-07 08:48:59,842 DEBUG   HandlerThread:85206 [meta.py:_setup_git():208] setup git done
+2021-09-07 08:48:59,843 DEBUG   HandlerThread:85206 [meta.py:_save_pip():58] save pip
+2021-09-07 08:48:59,843 DEBUG   HandlerThread:85206 [meta.py:_save_pip():72] save pip done
+2021-09-07 08:48:59,843 DEBUG   HandlerThread:85206 [meta.py:probe():249] probe done
+2021-09-07 08:48:59,846 DEBUG   SenderThread:85206 [sender.py:send():182] send: files
+2021-09-07 08:48:59,846 INFO    SenderThread:85206 [sender.py:_save_file():873] saving file wandb-metadata.json with policy now
+2021-09-07 08:48:59,854 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 08:48:59,854 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 08:49:00,079 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/requirements.txt
+2021-09-07 08:49:00,080 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:00,080 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/wandb-metadata.json
+2021-09-07 08:49:00,318 INFO    Thread-11 :85206 [upload_job.py:push():137] Uploaded file /tmp/tmprxsbzvczwandb/18957dvg-wandb-metadata.json
+2021-09-07 08:49:02,080 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:07,142 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:09,143 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:14,990 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 08:49:14,990 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 08:49:19,147 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:21,148 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:27,999 DEBUG   SenderThread:85206 [sender.py:send():182] send: stats
+2021-09-07 08:49:30,127 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 08:49:30,128 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 08:49:37,154 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:37,533 DEBUG   SenderThread:85206 [sender.py:send():182] send: telemetry
+2021-09-07 08:49:37,533 DEBUG   SenderThread:85206 [sender.py:send():182] send: exit
+2021-09-07 08:49:37,534 INFO    SenderThread:85206 [sender.py:send_exit():291] handling exit code: 1
+2021-09-07 08:49:37,534 INFO    SenderThread:85206 [sender.py:send_exit():293] handling runtime: 39
+2021-09-07 08:49:37,535 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:37,535 INFO    SenderThread:85206 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2021-09-07 08:49:37,535 INFO    SenderThread:85206 [sender.py:send_exit():303] send defer
+2021-09-07 08:49:37,535 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:37,536 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:37,536 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 0
+2021-09-07 08:49:37,536 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:37,536 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 0
+2021-09-07 08:49:37,536 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 1
+2021-09-07 08:49:37,536 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:37,536 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 1
+2021-09-07 08:49:37,624 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:37,624 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 1
+2021-09-07 08:49:37,624 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 2
+2021-09-07 08:49:37,624 DEBUG   SenderThread:85206 [sender.py:send():182] send: stats
+2021-09-07 08:49:37,624 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:37,624 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 2
+2021-09-07 08:49:37,625 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:37,625 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 2
+2021-09-07 08:49:37,625 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 3
+2021-09-07 08:49:37,625 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:37,625 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 3
+2021-09-07 08:49:37,625 DEBUG   SenderThread:85206 [sender.py:send():182] send: summary
+2021-09-07 08:49:37,625 INFO    SenderThread:85206 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2021-09-07 08:49:37,625 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:37,625 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 3
+2021-09-07 08:49:37,626 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 4
+2021-09-07 08:49:37,626 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:37,626 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 4
+2021-09-07 08:49:37,626 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:37,626 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 4
+2021-09-07 08:49:37,638 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:37,815 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 5
+2021-09-07 08:49:37,815 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:37,816 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:37,816 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 5
+2021-09-07 08:49:37,816 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:37,816 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 5
+2021-09-07 08:49:37,816 INFO    SenderThread:85206 [dir_watcher.py:finish():283] shutting down directory watcher
+2021-09-07 08:49:37,917 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,154 INFO    Thread-8  :85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:38,155 INFO    SenderThread:85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/config.yaml
+2021-09-07 08:49:38,155 INFO    SenderThread:85206 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/wandb-summary.json
+2021-09-07 08:49:38,155 INFO    SenderThread:85206 [dir_watcher.py:finish():313] scan: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files
+2021-09-07 08:49:38,155 INFO    SenderThread:85206 [dir_watcher.py:finish():327] scan save: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/wandb-summary.json wandb-summary.json
+2021-09-07 08:49:38,155 INFO    SenderThread:85206 [dir_watcher.py:finish():327] scan save: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log output.log
+2021-09-07 08:49:38,155 INFO    SenderThread:85206 [dir_watcher.py:finish():327] scan save: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/requirements.txt requirements.txt
+2021-09-07 08:49:38,156 INFO    SenderThread:85206 [dir_watcher.py:finish():327] scan save: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/wandb-metadata.json wandb-metadata.json
+2021-09-07 08:49:38,156 INFO    SenderThread:85206 [dir_watcher.py:finish():327] scan save: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/config.yaml config.yaml
+2021-09-07 08:49:38,156 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 6
+2021-09-07 08:49:38,156 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,159 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:38,159 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 6
+2021-09-07 08:49:38,160 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:38,160 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 6
+2021-09-07 08:49:38,160 INFO    SenderThread:85206 [file_pusher.py:finish():177] shutting down file pusher
+2021-09-07 08:49:38,258 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,258 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,360 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,360 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,462 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,462 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,564 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,564 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,628 INFO    Thread-14 :85206 [upload_job.py:push():137] Uploaded file /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/requirements.txt
+2021-09-07 08:49:38,640 INFO    Thread-15 :85206 [upload_job.py:push():137] Uploaded file /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/config.yaml
+2021-09-07 08:49:38,665 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,666 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,693 INFO    Thread-12 :85206 [upload_job.py:push():137] Uploaded file /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/wandb-summary.json
+2021-09-07 08:49:38,705 INFO    Thread-13 :85206 [upload_job.py:push():137] Uploaded file /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/files/output.log
+2021-09-07 08:49:38,767 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,767 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,869 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:38,869 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:38,906 INFO    Thread-7  :85206 [sender.py:transition_state():316] send defer: 7
+2021-09-07 08:49:38,906 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:38,906 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 7
+2021-09-07 08:49:38,906 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:38,906 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 7
+2021-09-07 08:49:38,971 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:39,207 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 8
+2021-09-07 08:49:39,207 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:39,208 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:39,208 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 8
+2021-09-07 08:49:39,208 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:39,208 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 8
+2021-09-07 08:49:39,208 INFO    SenderThread:85206 [sender.py:transition_state():316] send defer: 9
+2021-09-07 08:49:39,208 DEBUG   SenderThread:85206 [sender.py:send():182] send: final
+2021-09-07 08:49:39,209 DEBUG   SenderThread:85206 [sender.py:send():182] send: footer
+2021-09-07 08:49:39,209 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: defer
+2021-09-07 08:49:39,209 INFO    HandlerThread:85206 [handler.py:handle_request_defer():143] handle defer: 9
+2021-09-07 08:49:39,209 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: defer
+2021-09-07 08:49:39,209 INFO    SenderThread:85206 [sender.py:send_request_defer():312] handle sender defer: 9
+2021-09-07 08:49:39,309 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: poll_exit
+2021-09-07 08:49:39,309 DEBUG   SenderThread:85206 [sender.py:send_request():196] send_request: poll_exit
+2021-09-07 08:49:39,309 INFO    SenderThread:85206 [file_pusher.py:join():182] waiting for file pusher
+2021-09-07 08:49:39,310 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: get_summary
+2021-09-07 08:49:39,311 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: sampled_history
+2021-09-07 08:49:39,312 DEBUG   HandlerThread:85206 [handler.py:handle_request():126] handle_request: shutdown
+2021-09-07 08:49:39,312 INFO    HandlerThread:85206 [handler.py:finish():683] shutting down handler
+2021-09-07 08:49:40,209 INFO    WriterThread:85206 [datastore.py:close():281] close: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/run-bmxot28l.wandb
+2021-09-07 08:49:40,309 INFO    SenderThread:85206 [sender.py:finish():979] shutting down sender
+2021-09-07 08:49:40,309 INFO    SenderThread:85206 [file_pusher.py:finish():177] shutting down file pusher
+2021-09-07 08:49:40,310 INFO    SenderThread:85206 [file_pusher.py:join():182] waiting for file pusher
+2021-09-07 08:49:40,311 INFO    MainThread:85206 [internal.py:handle_exit():76] Internal process exited

wandb/run-20210907_084857-bmxot28l/logs/debug.log

@@ -0,0 +1,124 @@
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [wandb_setup.py:_flush():69] setting env: {}
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [wandb_init.py:_log_setup():342] Logging user logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/logs/debug.log
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [wandb_init.py:_log_setup():343] Logging internal logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_084857-bmxot28l/logs/debug-internal.log
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [wandb_init.py:init():375] calling init triggers
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [wandb_init.py:init():382] wandb.init called with sweep_config: {}
+config: {}
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [wandb_init.py:init():424] starting backend
+2021-09-07 08:48:57,135 INFO    MainThread:85036 [backend.py:_multiprocessing_setup():72] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-09-07 08:48:57,147 INFO    MainThread:85036 [backend.py:ensure_launched():135] starting backend process...
+2021-09-07 08:48:57,155 INFO    MainThread:85036 [backend.py:ensure_launched():140] started backend process with pid: 85206
+2021-09-07 08:48:57,156 INFO    MainThread:85036 [wandb_init.py:init():429] backend started and connected
+2021-09-07 08:48:57,158 INFO    MainThread:85036 [wandb_init.py:init():477] updated telemetry
+2021-09-07 08:48:57,159 INFO    MainThread:85036 [wandb_init.py:init():500] communicating current version
+2021-09-07 08:48:57,874 INFO    MainThread:85036 [wandb_init.py:init():505] got version response 
+2021-09-07 08:48:57,874 INFO    MainThread:85036 [wandb_init.py:init():513] communicating run to backend with 30 second timeout
+2021-09-07 08:48:58,076 INFO    MainThread:85036 [wandb_init.py:init():540] starting run threads in backend
+2021-09-07 08:48:59,850 INFO    MainThread:85036 [wandb_run.py:_console_start():1601] atexit reg
+2021-09-07 08:48:59,850 INFO    MainThread:85036 [wandb_run.py:_redirect():1475] redirect: SettingsConsole.REDIRECT
+2021-09-07 08:48:59,853 INFO    MainThread:85036 [wandb_run.py:_redirect():1480] Redirecting console.
+2021-09-07 08:48:59,855 INFO    MainThread:85036 [wandb_run.py:_redirect():1536] Redirects installed.
+2021-09-07 08:48:59,855 INFO    MainThread:85036 [wandb_init.py:init():565] run started, returning control to user process
+2021-09-07 08:49:35,103 INFO    MainThread:85036 [wandb_run.py:_atexit_cleanup():1571] got exitcode: 1
+2021-09-07 08:49:35,103 INFO    MainThread:85036 [wandb_run.py:_restore():1543] restore
+2021-09-07 08:49:37,536 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1349
+  total_bytes: 1349
+}
+
+2021-09-07 08:49:37,816 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 1
+}
+pusher_stats {
+  uploaded_bytes: 1349
+  total_bytes: 1349
+}
+
+2021-09-07 08:49:38,157 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 4
+}
+pusher_stats {
+  uploaded_bytes: 1349
+  total_bytes: 15591
+}
+
+2021-09-07 08:49:38,259 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 1349
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:38,361 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:38,463 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:38,564 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:38,666 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:38,768 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:38,870 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:39,208 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:39,310 INFO    MainThread:85036 [wandb_run.py:_wait_for_finish():1693] got exit ret: done: true
+exit_result {
+}
+file_counts {
+  wandb_count: 5
+}
+pusher_stats {
+  uploaded_bytes: 16034
+  total_bytes: 16034
+}
+
+2021-09-07 08:49:40,682 INFO    MainThread:85036 [wandb_run.py:_show_files():1915] logging synced files

wandb/run-20210907_085010-xa9s4yt2/files/config.yaml

@@ -0,0 +1,24 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.1
+    framework: huggingface
+    huggingface_version: 4.10.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.6.9
+    start_time: 1631004610
+    t:
+      1:
+      - 1
+      - 3
+      - 5
+      - 11
+      - 12
+      4: 3.6.9
+      5: 0.12.1
+      6: 4.10.0.dev0
+      8:
+      - 5

wandb/run-20210907_085010-xa9s4yt2/files/output.log

@@ -0,0 +1,212 @@
+09/07/2021 08:50:14 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 08:50:14 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+09/07/2021 08:50:15 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 08:50:15 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+09/07/2021 08:50:16 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 08:50:16 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+09/07/2021 08:50:16 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-db644ef2f80e3eb0.arrow
+09/07/2021 08:50:16 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-28ff0a758509a518.arrow
+09/07/2021 08:50:16 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-ebd4bfab3f9a52e1.arrow
+09/07/2021 08:50:16 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-22013ad6e1e490a3.arrow
+09/07/2021 08:50:16 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-76e336d8ca423b69.arrow
+09/07/2021 08:50:16 - WARNING - datasets.arrow_dataset - Loading cached processed dataset at /home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d/cache-4434cc61a19cb118.arrow
+09/07/2021 08:50:16 - INFO - absl - Starting the local TPU driver.
+09/07/2021 08:50:16 - INFO - absl - Unable to initialize backend 'tpu_driver': Not found: Unable to find driver in registry given worker: local://
+09/07/2021 08:50:16 - INFO - absl - Unable to initialize backend 'gpu': Not found: Could not find registered platform with name: "cuda". Available platform names are: TPU Interpreter Host
+2021-09-07 08:50:30.721334: W tensorflow/stream_executor/platform/default/dso_loader.cc:60] Could not load dynamic library 'libcuda.so.1'; dlerror: libcuda.so.1: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: :/home/patrick/hf_flax_tpu/lib:/home/patrick/hf_flax_tpu/lib
+2021-09-07 08:50:30.721379: W tensorflow/stream_executor/cuda/cuda_driver.cc:326] failed call to cuInit: UNKNOWN ERROR (303)
+/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/lib/xla_bridge.py:387: UserWarning: jax.host_count has been renamed to jax.process_count. This alias will eventually be removed; please update your code.
+  "jax.host_count has been renamed to jax.process_count. This alias "
+/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/jax/lib/xla_bridge.py:374: UserWarning: jax.host_id has been renamed to jax.process_index. This alias will eventually be removed; please update your code.
+  "jax.host_id has been renamed to jax.process_index. This alias "
+09/07/2021 08:50:30 - INFO - absl - A polynomial schedule was set with a non-positive `transition_steps` value; this results in a constant schedule with value `init_value`.
+Epoch ... (1/1000):   0%|                                                        | 0/1000 [00:00<?, ?it/s]
+
+Training...:   6%|███▎                                                    | 1/17 [02:26<39:02, 146.43s/it]
+
+
+Training...:  18%|██████████                                               | 3/17 [02:47<18:00, 77.16s/it]
+
+Training...:  24%|█████████████▍                                           | 4/17 [02:58<12:24, 57.30s/it]
+Step... 3 | %_mask_idx: 0.148829 | %_neg_is_pos: 0.02009 | acc: 0.397615 | code_ppl: 65.296341 | contrast_loss: 4.614569 | diversity_loss: 0.897934 | gumbel_temp: 1.999968 | learn_rate: 0.0 | loss: 4.794156
+
+Training...:  29%|████████████████▊                                        | 5/17 [03:09<08:39, 43.25s/it]
+
+
+Training...:  41%|███████████████████████▍                                 | 7/17 [03:30<04:25, 26.58s/it]
+
+Training...:  47%|██████████████████████████▊                              | 8/17 [03:40<03:15, 21.71s/it]
+
+Training...:  53%|██████████████████████████████▏                          | 9/17 [03:50<02:26, 18.34s/it]
+
+Training...:  59%|████████████████████████████████▉                       | 10/17 [04:01<01:52, 16.02s/it]
+
+Training...:  65%|████████████████████████████████████▏                   | 11/17 [04:12<01:26, 14.45s/it]
+Step... 10 | %_mask_idx: 0.154664 | %_neg_is_pos: 0.017656 | acc: 0.59001 | code_ppl: 64.84523 | contrast_loss: 4.486528 | diversity_loss: 0.898543 | gumbel_temp: 1.999897 | learn_rate: 0.0 | loss: 4.666236
+
+
+Training...:  76%|██████████████████████████████████████████▊             | 13/17 [04:33<00:50, 12.57s/it]
+Step... 12 | %_mask_idx: 0.147204 | %_neg_is_pos: 0.024369 | acc: 0.542137 | code_ppl: 64.376663 | contrast_loss: 4.484812 | diversity_loss: 0.899122 | gumbel_temp: 1.999878 | learn_rate: 0.0 | loss: 4.664637
+
+
+Training...:  88%|█████████████████████████████████████████████████▍      | 15/17 [04:55<00:23, 11.56s/it]
+
+Training...:  94%|████████████████████████████████████████████████████▋   | 16/17 [05:05<00:11, 11.21s/it]
+Epoch ... (1/1000):   0%|                                            | 1/1000 [05:16<87:56:58, 316.94s/it]
+Training...:   0%|                                                                 | 0/17 [00:00<?, ?it/s]
+Step... 16 | %_mask_idx: 0.147282 | %_neg_is_pos: 0.018483 | acc: 0.70988 | code_ppl: 66.037872 | contrast_loss: 4.417307 | diversity_loss: 0.896782 | gumbel_temp: 1.999838 | learn_rate: 0.0 | loss: 4.596663
+
+Training...:   6%|███▎                                                     | 1/17 [00:10<02:44, 10.31s/it]
+
+Training...:  12%|██████▋                                                  | 2/17 [00:20<02:36, 10.40s/it]
+
+Training...:  18%|██████████                                               | 3/17 [00:31<02:25, 10.42s/it]
+
+
+Training...:  29%|████████████████▊                                        | 5/17 [00:52<02:06, 10.58s/it]
+Step... 21 | %_mask_idx: 0.147224 | %_neg_is_pos: 0.021905 | acc: 0.647831 | code_ppl: 64.606194 | contrast_loss: 4.366142 | diversity_loss: 0.898966 | gumbel_temp: 1.999788 | learn_rate: 0.0 | loss: 4.545934
+
+Training...:  35%|████████████████████                                     | 6/17 [01:03<01:57, 10.68s/it]
+
+Training...:  41%|███████████████████████▍                                 | 7/17 [01:14<01:46, 10.64s/it]
+
+
+Training...:  53%|██████████████████████████████▏                          | 9/17 [01:35<01:25, 10.70s/it]
+
+Training...:  59%|████████████████████████████████▉                       | 10/17 [01:46<01:14, 10.60s/it]
+
+Training...:  65%|████████████████████████████████████▏                   | 11/17 [01:56<01:03, 10.61s/it]
+Step... 27 | %_mask_idx: 0.141408 | %_neg_is_pos: 0.023796 | acc: 0.777647 | code_ppl: 64.749557 | contrast_loss: 4.030019 | diversity_loss: 0.899018 | gumbel_temp: 1.999727 | learn_rate: 0.0 | loss: 4.209823
+
+
+Training...:  76%|██████████████████████████████████████████▊             | 13/17 [02:17<00:42, 10.56s/it]
+
+Training...:  82%|██████████████████████████████████████████████          | 14/17 [02:28<00:31, 10.54s/it]
+
+Training...:  88%|█████████████████████████████████████████████████▍      | 15/17 [02:38<00:21, 10.50s/it]
+
+Training...:  94%|████████████████████████████████████████████████████▋   | 16/17 [02:49<00:10, 10.44s/it]
+Epoch ... (1/1000):   0%|                                            | 2/1000 [08:16<76:27:23, 275.80s/it]
+Training...:   0%|                                                                 | 0/17 [00:00<?, ?it/s]
+
+Training...:   6%|███▎                                                     | 1/17 [00:10<02:48, 10.56s/it]
+
+Training...:  12%|██████▋                                                  | 2/17 [00:21<02:38, 10.58s/it]
+
+Training...:  18%|██████████                                               | 3/17 [00:31<02:28, 10.64s/it]
+Step... 36 | %_mask_idx: 0.150944 | %_neg_is_pos: 0.015462 | acc: 0.68468 | code_ppl: 66.00061 | contrast_loss: 3.968953 | diversity_loss: 0.89669 | gumbel_temp: 1.999637 | learn_rate: 1e-06 | loss: 4.148291
+
+Training...:  24%|█████████████▍                                           | 4/17 [00:42<02:19, 10.75s/it]
+
+Training...:  29%|████████████████▊                                        | 5/17 [00:53<02:07, 10.62s/it]
+
+
+Training...:  41%|███████████████████████▍                                 | 7/17 [01:13<01:44, 10.45s/it]
+
+Training...:  47%|██████████████████████████▊                              | 8/17 [01:24<01:34, 10.45s/it]
+
+Training...:  53%|██████████████████████████████▏                          | 9/17 [01:34<01:23, 10.38s/it]
+
+Training...:  59%|████████████████████████████████▉                       | 10/17 [01:45<01:12, 10.40s/it]
+Step... 43 | %_mask_idx: 0.150846 | %_neg_is_pos: 0.014129 | acc: 0.687187 | code_ppl: 63.311432 | contrast_loss: 4.039135 | diversity_loss: 0.900887 | gumbel_temp: 1.999566 | learn_rate: 1e-06 | loss: 4.219313
+
+Training...:  65%|████████████████████████████████████▏                   | 11/17 [01:55<01:02, 10.35s/it]
+
+
+Training...:  76%|██████████████████████████████████████████▊             | 13/17 [02:15<00:41, 10.30s/it]
+
+Training...:  82%|██████████████████████████████████████████████          | 14/17 [02:25<00:30, 10.24s/it]
+
+Training...:  88%|█████████████████████████████████████████████████▍      | 15/17 [02:36<00:20, 10.33s/it]
+Step... 48 | %_mask_idx: 0.153802 | %_neg_is_pos: 0.017595 | acc: 0.644443 | code_ppl: 62.821945 | contrast_loss: 3.884152 | diversity_loss: 0.901863 | gumbel_temp: 1.999516 | learn_rate: 1e-06 | loss: 4.064525
+
+Training...:  94%|████████████████████████████████████████████████████▋   | 16/17 [02:46<00:10, 10.34s/it]
+Epoch ... (1/1000):   0%|▏                                           | 3/1000 [11:13<68:09:55, 246.13s/it]
+Training...:   0%|                                                                 | 0/17 [00:00<?, ?it/s]
+
+Training...:   6%|███▎                                                     | 1/17 [00:10<02:44, 10.30s/it]
+
+
+Training...:  18%|██████████                                               | 3/17 [00:30<02:23, 10.28s/it]
+
+Training...:  24%|█████████████▍                                           | 4/17 [00:40<02:12, 10.21s/it]
+
+Training...:  29%|████████████████▊                                        | 5/17 [00:51<02:03, 10.28s/it]
+
+Training...:  35%|████████████████████                                     | 6/17 [01:01<01:53, 10.28s/it]
+Step... 56 | %_mask_idx: 0.146421 | %_neg_is_pos: 0.013453 | acc: 0.721942 | code_ppl: 62.72192 | contrast_loss: 4.029288 | diversity_loss: 0.901994 | gumbel_temp: 1.999436 | learn_rate: 1e-06 | loss: 4.209687
+
+
+Training...:  47%|██████████████████████████▊                              | 8/17 [01:21<01:31, 10.19s/it]
+
+Training...:  53%|██████████████████████████████▏                          | 9/17 [01:32<01:22, 10.25s/it]
+
+Training...:  59%|████████████████████████████████▉                       | 10/17 [01:42<01:12, 10.32s/it]
+
+Training...:  65%|████████████████████████████████████▏                   | 11/17 [01:52<01:01, 10.26s/it]
+
+Training...:  71%|███████████████████████████████████████▌                | 12/17 [02:03<00:51, 10.34s/it]
+
+Training...:  76%|███████████████████████████████████���██████▊             | 13/17 [02:13<00:41, 10.31s/it]
+
+Training...:  82%|██████████████████████████████████████████████          | 14/17 [02:24<00:31, 10.43s/it]
+Step... 64 | %_mask_idx: 0.151923 | %_neg_is_pos: 0.012708 | acc: 0.433114 | code_ppl: 62.871773 | contrast_loss: 4.64666 | diversity_loss: 0.901782 | gumbel_temp: 1.999356 | learn_rate: 1e-06 | loss: 4.827016
+
+
+Training...:  94%|████████████████████████████████████████████████████▋   | 16/17 [02:44<00:10, 10.37s/it]
+Step... 66 | %_mask_idx: 0.152138 | %_neg_is_pos: 0.010406 | acc: 0.584998 | code_ppl: 61.46981 | contrast_loss: 4.340511 | diversity_loss: 0.903897 | gumbel_temp: 1.999337 | learn_rate: 1e-06 | loss: 4.521291
+Epoch ... (1/1000):   0%|▏                                           | 4/1000 [14:08<62:12:52, 224.87s/it]
+
+Training...:   6%|███▎                                                     | 1/17 [00:10<02:40, 10.01s/it]
+
+Training...:  12%|██████▋                                                  | 2/17 [00:20<02:30, 10.06s/it]
+
+Training...:  18%|██████████                                               | 3/17 [00:30<02:22, 10.16s/it]
+Step... 70 | %_mask_idx: 0.155741 | %_neg_is_pos: 0.01797 | acc: 0.655349 | code_ppl: 62.328838 | contrast_loss: 4.522501 | diversity_loss: 0.902651 | gumbel_temp: 1.999297 | learn_rate: 1e-06 | loss: 4.703031
+
+Training...:  24%|█████████████▍                                           | 4/17 [00:40<02:12, 10.21s/it]
+
+Training...:  29%|████████████████▊                                        | 5/17 [00:51<02:02, 10.20s/it]
+
+Training...:  35%|████████████████████                                     | 6/17 [01:01<01:52, 10.22s/it]
+
+
+Training...:  47%|██████████████████████████▊                              | 8/17 [01:21<01:31, 10.22s/it]
+Step... 75 | %_mask_idx: 0.141075 | %_neg_is_pos: 0.018648 | acc: 0.577087 | code_ppl: 57.249924 | contrast_loss: 4.386855 | diversity_loss: 0.910558 | gumbel_temp: 1.999247 | learn_rate: 1e-06 | loss: 4.568966
+
+
+Training...:  59%|████████████████████████████████▉                       | 10/17 [01:42<01:12, 10.29s/it]
+Step... 77 | %_mask_idx: 0.146225 | %_neg_is_pos: 0.025055 | acc: 0.711995 | code_ppl: 58.103603 | contrast_loss: 4.043971 | diversity_loss: 0.909113 | gumbel_temp: 1.999226 | learn_rate: 1e-06 | loss: 4.225793
+
+Training...:  65%|████████████████████████████████████▏                   | 11/17 [01:52<01:01, 10.24s/it]
+
+Training...:  71%|███████████████████████████████████████▌                | 12/17 [02:02<00:51, 10.26s/it]
+
+Training...:  76%|██████████████████████████████████████████▊             | 13/17 [02:13<00:41, 10.31s/it]
+
+
+Training...:  88%|█████████████████████████████████████████████████▍      | 15/17 [02:33<00:20, 10.17s/it]
+
+Training...:  94%|████████████████████████████████████████████████████▋   | 16/17 [02:43<00:10, 10.25s/it]
+Epoch ... (1/1000):   0%|▏                                           | 5/1000 [17:03<57:56:35, 209.64s/it]
+Training...:   0%|                                                                 | 0/17 [00:00<?, ?it/s]
+Step... 84 | %_mask_idx: 0.14131 | %_neg_is_pos: 0.014478 | acc: 0.514822 | code_ppl: 54.038326 | contrast_loss: 4.383283 | diversity_loss: 0.915511 | gumbel_temp: 1.999157 | learn_rate: 1e-06 | loss: 4.566385
+
+
+Training...:  12%|██████▋                                                  | 2/17 [00:20<02:31, 10.12s/it]
+
+Training...:  18%|██████████                                               | 3/17 [00:30<02:22, 10.21s/it]
+
+Training...:  24%|█████████████▍                                           | 4/17 [00:41<02:13, 10.31s/it]
+
+Training...:  29%|████████████████▊                                        | 5/17 [00:51<02:03, 10.29s/it]
+
+Training...:  35%|████████████████████                                     | 6/17 [01:01<01:51, 10.17s/it]
+Step... 90 | %_mask_idx: 0.141761 | %_neg_is_pos: 0.027947 | acc: 0.867814 | code_ppl: 49.4561 | contrast_loss: 3.399678 | diversity_loss: 0.922551 | gumbel_temp: 1.999096 | learn_rate: 1e-06 | loss: 3.584189
+
+
+Training...:  47%|██████████████████████████▊                              | 8/17 [01:21<01:31, 10.13s/it]
+Step... 92 | %_mask_idx: 0.148947 | %_neg_is_pos: 0.028889 | acc: 0.711877 | code_ppl: 48.757538 | contrast_loss: 3.726825 | diversity_loss: 0.92388 | gumbel_temp: 1.999077 | learn_rate: 1e-06 | loss: 3.911601
+
+
+Training...:  59%|████████████████████████████████▉                       | 10/17 [01:42<01:11, 10.28s/it]

wandb/run-20210907_085010-xa9s4yt2/files/requirements.txt

@@ -0,0 +1,293 @@
+absl-py==0.12.0
+aiobotocore==1.2.2
+aiohttp==3.7.4.post0
+aioitertools==0.7.1
+alabaster==0.7.12
+antlr4-python3-runtime==4.8
+apache-beam==2.28.0
+apipkg==1.5
+appdirs==1.4.4
+argcomplete==1.12.3
+arrow==1.0.3
+astunparse==1.6.3
+async-timeout==3.0.1
+attrs==20.3.0
+audioread==2.1.9
+avro-python3==1.9.2.1
+aws-sam-translator==1.35.0
+aws-xray-sdk==2.7.0
+babel==2.9.0
+backcall==0.2.0
+bcj-cffi==0.5.1
+beautifulsoup4==4.9.3
+bert-score==0.3.9
+binaryornot==0.4.4
+black==21.4b0
+boto3==1.16.43
+boto==2.49.0
+botocore==1.19.52
+bs4==0.0.1
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+cfn-lint==0.48.3
+chardet==4.0.0
+chex==0.0.6
+click==7.1.2
+cloud-tpu-client==0.10
+cloud-tpu-profiler==2.4.0
+commonmark==0.9.1
+configparser==5.0.2
+conllu==4.4
+cookiecutter==1.7.2
+crcmod==1.7
+cryptography==3.4.7
+cycler==0.10.0
+cython==0.29.23
+dataclasses==0.8
+datasets==1.9.0
+decorator==4.4.2
+dill==0.3.1.1
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+docker==5.0.0
+docopt==0.6.2
+docutils==0.16
+ecdsa==0.14.1
+elasticsearch==7.12.0
+entrypoints==0.3
+et-xmlfile==1.1.0
+execnet==1.8.0
+fairseq==1.0.0a0+9825786
+faiss-cpu==1.7.0
+fastavro==1.4.0
+fastbpe==0.1.0
+fasteners==0.16
+filelock==3.0.12
+fire==0.4.0
+flake8==3.7.9
+flask-cors==3.0.10
+flask==1.1.2
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.6.1
+fugashi==1.1.0
+future==0.18.2
+gast==0.3.3
+gcs-oauth2-boto-plugin==2.7
+gitdb==4.0.7
+gitpython==3.1.18
+google-api-core==1.26.3
+google-api-python-client==1.8.0
+google-apitools==0.5.32
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.4
+google-auth==1.30.0
+google-pasta==0.2.0
+google-reauth==0.1.1
+googleapis-common-protos==1.53.0
+grpcio==1.32.0
+gsutil==4.62
+gviz-api==1.9.0
+h5py==2.10.0
+hdfs==2.6.0
+httplib2==0.19.1
+huggingface-hub==0.0.8
+hydra-core==1.0.7
+idna-ssl==1.1.0
+idna==2.10
+imagesize==1.2.0
+importlib-metadata==4.0.1
+importlib-resources==3.3.1
+iniconfig==1.1.1
+intel-openmp==2021.2.0
+ipadic==1.0.0
+ipdb==0.13.7
+ipython-genutils==0.2.0
+ipython==7.16.1
+isort==5.8.0
+itsdangerous==1.1.0
+jax==0.2.17
+jaxlib==0.1.68
+jedi==0.17.2
+jinja2-time==0.2.0
+jinja2==2.11.3
+jiwer==2.2.0
+jmespath==0.10.0
+joblib==1.0.1
+jsondiff==1.3.0
+jsonpatch==1.32
+jsonpointer==2.1
+jsonschema==3.2.0
+junit-xml==1.9
+keras-preprocessing==1.1.2
+keras2onnx==1.7.0
+kiwisolver==1.3.1
+langdetect==1.0.8
+librosa==0.8.1
+libtpu-nightly==0.1.dev20210615
+llvmlite==0.36.0
+lxml==4.6.3
+markdown==3.3.4
+markupsafe==1.1.1
+matplotlib==3.3.4
+mccabe==0.6.1
+mkl==2021.2.0
+mock==2.0.0
+monotonic==1.6
+more-itertools==8.7.0
+moto==2.0.4
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.9
+multivolumefile==0.2.2
+mwparserfromhell==0.6
+mypy-extensions==0.4.3
+networkx==2.5.1
+nltk==3.6.2
+numba==0.53.1
+numpy==1.19.5
+oauth2client==4.1.3
+oauthlib==3.1.0
+omegaconf==2.0.6
+onnx==1.9.0
+onnxconverter-common==1.8.1
+openpyxl==3.0.7
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==20.9
+pandas==1.1.5
+parameterized==0.8.1
+parso==0.7.1
+pathspec==0.8.1
+pathtools==0.1.2
+pbr==5.6.0
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.2.0
+pip==21.1
+pkg-resources==0.0.0
+plac==1.3.3
+pluggy==0.13.1
+pooch==1.4.0
+portalocker==2.0.0
+poyo==0.5.0
+ppmd-cffi==0.4.2
+promise==2.3
+prompt-toolkit==3.0.18
+protobuf==3.15.8
+psutil==5.8.0
+ptyprocess==0.7.0
+py7zr==0.15.1
+py==1.10.0
+pyarrow==2.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pycodestyle==2.5.0
+pycparser==2.20
+pycryptodome==3.10.1
+pydot==1.4.2
+pyflakes==2.1.1
+pygments==2.8.1
+pymongo==3.11.3
+pyopenssl==20.0.1
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pytest-forked==1.3.0
+pytest-sugar==0.9.4
+pytest-xdist==2.2.1
+pytest==6.2.3
+python-dateutil==2.8.1
+python-jose==3.2.0
+python-levenshtein==0.12.2
+python-slugify==4.0.1
+pytorch-lightning==1.1.2
+pytorch-nlp==0.5.0
+pytz==2021.1
+pyu2f==0.1.5
+pyyaml==5.4.1
+pyzstd==0.14.4
+rarfile==4.0
+recommonmark==0.7.1
+regex==2021.4.4
+requests-file==1.5.1
+requests-oauthlib==1.3.0
+requests==2.25.1
+resampy==0.2.2
+responses==0.13.2
+retry-decorator==1.1.1
+rouge-score==0.0.4
+rsa==4.7.2
+s3fs==0.4.2
+s3transfer==0.3.7
+sacrebleu==1.5.1
+sacremoses==0.0.45
+scikit-learn==0.24.1
+scipy==1.5.4
+sentencepiece==0.1.91
+sentry-sdk==1.3.1
+seqeval==1.2.2
+setuptools==56.0.0
+shortuuid==1.0.1
+six==1.15.0
+sklearn==0.0
+smmap==4.0.0
+snowballstemmer==2.1.0
+soundfile==0.10.3.post1
+soupsieve==2.2.1
+sphinx-copybutton==0.3.1
+sphinx-markdown-tables==0.0.15
+sphinx-rtd-theme==0.4.3
+sphinx==3.2.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxext-opengraph==0.4.1
+sshpubkeys==3.3.1
+style==1.1.0
+subprocess32==3.5.4
+tbb==2021.2.0
+tbp-nightly==2.5.0a20210511
+tensorboard-data-server==0.6.0
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.4.0
+tensorflow==2.4.1
+termcolor==1.1.0
+text-unidecode==1.3
+texttable==1.6.3
+threadpoolctl==2.1.0
+timeout-decorator==0.5.0
+tldextract==3.1.0
+tokenizers==0.10.2
+toml==0.10.2
+toolz==0.11.1
+torch==1.9.0
+torchaudio==0.8.1
+torchvision==0.7.0a0+a6073f0
+tqdm==4.49.0
+traitlets==4.3.3
+transformers==4.9.0.dev0
+typed-ast==1.4.3
+typing-extensions==3.7.4.3
+unidic-lite==1.0.8
+unidic==1.0.3
+update==0.0.1
+uritemplate==3.0.1
+urllib3==1.26.4
+wandb==0.12.1
+wasabi==0.8.2
+wcwidth==0.2.5
+websocket-client==0.58.0
+werkzeug==1.0.1
+wget==3.2
+wheel==0.36.2
+wrapt==1.12.1
+xmltodict==0.12.0
+xxhash==2.0.2
+yarl==1.6.3
+zipp==3.4.1
+zstandard==0.15.2

wandb/run-20210907_085010-xa9s4yt2/files/wandb-metadata.json

@@ -0,0 +1,39 @@
+{
+    "os": "Linux-5.4.0-1042-gcp-x86_64-with-Ubuntu-18.04-bionic",
+    "python": "3.6.9",
+    "heartbeatAt": "2021-09-07T08:50:13.128024",
+    "startedAt": "2021-09-07T08:50:10.525558",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--num_train_epochs=1000",
+        "--warmup_steps=32000",
+        "--dataset_name=patrickvonplaten/librispeech_local",
+        "--dataset_config_name=clean",
+        "--train_split_name=train",
+        "--per_device_train_batch_size=16",
+        "--per_device_eval_batch_size=16",
+        "--learning_rate=0.0005",
+        "--weight_decay=0.01",
+        "--model_name_or_path=./",
+        "--logging_steps=1",
+        "--max_duration_in_seconds=8.0",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--adam_epsilon=1e-06"
+    ],
+    "state": "running",
+    "program": "./run_wav2vec2_pretrain_flax.py",
+    "codePath": "run_wav2vec2_pretrain_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/patrickvonplaten/wav2vec2_libri",
+        "commit": "365d5efbbfdc7c14d0dfcf152e1a1f1de74aa15d"
+    },
+    "email": "patrick@huggingface.co",
+    "root": "/home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri",
+    "host": "t1v-n-71556209-w-0",
+    "username": "patrick",
+    "executable": "/home/patrick/hf_flax_tpu/bin/python3"
+}

wandb/run-20210907_085010-xa9s4yt2/files/wandb-summary.json

@@ -0,0 +1 @@
+{"%_mask_idx": 0.1511591523885727, "%_neg_is_pos": 0.04813283681869507, "acc": 0.6854244470596313, "code_ppl": 45.653076171875, "contrast_loss": 4.309719562530518, "diversity_loss": 0.928508996963501, "gumbel_temp": 1.9990558624267578, "learn_rate": 1.468753907829523e-06, "loss": 4.495421409606934, "_runtime": 1148, "_timestamp": 1631005758, "_step": 93}

wandb/run-20210907_085010-xa9s4yt2/logs/debug.log

@@ -0,0 +1,24 @@
+2021-09-07 08:50:10,526 INFO    MainThread:86745 [wandb_setup.py:_flush():69] setting env: {}
+2021-09-07 08:50:10,526 INFO    MainThread:86745 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-09-07 08:50:10,526 INFO    MainThread:86745 [wandb_init.py:_log_setup():342] Logging user logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_085010-xa9s4yt2/logs/debug.log
+2021-09-07 08:50:10,526 INFO    MainThread:86745 [wandb_init.py:_log_setup():343] Logging internal logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_085010-xa9s4yt2/logs/debug-internal.log
+2021-09-07 08:50:10,526 INFO    MainThread:86745 [wandb_init.py:init():375] calling init triggers
+2021-09-07 08:50:10,527 INFO    MainThread:86745 [wandb_init.py:init():382] wandb.init called with sweep_config: {}
+config: {}
+2021-09-07 08:50:10,527 INFO    MainThread:86745 [wandb_init.py:init():424] starting backend
+2021-09-07 08:50:10,527 INFO    MainThread:86745 [backend.py:_multiprocessing_setup():72] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-09-07 08:50:10,538 INFO    MainThread:86745 [backend.py:ensure_launched():135] starting backend process...
+2021-09-07 08:50:10,546 INFO    MainThread:86745 [backend.py:ensure_launched():140] started backend process with pid: 86893
+2021-09-07 08:50:10,547 INFO    MainThread:86745 [wandb_init.py:init():429] backend started and connected
+2021-09-07 08:50:10,549 INFO    MainThread:86745 [wandb_init.py:init():477] updated telemetry
+2021-09-07 08:50:10,550 INFO    MainThread:86745 [wandb_init.py:init():500] communicating current version
+2021-09-07 08:50:11,282 INFO    MainThread:86745 [wandb_init.py:init():505] got version response 
+2021-09-07 08:50:11,283 INFO    MainThread:86745 [wandb_init.py:init():513] communicating run to backend with 30 second timeout
+2021-09-07 08:50:11,458 INFO    MainThread:86745 [wandb_init.py:init():540] starting run threads in backend
+2021-09-07 08:50:13,160 INFO    MainThread:86745 [wandb_run.py:_console_start():1601] atexit reg
+2021-09-07 08:50:13,161 INFO    MainThread:86745 [wandb_run.py:_redirect():1475] redirect: SettingsConsole.REDIRECT
+2021-09-07 08:50:13,163 INFO    MainThread:86745 [wandb_run.py:_redirect():1480] Redirecting console.
+2021-09-07 08:50:13,165 INFO    MainThread:86745 [wandb_run.py:_redirect():1536] Redirects installed.
+2021-09-07 08:50:13,165 INFO    MainThread:86745 [wandb_init.py:init():565] run started, returning control to user process
+2021-09-07 09:09:24,673 INFO    MainThread:86745 [wandb_run.py:_atexit_cleanup():1571] got exitcode: 255
+2021-09-07 09:09:24,673 INFO    MainThread:86745 [wandb_run.py:_restore():1543] restore

wandb/run-20210907_094005-1jmcrt7a/files/config.yaml

@@ -0,0 +1,24 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.1
+    framework: huggingface
+    huggingface_version: 4.10.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.6.9
+    start_time: 1631007605
+    t:
+      1:
+      - 1
+      - 3
+      - 5
+      - 11
+      - 12
+      4: 3.6.9
+      5: 0.12.1
+      6: 4.10.0.dev0
+      8:
+      - 5

wandb/run-20210907_094005-1jmcrt7a/files/output.log

@@ -0,0 +1,72 @@
+09/07/2021 09:40:09 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 09:40:09 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+  0%|                                                                             | 0/135 [00:00<?, ?ex/s]
+09/07/2021 09:40:10 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+09/07/2021 09:40:10 - WARNING - datasets.builder - Reusing dataset librispeech_asr (/home/patrick/.cache/huggingface/datasets/librispeech_asr/clean-5e73ba74b650b050/2.1.0/4a3512d5532dc4a75a0072aace5d2f46280550843efadb78ee570c340d0fcb9d)
+09/07/2021 09:40:11 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050
+100%|██████████████████████████████████████████████████████████████████| 135/135 [00:00<00:00, 212.58ex/s]
+
+
+
+
+
+
+
+
+
+
+100%|█████████████████████████████████████████████████████████████████| 2568/2568 [00:25<00:00, 99.82ex/s]
+100%|███████████████████████████████████████████████████████████████████████| 1/1 [00:03<00:00,  3.99s/ba]
+
+ 67%|███████████████████████████████████████████████▎                       | 2/3 [01:34<00:47, 47.21s/ba]
+Exception in thread Thread-3:
+Traceback (most recent call last):
+  File "/usr/lib/python3.6/threading.py", line 916, in _bootstrap_inner
+    self.run()
+  File "/usr/lib/python3.6/threading.py", line 864, in run
+    self._target(*self._args, **self._kwargs)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/wandb/sdk/wandb_run.py", line 203, in check_status
+    status_response = self._interface.communicate_stop_status()
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/wandb/sdk/interface/interface.py", line 737, in communicate_stop_status
+    resp = self._communicate(req, timeout=timeout, local=True)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/wandb/sdk/interface/interface.py", line 539, in _communicate
+    return self._communicate_async(rec, local=local).get(timeout=timeout)
+  File "/home/patrick/hf_flax_tpu/lib/python3.6/site-packages/wandb/sdk/interface/interface.py", line 544, in _communicate_async
+    raise Exception("The wandb backend process has shutdown")
+Exception: The wandb backend process has shutdown
+Traceback (most recent call last):
+  File "./run_wav2vec2_pretrain_flax.py", line 677, in <module>
+    main()
+  File "./run_wav2vec2_pretrain_flax.py", line 365, in main
+    vectorized_datasets = vectorized_datasets.filter(lambda x: len(x["speech"]) > min_length)
+  File "/home/patrick/python_bin/datasets/dataset_dict.py", line 554, in filter
+    for k, dataset in self.items()
+  File "/home/patrick/python_bin/datasets/dataset_dict.py", line 554, in <dictcomp>
+    for k, dataset in self.items()
+  File "/home/patrick/python_bin/datasets/arrow_dataset.py", line 185, in wrapper
+    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
+  File "/home/patrick/python_bin/datasets/fingerprint.py", line 397, in wrapper
+    out = func(self, *args, **kwargs)
+  File "/home/patrick/python_bin/datasets/arrow_dataset.py", line 2157, in filter
+    new_fingerprint=new_fingerprint,
+  File "/home/patrick/python_bin/datasets/arrow_dataset.py", line 1673, in map
+    desc=desc,
+  File "/home/patrick/python_bin/datasets/arrow_dataset.py", line 185, in wrapper
+    out: Union["Dataset", "DatasetDict"] = func(self, *args, **kwargs)
+  File "/home/patrick/python_bin/datasets/fingerprint.py", line 397, in wrapper
+    out = func(self, *args, **kwargs)
+  File "/home/patrick/python_bin/datasets/arrow_dataset.py", line 2001, in _map_single
+    batch = input_dataset[i : i + batch_size]
+  File "/home/patrick/python_bin/datasets/arrow_dataset.py", line 1522, in __getitem__
+    format_kwargs=self._format_kwargs,
+  File "/home/patrick/python_bin/datasets/arrow_dataset.py", line 1511, in _getitem
+    pa_subtable, key, formatter=formatter, format_columns=format_columns, output_all_columns=output_all_columns
+  File "/home/patrick/python_bin/datasets/formatting/formatting.py", line 414, in format_table
+    return formatter(pa_table, query_type=query_type)
+  File "/home/patrick/python_bin/datasets/formatting/formatting.py", line 198, in __call__
+    return self.format_batch(pa_table)
+  File "/home/patrick/python_bin/datasets/formatting/formatting.py", line 229, in format_batch
+    return self.python_arrow_extractor().extract_batch(pa_table)
+  File "/home/patrick/python_bin/datasets/formatting/formatting.py", line 140, in extract_batch
+    return pa_table.to_pydict()
+KeyboardInterrupt

wandb/run-20210907_094005-1jmcrt7a/files/requirements.txt

@@ -0,0 +1,293 @@
+absl-py==0.12.0
+aiobotocore==1.2.2
+aiohttp==3.7.4.post0
+aioitertools==0.7.1
+alabaster==0.7.12
+antlr4-python3-runtime==4.8
+apache-beam==2.28.0
+apipkg==1.5
+appdirs==1.4.4
+argcomplete==1.12.3
+arrow==1.0.3
+astunparse==1.6.3
+async-timeout==3.0.1
+attrs==20.3.0
+audioread==2.1.9
+avro-python3==1.9.2.1
+aws-sam-translator==1.35.0
+aws-xray-sdk==2.7.0
+babel==2.9.0
+backcall==0.2.0
+bcj-cffi==0.5.1
+beautifulsoup4==4.9.3
+bert-score==0.3.9
+binaryornot==0.4.4
+black==21.4b0
+boto3==1.16.43
+boto==2.49.0
+botocore==1.19.52
+bs4==0.0.1
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+cfn-lint==0.48.3
+chardet==4.0.0
+chex==0.0.6
+click==7.1.2
+cloud-tpu-client==0.10
+cloud-tpu-profiler==2.4.0
+commonmark==0.9.1
+configparser==5.0.2
+conllu==4.4
+cookiecutter==1.7.2
+crcmod==1.7
+cryptography==3.4.7
+cycler==0.10.0
+cython==0.29.23
+dataclasses==0.8
+datasets==1.9.0
+decorator==4.4.2
+dill==0.3.1.1
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+docker==5.0.0
+docopt==0.6.2
+docutils==0.16
+ecdsa==0.14.1
+elasticsearch==7.12.0
+entrypoints==0.3
+et-xmlfile==1.1.0
+execnet==1.8.0
+fairseq==1.0.0a0+9825786
+faiss-cpu==1.7.0
+fastavro==1.4.0
+fastbpe==0.1.0
+fasteners==0.16
+filelock==3.0.12
+fire==0.4.0
+flake8==3.7.9
+flask-cors==3.0.10
+flask==1.1.2
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.6.1
+fugashi==1.1.0
+future==0.18.2
+gast==0.3.3
+gcs-oauth2-boto-plugin==2.7
+gitdb==4.0.7
+gitpython==3.1.18
+google-api-core==1.26.3
+google-api-python-client==1.8.0
+google-apitools==0.5.32
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.4
+google-auth==1.30.0
+google-pasta==0.2.0
+google-reauth==0.1.1
+googleapis-common-protos==1.53.0
+grpcio==1.32.0
+gsutil==4.62
+gviz-api==1.9.0
+h5py==2.10.0
+hdfs==2.6.0
+httplib2==0.19.1
+huggingface-hub==0.0.8
+hydra-core==1.0.7
+idna-ssl==1.1.0
+idna==2.10
+imagesize==1.2.0
+importlib-metadata==4.0.1
+importlib-resources==3.3.1
+iniconfig==1.1.1
+intel-openmp==2021.2.0
+ipadic==1.0.0
+ipdb==0.13.7
+ipython-genutils==0.2.0
+ipython==7.16.1
+isort==5.8.0
+itsdangerous==1.1.0
+jax==0.2.17
+jaxlib==0.1.68
+jedi==0.17.2
+jinja2-time==0.2.0
+jinja2==2.11.3
+jiwer==2.2.0
+jmespath==0.10.0
+joblib==1.0.1
+jsondiff==1.3.0
+jsonpatch==1.32
+jsonpointer==2.1
+jsonschema==3.2.0
+junit-xml==1.9
+keras-preprocessing==1.1.2
+keras2onnx==1.7.0
+kiwisolver==1.3.1
+langdetect==1.0.8
+librosa==0.8.1
+libtpu-nightly==0.1.dev20210615
+llvmlite==0.36.0
+lxml==4.6.3
+markdown==3.3.4
+markupsafe==1.1.1
+matplotlib==3.3.4
+mccabe==0.6.1
+mkl==2021.2.0
+mock==2.0.0
+monotonic==1.6
+more-itertools==8.7.0
+moto==2.0.4
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.9
+multivolumefile==0.2.2
+mwparserfromhell==0.6
+mypy-extensions==0.4.3
+networkx==2.5.1
+nltk==3.6.2
+numba==0.53.1
+numpy==1.19.5
+oauth2client==4.1.3
+oauthlib==3.1.0
+omegaconf==2.0.6
+onnx==1.9.0
+onnxconverter-common==1.8.1
+openpyxl==3.0.7
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==20.9
+pandas==1.1.5
+parameterized==0.8.1
+parso==0.7.1
+pathspec==0.8.1
+pathtools==0.1.2
+pbr==5.6.0
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.2.0
+pip==21.1
+pkg-resources==0.0.0
+plac==1.3.3
+pluggy==0.13.1
+pooch==1.4.0
+portalocker==2.0.0
+poyo==0.5.0
+ppmd-cffi==0.4.2
+promise==2.3
+prompt-toolkit==3.0.18
+protobuf==3.15.8
+psutil==5.8.0
+ptyprocess==0.7.0
+py7zr==0.15.1
+py==1.10.0
+pyarrow==2.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pycodestyle==2.5.0
+pycparser==2.20
+pycryptodome==3.10.1
+pydot==1.4.2
+pyflakes==2.1.1
+pygments==2.8.1
+pymongo==3.11.3
+pyopenssl==20.0.1
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pytest-forked==1.3.0
+pytest-sugar==0.9.4
+pytest-xdist==2.2.1
+pytest==6.2.3
+python-dateutil==2.8.1
+python-jose==3.2.0
+python-levenshtein==0.12.2
+python-slugify==4.0.1
+pytorch-lightning==1.1.2
+pytorch-nlp==0.5.0
+pytz==2021.1
+pyu2f==0.1.5
+pyyaml==5.4.1
+pyzstd==0.14.4
+rarfile==4.0
+recommonmark==0.7.1
+regex==2021.4.4
+requests-file==1.5.1
+requests-oauthlib==1.3.0
+requests==2.25.1
+resampy==0.2.2
+responses==0.13.2
+retry-decorator==1.1.1
+rouge-score==0.0.4
+rsa==4.7.2
+s3fs==0.4.2
+s3transfer==0.3.7
+sacrebleu==1.5.1
+sacremoses==0.0.45
+scikit-learn==0.24.1
+scipy==1.5.4
+sentencepiece==0.1.91
+sentry-sdk==1.3.1
+seqeval==1.2.2
+setuptools==56.0.0
+shortuuid==1.0.1
+six==1.15.0
+sklearn==0.0
+smmap==4.0.0
+snowballstemmer==2.1.0
+soundfile==0.10.3.post1
+soupsieve==2.2.1
+sphinx-copybutton==0.3.1
+sphinx-markdown-tables==0.0.15
+sphinx-rtd-theme==0.4.3
+sphinx==3.2.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxext-opengraph==0.4.1
+sshpubkeys==3.3.1
+style==1.1.0
+subprocess32==3.5.4
+tbb==2021.2.0
+tbp-nightly==2.5.0a20210511
+tensorboard-data-server==0.6.0
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.4.0
+tensorflow==2.4.1
+termcolor==1.1.0
+text-unidecode==1.3
+texttable==1.6.3
+threadpoolctl==2.1.0
+timeout-decorator==0.5.0
+tldextract==3.1.0
+tokenizers==0.10.2
+toml==0.10.2
+toolz==0.11.1
+torch==1.9.0
+torchaudio==0.8.1
+torchvision==0.7.0a0+a6073f0
+tqdm==4.49.0
+traitlets==4.3.3
+transformers==4.9.0.dev0
+typed-ast==1.4.3
+typing-extensions==3.7.4.3
+unidic-lite==1.0.8
+unidic==1.0.3
+update==0.0.1
+uritemplate==3.0.1
+urllib3==1.26.4
+wandb==0.12.1
+wasabi==0.8.2
+wcwidth==0.2.5
+websocket-client==0.58.0
+werkzeug==1.0.1
+wget==3.2
+wheel==0.36.2
+wrapt==1.12.1
+xmltodict==0.12.0
+xxhash==2.0.2
+yarl==1.6.3
+zipp==3.4.1
+zstandard==0.15.2

wandb/run-20210907_094005-1jmcrt7a/files/wandb-metadata.json

@@ -0,0 +1,39 @@
+{
+    "os": "Linux-5.4.0-1042-gcp-x86_64-with-Ubuntu-18.04-bionic",
+    "python": "3.6.9",
+    "heartbeatAt": "2021-09-07T09:40:08.202022",
+    "startedAt": "2021-09-07T09:40:05.605187",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--num_train_epochs=1000",
+        "--warmup_steps=32000",
+        "--dataset_name=patrickvonplaten/librispeech_local",
+        "--dataset_config_name=clean",
+        "--train_split_name=train",
+        "--per_device_train_batch_size=8",
+        "--per_device_eval_batch_size=8",
+        "--learning_rate=0.0005",
+        "--weight_decay=0.01",
+        "--model_name_or_path=./",
+        "--logging_steps=1",
+        "--max_duration_in_seconds=16.0",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--adam_epsilon=1e-06"
+    ],
+    "state": "running",
+    "program": "./run_wav2vec2_pretrain_flax.py",
+    "codePath": "run_wav2vec2_pretrain_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/patrickvonplaten/wav2vec2_libri",
+        "commit": "365d5efbbfdc7c14d0dfcf152e1a1f1de74aa15d"
+    },
+    "email": "patrick@huggingface.co",
+    "root": "/home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri",
+    "host": "t1v-n-71556209-w-0",
+    "username": "patrick",
+    "executable": "/home/patrick/hf_flax_tpu/bin/python3"
+}

wandb/run-20210907_094005-1jmcrt7a/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210907_094005-1jmcrt7a/logs/debug-internal.log

@@ -0,0 +1,62 @@
+2021-09-07 09:40:06,294 INFO    MainThread:92124 [internal.py:wandb_internal():89] W&B internal server running at pid: 92124, started at: 2021-09-07 09:40:06.293950
+2021-09-07 09:40:06,296 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: check_version
+2021-09-07 09:40:06,296 INFO    WriterThread:92124 [datastore.py:open_for_write():77] open: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/run-1jmcrt7a.wandb
+2021-09-07 09:40:06,297 DEBUG   SenderThread:92124 [sender.py:send():182] send: header
+2021-09-07 09:40:06,298 DEBUG   SenderThread:92124 [sender.py:send_request():196] send_request: check_version
+2021-09-07 09:40:06,337 DEBUG   SenderThread:92124 [sender.py:send():182] send: run
+2021-09-07 09:40:06,520 INFO    SenderThread:92124 [dir_watcher.py:__init__():169] watching files in: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files
+2021-09-07 09:40:06,521 INFO    SenderThread:92124 [sender.py:_start_run_threads():743] run started: 1jmcrt7a with start time 1631007605
+2021-09-07 09:40:06,521 DEBUG   SenderThread:92124 [sender.py:send():182] send: summary
+2021-09-07 09:40:06,521 INFO    SenderThread:92124 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2021-09-07 09:40:06,522 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: run_start
+2021-09-07 09:40:07,526 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/wandb-summary.json
+2021-09-07 09:40:08,201 DEBUG   HandlerThread:92124 [meta.py:__init__():40] meta init
+2021-09-07 09:40:08,201 DEBUG   HandlerThread:92124 [meta.py:__init__():54] meta init done
+2021-09-07 09:40:08,201 DEBUG   HandlerThread:92124 [meta.py:probe():211] probe
+2021-09-07 09:40:08,203 DEBUG   HandlerThread:92124 [meta.py:_setup_git():201] setup git
+2021-09-07 09:40:08,228 DEBUG   HandlerThread:92124 [meta.py:_setup_git():208] setup git done
+2021-09-07 09:40:08,228 DEBUG   HandlerThread:92124 [meta.py:_save_pip():58] save pip
+2021-09-07 09:40:08,229 DEBUG   HandlerThread:92124 [meta.py:_save_pip():72] save pip done
+2021-09-07 09:40:08,229 DEBUG   HandlerThread:92124 [meta.py:probe():249] probe done
+2021-09-07 09:40:08,232 DEBUG   SenderThread:92124 [sender.py:send():182] send: files
+2021-09-07 09:40:08,232 INFO    SenderThread:92124 [sender.py:_save_file():873] saving file wandb-metadata.json with policy now
+2021-09-07 09:40:08,239 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 09:40:08,239 DEBUG   SenderThread:92124 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 09:40:08,525 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/requirements.txt
+2021-09-07 09:40:08,525 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/wandb-metadata.json
+2021-09-07 09:40:08,526 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:08,778 INFO    Thread-11 :92124 [upload_job.py:push():137] Uploaded file /tmp/tmpcz_z3tx7wandb/3v46qxnn-wandb-metadata.json
+2021-09-07 09:40:10,525 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:12,526 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:14,527 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:16,528 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:18,529 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:23,839 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 09:40:23,839 DEBUG   SenderThread:92124 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 09:40:24,531 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:28,532 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:30,533 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:34,535 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:36,287 DEBUG   SenderThread:92124 [sender.py:send():182] send: stats
+2021-09-07 09:40:36,536 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:38,537 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:38,969 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 09:40:38,970 DEBUG   SenderThread:92124 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 09:40:41,538 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:43,539 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:40:54,103 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 09:40:54,103 DEBUG   SenderThread:92124 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 09:41:06,360 DEBUG   SenderThread:92124 [sender.py:send():182] send: stats
+2021-09-07 09:41:36,431 DEBUG   SenderThread:92124 [sender.py:send():182] send: stats
+2021-09-07 09:41:41,586 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 09:41:41,587 DEBUG   SenderThread:92124 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 09:41:42,558 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:41:45,559 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:42:06,506 DEBUG   SenderThread:92124 [sender.py:send():182] send: stats
+2021-09-07 09:42:20,044 DEBUG   HandlerThread:92124 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 09:42:20,044 DEBUG   SenderThread:92124 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 09:42:23,572 INFO    Thread-8  :92124 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/files/output.log
+2021-09-07 09:42:34,027 WARNING MainThread:92124 [internal.py:wandb_internal():145] Internal process interrupt: 1
+2021-09-07 09:42:34,174 WARNING MainThread:92124 [internal.py:wandb_internal():145] Internal process interrupt: 2
+2021-09-07 09:42:34,174 ERROR   MainThread:92124 [internal.py:wandb_internal():148] Internal process interrupted.
+2021-09-07 09:42:34,486 INFO    MainThread:92124 [internal.py:handle_exit():76] Internal process exited

wandb/run-20210907_094005-1jmcrt7a/logs/debug.log

@@ -0,0 +1,25 @@
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [wandb_setup.py:_flush():69] setting env: {}
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [wandb_init.py:_log_setup():342] Logging user logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/logs/debug.log
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [wandb_init.py:_log_setup():343] Logging internal logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094005-1jmcrt7a/logs/debug-internal.log
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [wandb_init.py:init():375] calling init triggers
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [wandb_init.py:init():382] wandb.init called with sweep_config: {}
+config: {}
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [wandb_init.py:init():424] starting backend
+2021-09-07 09:40:05,606 INFO    MainThread:91950 [backend.py:_multiprocessing_setup():72] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-09-07 09:40:05,618 INFO    MainThread:91950 [backend.py:ensure_launched():135] starting backend process...
+2021-09-07 09:40:05,625 INFO    MainThread:91950 [backend.py:ensure_launched():140] started backend process with pid: 92124
+2021-09-07 09:40:05,627 INFO    MainThread:91950 [wandb_init.py:init():429] backend started and connected
+2021-09-07 09:40:05,629 INFO    MainThread:91950 [wandb_init.py:init():477] updated telemetry
+2021-09-07 09:40:05,629 INFO    MainThread:91950 [wandb_init.py:init():500] communicating current version
+2021-09-07 09:40:06,336 INFO    MainThread:91950 [wandb_init.py:init():505] got version response 
+2021-09-07 09:40:06,336 INFO    MainThread:91950 [wandb_init.py:init():513] communicating run to backend with 30 second timeout
+2021-09-07 09:40:06,521 INFO    MainThread:91950 [wandb_init.py:init():540] starting run threads in backend
+2021-09-07 09:40:08,236 INFO    MainThread:91950 [wandb_run.py:_console_start():1601] atexit reg
+2021-09-07 09:40:08,236 INFO    MainThread:91950 [wandb_run.py:_redirect():1475] redirect: SettingsConsole.REDIRECT
+2021-09-07 09:40:08,238 INFO    MainThread:91950 [wandb_run.py:_redirect():1480] Redirecting console.
+2021-09-07 09:40:08,240 INFO    MainThread:91950 [wandb_run.py:_redirect():1536] Redirects installed.
+2021-09-07 09:40:08,241 INFO    MainThread:91950 [wandb_init.py:init():565] run started, returning control to user process
+2021-09-07 09:42:49,330 INFO    MainThread:91950 [wandb_run.py:_atexit_cleanup():1571] got exitcode: 255
+2021-09-07 09:42:49,333 INFO    MainThread:91950 [wandb_run.py:_restore():1543] restore
+2021-09-07 09:42:52,108 INFO    MainThread:91950 [wandb_run.py:_restore():1543] restore

wandb/run-20210907_094306-1gii9ass/files/config.yaml

@@ -0,0 +1,24 @@
+wandb_version: 1
+
+_wandb:
+  desc: null
+  value:
+    cli_version: 0.12.1
+    framework: huggingface
+    huggingface_version: 4.10.0.dev0
+    is_jupyter_run: false
+    is_kaggle_kernel: false
+    python_version: 3.6.9
+    start_time: 1631007786
+    t:
+      1:
+      - 1
+      - 3
+      - 5
+      - 11
+      - 12
+      4: 3.6.9
+      5: 0.12.1
+      6: 4.10.0.dev0
+      8:
+      - 5

wandb/run-20210907_094306-1gii9ass/files/output.log

@@ -0,0 +1 @@
+09/07/2021 09:43:10 - WARNING - datasets.builder - Using custom data configuration clean-5e73ba74b650b050

wandb/run-20210907_094306-1gii9ass/files/requirements.txt

@@ -0,0 +1,293 @@
+absl-py==0.12.0
+aiobotocore==1.2.2
+aiohttp==3.7.4.post0
+aioitertools==0.7.1
+alabaster==0.7.12
+antlr4-python3-runtime==4.8
+apache-beam==2.28.0
+apipkg==1.5
+appdirs==1.4.4
+argcomplete==1.12.3
+arrow==1.0.3
+astunparse==1.6.3
+async-timeout==3.0.1
+attrs==20.3.0
+audioread==2.1.9
+avro-python3==1.9.2.1
+aws-sam-translator==1.35.0
+aws-xray-sdk==2.7.0
+babel==2.9.0
+backcall==0.2.0
+bcj-cffi==0.5.1
+beautifulsoup4==4.9.3
+bert-score==0.3.9
+binaryornot==0.4.4
+black==21.4b0
+boto3==1.16.43
+boto==2.49.0
+botocore==1.19.52
+bs4==0.0.1
+cachetools==4.2.1
+certifi==2020.12.5
+cffi==1.14.5
+cfn-lint==0.48.3
+chardet==4.0.0
+chex==0.0.6
+click==7.1.2
+cloud-tpu-client==0.10
+cloud-tpu-profiler==2.4.0
+commonmark==0.9.1
+configparser==5.0.2
+conllu==4.4
+cookiecutter==1.7.2
+crcmod==1.7
+cryptography==3.4.7
+cycler==0.10.0
+cython==0.29.23
+dataclasses==0.8
+datasets==1.9.0
+decorator==4.4.2
+dill==0.3.1.1
+dm-tree==0.1.6
+docker-pycreds==0.4.0
+docker==5.0.0
+docopt==0.6.2
+docutils==0.16
+ecdsa==0.14.1
+elasticsearch==7.12.0
+entrypoints==0.3
+et-xmlfile==1.1.0
+execnet==1.8.0
+fairseq==1.0.0a0+9825786
+faiss-cpu==1.7.0
+fastavro==1.4.0
+fastbpe==0.1.0
+fasteners==0.16
+filelock==3.0.12
+fire==0.4.0
+flake8==3.7.9
+flask-cors==3.0.10
+flask==1.1.2
+flatbuffers==1.12
+flax==0.3.4
+fsspec==2021.6.1
+fugashi==1.1.0
+future==0.18.2
+gast==0.3.3
+gcs-oauth2-boto-plugin==2.7
+gitdb==4.0.7
+gitpython==3.1.18
+google-api-core==1.26.3
+google-api-python-client==1.8.0
+google-apitools==0.5.32
+google-auth-httplib2==0.1.0
+google-auth-oauthlib==0.4.4
+google-auth==1.30.0
+google-pasta==0.2.0
+google-reauth==0.1.1
+googleapis-common-protos==1.53.0
+grpcio==1.32.0
+gsutil==4.62
+gviz-api==1.9.0
+h5py==2.10.0
+hdfs==2.6.0
+httplib2==0.19.1
+huggingface-hub==0.0.8
+hydra-core==1.0.7
+idna-ssl==1.1.0
+idna==2.10
+imagesize==1.2.0
+importlib-metadata==4.0.1
+importlib-resources==3.3.1
+iniconfig==1.1.1
+intel-openmp==2021.2.0
+ipadic==1.0.0
+ipdb==0.13.7
+ipython-genutils==0.2.0
+ipython==7.16.1
+isort==5.8.0
+itsdangerous==1.1.0
+jax==0.2.17
+jaxlib==0.1.68
+jedi==0.17.2
+jinja2-time==0.2.0
+jinja2==2.11.3
+jiwer==2.2.0
+jmespath==0.10.0
+joblib==1.0.1
+jsondiff==1.3.0
+jsonpatch==1.32
+jsonpointer==2.1
+jsonschema==3.2.0
+junit-xml==1.9
+keras-preprocessing==1.1.2
+keras2onnx==1.7.0
+kiwisolver==1.3.1
+langdetect==1.0.8
+librosa==0.8.1
+libtpu-nightly==0.1.dev20210615
+llvmlite==0.36.0
+lxml==4.6.3
+markdown==3.3.4
+markupsafe==1.1.1
+matplotlib==3.3.4
+mccabe==0.6.1
+mkl==2021.2.0
+mock==2.0.0
+monotonic==1.6
+more-itertools==8.7.0
+moto==2.0.4
+msgpack==1.0.2
+multidict==5.1.0
+multiprocess==0.70.9
+multivolumefile==0.2.2
+mwparserfromhell==0.6
+mypy-extensions==0.4.3
+networkx==2.5.1
+nltk==3.6.2
+numba==0.53.1
+numpy==1.19.5
+oauth2client==4.1.3
+oauthlib==3.1.0
+omegaconf==2.0.6
+onnx==1.9.0
+onnxconverter-common==1.8.1
+openpyxl==3.0.7
+opt-einsum==3.3.0
+optax==0.0.9
+packaging==20.9
+pandas==1.1.5
+parameterized==0.8.1
+parso==0.7.1
+pathspec==0.8.1
+pathtools==0.1.2
+pbr==5.6.0
+pexpect==4.8.0
+pickleshare==0.7.5
+pillow==8.2.0
+pip==21.1
+pkg-resources==0.0.0
+plac==1.3.3
+pluggy==0.13.1
+pooch==1.4.0
+portalocker==2.0.0
+poyo==0.5.0
+ppmd-cffi==0.4.2
+promise==2.3
+prompt-toolkit==3.0.18
+protobuf==3.15.8
+psutil==5.8.0
+ptyprocess==0.7.0
+py7zr==0.15.1
+py==1.10.0
+pyarrow==2.0.0
+pyasn1-modules==0.2.8
+pyasn1==0.4.8
+pycodestyle==2.5.0
+pycparser==2.20
+pycryptodome==3.10.1
+pydot==1.4.2
+pyflakes==2.1.1
+pygments==2.8.1
+pymongo==3.11.3
+pyopenssl==20.0.1
+pyparsing==2.4.7
+pyrsistent==0.17.3
+pytest-forked==1.3.0
+pytest-sugar==0.9.4
+pytest-xdist==2.2.1
+pytest==6.2.3
+python-dateutil==2.8.1
+python-jose==3.2.0
+python-levenshtein==0.12.2
+python-slugify==4.0.1
+pytorch-lightning==1.1.2
+pytorch-nlp==0.5.0
+pytz==2021.1
+pyu2f==0.1.5
+pyyaml==5.4.1
+pyzstd==0.14.4
+rarfile==4.0
+recommonmark==0.7.1
+regex==2021.4.4
+requests-file==1.5.1
+requests-oauthlib==1.3.0
+requests==2.25.1
+resampy==0.2.2
+responses==0.13.2
+retry-decorator==1.1.1
+rouge-score==0.0.4
+rsa==4.7.2
+s3fs==0.4.2
+s3transfer==0.3.7
+sacrebleu==1.5.1
+sacremoses==0.0.45
+scikit-learn==0.24.1
+scipy==1.5.4
+sentencepiece==0.1.91
+sentry-sdk==1.3.1
+seqeval==1.2.2
+setuptools==56.0.0
+shortuuid==1.0.1
+six==1.15.0
+sklearn==0.0
+smmap==4.0.0
+snowballstemmer==2.1.0
+soundfile==0.10.3.post1
+soupsieve==2.2.1
+sphinx-copybutton==0.3.1
+sphinx-markdown-tables==0.0.15
+sphinx-rtd-theme==0.4.3
+sphinx==3.2.1
+sphinxcontrib-applehelp==1.0.2
+sphinxcontrib-devhelp==1.0.2
+sphinxcontrib-htmlhelp==1.0.3
+sphinxcontrib-jsmath==1.0.1
+sphinxcontrib-qthelp==1.0.3
+sphinxcontrib-serializinghtml==1.1.4
+sphinxext-opengraph==0.4.1
+sshpubkeys==3.3.1
+style==1.1.0
+subprocess32==3.5.4
+tbb==2021.2.0
+tbp-nightly==2.5.0a20210511
+tensorboard-data-server==0.6.0
+tensorboard-plugin-wit==1.8.0
+tensorboard==2.5.0
+tensorflow-estimator==2.4.0
+tensorflow==2.4.1
+termcolor==1.1.0
+text-unidecode==1.3
+texttable==1.6.3
+threadpoolctl==2.1.0
+timeout-decorator==0.5.0
+tldextract==3.1.0
+tokenizers==0.10.2
+toml==0.10.2
+toolz==0.11.1
+torch==1.9.0
+torchaudio==0.8.1
+torchvision==0.7.0a0+a6073f0
+tqdm==4.49.0
+traitlets==4.3.3
+transformers==4.9.0.dev0
+typed-ast==1.4.3
+typing-extensions==3.7.4.3
+unidic-lite==1.0.8
+unidic==1.0.3
+update==0.0.1
+uritemplate==3.0.1
+urllib3==1.26.4
+wandb==0.12.1
+wasabi==0.8.2
+wcwidth==0.2.5
+websocket-client==0.58.0
+werkzeug==1.0.1
+wget==3.2
+wheel==0.36.2
+wrapt==1.12.1
+xmltodict==0.12.0
+xxhash==2.0.2
+yarl==1.6.3
+zipp==3.4.1
+zstandard==0.15.2

wandb/run-20210907_094306-1gii9ass/files/wandb-metadata.json

@@ -0,0 +1,39 @@
+{
+    "os": "Linux-5.4.0-1042-gcp-x86_64-with-Ubuntu-18.04-bionic",
+    "python": "3.6.9",
+    "heartbeatAt": "2021-09-07T09:43:08.936401",
+    "startedAt": "2021-09-07T09:43:06.315399",
+    "docker": null,
+    "cpu_count": 96,
+    "cuda": null,
+    "args": [
+        "--output_dir=./",
+        "--num_train_epochs=1000",
+        "--warmup_steps=32000",
+        "--dataset_name=patrickvonplaten/librispeech_local",
+        "--dataset_config_name=clean",
+        "--train_split_name=train",
+        "--per_device_train_batch_size=8",
+        "--per_device_eval_batch_size=8",
+        "--learning_rate=0.0005",
+        "--weight_decay=0.01",
+        "--model_name_or_path=./",
+        "--logging_steps=1",
+        "--max_duration_in_seconds=16.0",
+        "--adam_beta1=0.9",
+        "--adam_beta2=0.98",
+        "--adam_epsilon=1e-06"
+    ],
+    "state": "running",
+    "program": "./run_wav2vec2_pretrain_flax.py",
+    "codePath": "run_wav2vec2_pretrain_flax.py",
+    "git": {
+        "remote": "https://huggingface.co/patrickvonplaten/wav2vec2_libri",
+        "commit": "365d5efbbfdc7c14d0dfcf152e1a1f1de74aa15d"
+    },
+    "email": "patrick@huggingface.co",
+    "root": "/home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri",
+    "host": "t1v-n-71556209-w-0",
+    "username": "patrick",
+    "executable": "/home/patrick/hf_flax_tpu/bin/python3"
+}

wandb/run-20210907_094306-1gii9ass/files/wandb-summary.json

@@ -0,0 +1 @@
+{}

wandb/run-20210907_094306-1gii9ass/logs/debug-internal.log

@@ -0,0 +1,29 @@
+2021-09-07 09:43:07,011 INFO    MainThread:92771 [internal.py:wandb_internal():89] W&B internal server running at pid: 92771, started at: 2021-09-07 09:43:07.011628
+2021-09-07 09:43:07,014 DEBUG   HandlerThread:92771 [handler.py:handle_request():126] handle_request: check_version
+2021-09-07 09:43:07,014 INFO    WriterThread:92771 [datastore.py:open_for_write():77] open: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/run-1gii9ass.wandb
+2021-09-07 09:43:07,015 DEBUG   SenderThread:92771 [sender.py:send():182] send: header
+2021-09-07 09:43:07,015 DEBUG   SenderThread:92771 [sender.py:send_request():196] send_request: check_version
+2021-09-07 09:43:07,054 DEBUG   SenderThread:92771 [sender.py:send():182] send: run
+2021-09-07 09:43:07,217 INFO    SenderThread:92771 [dir_watcher.py:__init__():169] watching files in: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/files
+2021-09-07 09:43:07,217 INFO    SenderThread:92771 [sender.py:_start_run_threads():743] run started: 1gii9ass with start time 1631007786
+2021-09-07 09:43:07,217 DEBUG   SenderThread:92771 [sender.py:send():182] send: summary
+2021-09-07 09:43:07,217 DEBUG   HandlerThread:92771 [handler.py:handle_request():126] handle_request: run_start
+2021-09-07 09:43:07,218 INFO    SenderThread:92771 [sender.py:_save_file():873] saving file wandb-summary.json with policy end
+2021-09-07 09:43:08,220 INFO    Thread-8  :92771 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/files/wandb-summary.json
+2021-09-07 09:43:08,936 DEBUG   HandlerThread:92771 [meta.py:__init__():40] meta init
+2021-09-07 09:43:08,936 DEBUG   HandlerThread:92771 [meta.py:__init__():54] meta init done
+2021-09-07 09:43:08,936 DEBUG   HandlerThread:92771 [meta.py:probe():211] probe
+2021-09-07 09:43:08,937 DEBUG   HandlerThread:92771 [meta.py:_setup_git():201] setup git
+2021-09-07 09:43:08,959 DEBUG   HandlerThread:92771 [meta.py:_setup_git():208] setup git done
+2021-09-07 09:43:08,959 DEBUG   HandlerThread:92771 [meta.py:_save_pip():58] save pip
+2021-09-07 09:43:08,960 DEBUG   HandlerThread:92771 [meta.py:_save_pip():72] save pip done
+2021-09-07 09:43:08,960 DEBUG   HandlerThread:92771 [meta.py:probe():249] probe done
+2021-09-07 09:43:08,963 DEBUG   SenderThread:92771 [sender.py:send():182] send: files
+2021-09-07 09:43:08,963 INFO    SenderThread:92771 [sender.py:_save_file():873] saving file wandb-metadata.json with policy now
+2021-09-07 09:43:08,972 DEBUG   HandlerThread:92771 [handler.py:handle_request():126] handle_request: stop_status
+2021-09-07 09:43:08,972 DEBUG   SenderThread:92771 [sender.py:send_request():196] send_request: stop_status
+2021-09-07 09:43:09,220 INFO    Thread-8  :92771 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/files/output.log
+2021-09-07 09:43:09,221 INFO    Thread-8  :92771 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/files/requirements.txt
+2021-09-07 09:43:09,221 INFO    Thread-8  :92771 [dir_watcher.py:_on_file_created():217] file/dir created: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/files/wandb-metadata.json
+2021-09-07 09:43:09,435 INFO    Thread-11 :92771 [upload_job.py:push():137] Uploaded file /tmp/tmp3z4hk3fswandb/3sa8u2xh-wandb-metadata.json
+2021-09-07 09:43:11,221 INFO    Thread-8  :92771 [dir_watcher.py:_on_file_modified():230] file/dir modified: /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/files/output.log

wandb/run-20210907_094306-1gii9ass/logs/debug.log

@@ -0,0 +1,22 @@
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [wandb_setup.py:_flush():69] setting env: {}
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [wandb_setup.py:_flush():69] setting login settings: {}
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [wandb_init.py:_log_setup():342] Logging user logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/logs/debug.log
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [wandb_init.py:_log_setup():343] Logging internal logs to /home/patrick/wav2vec2_tpu_replicate/wav2vec2_libri/wandb/run-20210907_094306-1gii9ass/logs/debug-internal.log
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [wandb_init.py:init():375] calling init triggers
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [wandb_init.py:init():382] wandb.init called with sweep_config: {}
+config: {}
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [wandb_init.py:init():424] starting backend
+2021-09-07 09:43:06,316 INFO    MainThread:92593 [backend.py:_multiprocessing_setup():72] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2021-09-07 09:43:06,328 INFO    MainThread:92593 [backend.py:ensure_launched():135] starting backend process...
+2021-09-07 09:43:06,336 INFO    MainThread:92593 [backend.py:ensure_launched():140] started backend process with pid: 92771
+2021-09-07 09:43:06,337 INFO    MainThread:92593 [wandb_init.py:init():429] backend started and connected
+2021-09-07 09:43:06,339 INFO    MainThread:92593 [wandb_init.py:init():477] updated telemetry
+2021-09-07 09:43:06,340 INFO    MainThread:92593 [wandb_init.py:init():500] communicating current version
+2021-09-07 09:43:07,053 INFO    MainThread:92593 [wandb_init.py:init():505] got version response 
+2021-09-07 09:43:07,053 INFO    MainThread:92593 [wandb_init.py:init():513] communicating run to backend with 30 second timeout
+2021-09-07 09:43:07,216 INFO    MainThread:92593 [wandb_init.py:init():540] starting run threads in backend
+2021-09-07 09:43:08,967 INFO    MainThread:92593 [wandb_run.py:_console_start():1601] atexit reg
+2021-09-07 09:43:08,967 INFO    MainThread:92593 [wandb_run.py:_redirect():1475] redirect: SettingsConsole.REDIRECT
+2021-09-07 09:43:08,969 INFO    MainThread:92593 [wandb_run.py:_redirect():1480] Redirecting console.
+2021-09-07 09:43:08,971 INFO    MainThread:92593 [wandb_run.py:_redirect():1536] Redirects installed.
+2021-09-07 09:43:08,971 INFO    MainThread:92593 [wandb_init.py:init():565] run started, returning control to user process