From 697ef9b7fd94a91ffcbff297d374198096371adb Mon Sep 17 00:00:00 2001 From: Farzad Abdolhosseini Date: Wed, 29 May 2024 16:35:43 -0700 Subject: [PATCH] remove legacy directory --- legacy/Justfile | 25 - legacy/README.md | 94 - legacy/audio_generation_time.md | 35 - legacy/azureml/configs/audiollm.yml | 29 - legacy/journal.md | 89 - legacy/ltu2_notes.md | 9 - legacy/src/ultravox/.gitignore | 167 - legacy/src/ultravox/__init__.py | 0 legacy/src/ultravox/inference/asr_eval.py | 43 - legacy/src/ultravox/inference/epd_eval.py | 61 - .../ultravox/inference/epd_eval_results.md | 57 - legacy/src/ultravox/inference/gradio_demo.py | 174 - .../src/ultravox/inference/run_audio_llm.py | 217 - .../ultravox/inference/speechlm_inference.py | 231 - legacy/src/ultravox/poetry.lock | 6289 ----------------- legacy/src/ultravox/pyproject.toml | 43 - legacy/src/ultravox/requirements.txt | 21 - legacy/src/ultravox/runjob.sh | 18 - legacy/src/ultravox/train/__init__.py | 0 legacy/src/ultravox/train/config.yaml | 72 - legacy/src/ultravox/train/configs_base.py | 88 - legacy/src/ultravox/train/data/__init__.py | 15 - legacy/src/ultravox/train/data/base.py | 365 - legacy/src/ultravox/train/data/epd.py | 62 - legacy/src/ultravox/train/data/gigaspeech.py | 82 - legacy/src/ultravox/train/data/prompts.py | 113 - legacy/src/ultravox/train/env.py | 29 - legacy/src/ultravox/train/models/__init__.py | 0 .../ultravox/train/models/audio/__init__.py | 3 - .../ultravox/train/models/audio/encoders.py | 143 - .../ultravox/train/models/audio/mhubert.py | 194 - .../train/models/multimodal/__init__.py | 6 - .../train/models/multimodal/audio_llm.py | 391 - .../train/models/multimodal/config.py | 90 - .../train/models/multimodal/processors.py | 26 - .../ultravox/train/models/text/__init__.py | 5 - legacy/src/ultravox/train/models/text/llms.py | 38 - legacy/src/ultravox/train/models/utils.py | 34 - legacy/src/ultravox/train/train.py | 258 - 39 files changed, 9616 deletions(-) delete mode 100644 legacy/Justfile delete mode 100644 legacy/README.md delete mode 100644 legacy/audio_generation_time.md delete mode 100644 legacy/azureml/configs/audiollm.yml delete mode 100644 legacy/journal.md delete mode 100644 legacy/ltu2_notes.md delete mode 100644 legacy/src/ultravox/.gitignore delete mode 100644 legacy/src/ultravox/__init__.py delete mode 100644 legacy/src/ultravox/inference/asr_eval.py delete mode 100644 legacy/src/ultravox/inference/epd_eval.py delete mode 100644 legacy/src/ultravox/inference/epd_eval_results.md delete mode 100644 legacy/src/ultravox/inference/gradio_demo.py delete mode 100644 legacy/src/ultravox/inference/run_audio_llm.py delete mode 100644 legacy/src/ultravox/inference/speechlm_inference.py delete mode 100644 legacy/src/ultravox/poetry.lock delete mode 100644 legacy/src/ultravox/pyproject.toml delete mode 100644 legacy/src/ultravox/requirements.txt delete mode 100755 legacy/src/ultravox/runjob.sh delete mode 100644 legacy/src/ultravox/train/__init__.py delete mode 100644 legacy/src/ultravox/train/config.yaml delete mode 100644 legacy/src/ultravox/train/configs_base.py delete mode 100644 legacy/src/ultravox/train/data/__init__.py delete mode 100644 legacy/src/ultravox/train/data/base.py delete mode 100644 legacy/src/ultravox/train/data/epd.py delete mode 100644 legacy/src/ultravox/train/data/gigaspeech.py delete mode 100644 legacy/src/ultravox/train/data/prompts.py delete mode 100644 legacy/src/ultravox/train/env.py delete mode 100644 legacy/src/ultravox/train/models/__init__.py delete mode 100644 legacy/src/ultravox/train/models/audio/__init__.py delete mode 100644 legacy/src/ultravox/train/models/audio/encoders.py delete mode 100644 legacy/src/ultravox/train/models/audio/mhubert.py delete mode 100644 legacy/src/ultravox/train/models/multimodal/__init__.py delete mode 100644 legacy/src/ultravox/train/models/multimodal/audio_llm.py delete mode 100644 legacy/src/ultravox/train/models/multimodal/config.py delete mode 100644 legacy/src/ultravox/train/models/multimodal/processors.py delete mode 100644 legacy/src/ultravox/train/models/text/__init__.py delete mode 100644 legacy/src/ultravox/train/models/text/llms.py delete mode 100644 legacy/src/ultravox/train/models/utils.py delete mode 100644 legacy/src/ultravox/train/train.py diff --git a/legacy/Justfile b/legacy/Justfile deleted file mode 100644 index 13ebd912..00000000 --- a/legacy/Justfile +++ /dev/null @@ -1,25 +0,0 @@ -export WANDB_PROJECT:="ultravox" -export WANDB_LOG_MODEL:="checkpoint" -export PROJECT_DIR:="ultravox" - -poetry *FLAGS: - cd ${PROJECT_DIR} && poetry {{FLAGS}} - -python *FLAGS: - cd ${PROJECT_DIR} && poetry run python {{FLAGS}} - -format: - cd ${PROJECT_DIR} && poetry run autoflake . --remove-all-unused-imports --quiet --in-place -r --exclude third_party - cd ${PROJECT_DIR} && poetry run isort . --force-single-line-imports - cd ${PROJECT_DIR} && poetry run black . - -check: - cd ${PROJECT_DIR} && poetry check - cd ${PROJECT_DIR} && poetry run black . --check - cd ${PROJECT_DIR} && poetry run isort . --check --force-single-line-imports - cd ${PROJECT_DIR} && poetry run autoflake . --check --quiet --remove-all-unused-imports -r --exclude third_party - cd ${PROJECT_DIR} && poetry run mypy . - cd ${PROJECT_DIR} && poetry run deptry . - -test *ARGS="--dist loadgroup -n auto .": - cd ${PROJECT_DIR} && poetry run pytest --ignore third_party {{ARGS}} diff --git a/legacy/README.md b/legacy/README.md deleted file mode 100644 index 4497d3d5..00000000 --- a/legacy/README.md +++ /dev/null @@ -1,94 +0,0 @@ -# UltraVox - -## AzureML - -### Installation and Config - -```bash -brew update && brew install azure-cli -az extension add --name ml --yes - -az login -az account set --subscription 520aa0b2-6a19-4a45-8c03-4c301d1f847a -az configure --defaults workspace=gpu-supercomput -``` - -```bash -az ml job create -f ./azureml/configs/audiollm.yml --web -``` - -## Random Documentation - -### LLM + AudioEnc (ours) vs SpeechGPT - -```python -# SpeechGPT adds new tokens to the embedding and then trains them -nn.Embedding(32000, 2048) + nn.Embedding(4000, 2048) # old text tokens + new audio tokens -nn.Embedding(36000, 2048) - -### -# In other words: -### - -# SpeechGPT tokenizes audio and text separately, then concatenates the embeddings -llm(embed(concat(audio_tokenizer(audio), text_token))) -## ------------------- vs ------------------- -# We create the audio embeddings directly from the audio and skip embedding the audio tokens -llm(concat(audio_enc(audio) * weight, embed(text_token))) -# This means we can easily propagate gradients to the audio encoder (i.e. train end to end) -``` - -### How does language modeling work with audio? - -```python -# t[n] <- t[1..n-1] -# The brown fox jumps over the fence - -# a1 a2 a3 a4 The brown fox jumps over the fence -# samples: -# a1 a2 a3 -> a4 -# a1 a2 a3 a4 -> The -# a1 a2 a3 a4 The -> brown -# a1 a2 a3 a4 The brown -> fox -# a1 a2 a3 a4 The brown fox -> jumps -# a1 a2 a3 a4 The brown fox jumps -> over -# a1 a2 a3 a4 The brown fox jumps over -> the -# a1 a2 a3 a4 The brown fox jumps over the -> fence -``` - -## TODO - -- [ ] generation metrics (low_pri: added more metrics to cover shifts) -- [x] more metrics to cover shifts -- [x] torchrun -- [ ] shard dataset -- [ ] datasets.distributed.split_dataset_by_node -- [ ] cache preprocessed data -- [ ]