From 8cb412eb189f8a64b7a8b00871b07a125e1b1f01 Mon Sep 17 00:00:00 2001 From: rasbt Date: Wed, 19 Feb 2025 19:19:18 -0600 Subject: [PATCH 1/6] improve setup --- .github/workflows/basic-tests-linux-uv.yml | 3 +- .github/workflows/basic-tests-macos-uv.yml | 3 +- .github/workflows/basic-tests-windows-uv.yml | 60 ++++++++++++++++++++ requirements.txt | 3 +- 4 files changed, 63 insertions(+), 6 deletions(-) create mode 100644 .github/workflows/basic-tests-windows-uv.yml diff --git a/.github/workflows/basic-tests-linux-uv.yml b/.github/workflows/basic-tests-linux-uv.yml index 2ce889ef0..82c7e567a 100644 --- a/.github/workflows/basic-tests-linux-uv.yml +++ b/.github/workflows/basic-tests-linux-uv.yml @@ -39,8 +39,7 @@ jobs: shell: bash run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.10 - uv add . --dev + uv sync --dev --python 3.11 uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv add pytest-ruff nbval diff --git a/.github/workflows/basic-tests-macos-uv.yml b/.github/workflows/basic-tests-macos-uv.yml index b0c584458..138467bea 100644 --- a/.github/workflows/basic-tests-macos-uv.yml +++ b/.github/workflows/basic-tests-macos-uv.yml @@ -39,8 +39,7 @@ jobs: shell: bash run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.10 - uv add . --dev + uv sync --dev --python 3.11 uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv add pytest-ruff nbval diff --git a/.github/workflows/basic-tests-windows-uv.yml b/.github/workflows/basic-tests-windows-uv.yml new file mode 100644 index 000000000..fe0ab0311 --- /dev/null +++ b/.github/workflows/basic-tests-windows-uv.yml @@ -0,0 +1,60 @@ +name: Code tests Windows (uv) + +on: + push: + branches: [ main ] + paths: + - '**/*.py' + - '**/*.ipynb' + - '**/*.yaml' + - '**/*.yml' + - '**/*.sh' + pull_request: + branches: [ main ] + paths: + - '**/*.py' + - '**/*.ipynb' + - '**/*.yaml' + - '**/*.yml' + - '**/*.sh' + +jobs: + test: + runs-on: windows-latest + + steps: + - name: Checkout Code + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install dependencies + shell: pwsh + run: | + powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" + uv python install 3.11 + $env:UV_PIP_OPTS="--no-binary tensorflow-io-gcs-filesystem" + uv sync --dev + uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt + uv pip install pytest-ruff nbval + + - name: Run Python Tests + shell: pwsh + run: | + .\.venv\Scripts\Activate.ps1 + pytest --ruff setup/02_installing-python-libraries/tests.py + pytest --ruff ch04/01_main-chapter-code/tests.py + pytest --ruff ch05/01_main-chapter-code/tests.py + pytest --ruff ch05/07_gpt_to_llama/tests/tests.py + pytest --ruff ch06/01_main-chapter-code/tests.py + + - name: Run Jupyter Notebook Tests + shell: pwsh + run: | + .\.venv\Scripts\Activate.ps1 + pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb + pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb + pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb diff --git a/requirements.txt b/requirements.txt index 60d486a65..985933eec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,7 @@ torch >= 2.3.0 # all jupyterlab >= 4.0 # all tiktoken >= 0.5.1 # ch02; ch04; ch05 matplotlib >= 3.7.1 # ch04; ch05 -tensorflow>=2.18.0; sys_platform != "win32" # ch05 (non-Windows) -tensorflow-cpu>=2.18.0; sys_platform == "win32" # ch05 (Windows) +tensorflow>=2.18.0 # ch05; ch06; ch07 tqdm >= 4.66.1 # ch05; ch07 numpy >= 1.26, < 2.1 # dependency of several other libraries like torch and pandas pandas >= 2.2.1 # ch06 From b13abe3187f5eca9b7449efaf42b998d1c54bcf0 Mon Sep 17 00:00:00 2001 From: rasbt Date: Wed, 19 Feb 2025 19:22:29 -0600 Subject: [PATCH 2/6] improve setup --- .github/workflows/basic-tests-windows-uv.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/basic-tests-windows-uv.yml b/.github/workflows/basic-tests-windows-uv.yml index fe0ab0311..3539be787 100644 --- a/.github/workflows/basic-tests-windows-uv.yml +++ b/.github/workflows/basic-tests-windows-uv.yml @@ -35,6 +35,7 @@ jobs: shell: pwsh run: | powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" + $env:Path = "C:\Users\runneradmin\.local\bin;$env:Path" uv python install 3.11 $env:UV_PIP_OPTS="--no-binary tensorflow-io-gcs-filesystem" uv sync --dev From 6b3b8d545b1a0e23de78b8e76022206a96dea5e8 Mon Sep 17 00:00:00 2001 From: rasbt Date: Wed, 19 Feb 2025 19:27:23 -0600 Subject: [PATCH 3/6] use env --- .github/workflows/basic-tests-windows-uv.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/basic-tests-windows-uv.yml b/.github/workflows/basic-tests-windows-uv.yml index 3539be787..d49a93cef 100644 --- a/.github/workflows/basic-tests-windows-uv.yml +++ b/.github/workflows/basic-tests-windows-uv.yml @@ -45,17 +45,15 @@ jobs: - name: Run Python Tests shell: pwsh run: | - .\.venv\Scripts\Activate.ps1 - pytest --ruff setup/02_installing-python-libraries/tests.py - pytest --ruff ch04/01_main-chapter-code/tests.py - pytest --ruff ch05/01_main-chapter-code/tests.py - pytest --ruff ch05/07_gpt_to_llama/tests/tests.py - pytest --ruff ch06/01_main-chapter-code/tests.py + .venv\Scripts\python -m pytest --ruff setup/02_installing-python-libraries/tests.py + .venv\Scripts\python -m pytest --ruff ch04/01_main-chapter-code/tests.py + .venv\Scripts\python -m pytest --ruff ch05/01_main-chapter-code/tests.py + .venv\Scripts\python -m pytest --ruff ch05/07_gpt_to_llama/tests/tests.py + .venv\Scripts\python -m pytest --ruff ch06/01_main-chapter-code/tests.py - name: Run Jupyter Notebook Tests shell: pwsh run: | - .\.venv\Scripts\Activate.ps1 - pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb - pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb - pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb + .venv\Scripts\python -m pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb + .venv\Scripts\python -m pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb + .venv\Scripts\python -m pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb From 0e1d1efc106ba43e802c41ccb9e765ee1b2b07ad Mon Sep 17 00:00:00 2001 From: rasbt Date: Wed, 19 Feb 2025 19:33:20 -0600 Subject: [PATCH 4/6] powershell -> bash --- .../workflows/basic-tests-windows-uv-pip.yml | 10 +++--- .github/workflows/basic-tests-windows-uv.yml | 34 ++++++++++--------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/.github/workflows/basic-tests-windows-uv-pip.yml b/.github/workflows/basic-tests-windows-uv-pip.yml index bf7aeb150..a89bd25f7 100644 --- a/.github/workflows/basic-tests-windows-uv-pip.yml +++ b/.github/workflows/basic-tests-windows-uv-pip.yml @@ -39,10 +39,10 @@ jobs: pip install uv uv venv --python=python3.11 source .venv/Scripts/activate - pip install -r requirements.txt # because of dependency issue on Windows when using `uv pip` - pip install tensorflow-io-gcs-filesystem==0.31.0 # Explicit for Windows - pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt - pip install pytest-ruff nbval + uv pip install --no-binary tensorflow-io-gcs-filesystem + uv pip install -r requirements.txt + uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt + uv pip install pytest-ruff nbval - name: Run Python Tests shell: bash @@ -60,4 +60,4 @@ jobs: source .venv/Scripts/activate pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb - pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb \ No newline at end of file + pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb diff --git a/.github/workflows/basic-tests-windows-uv.yml b/.github/workflows/basic-tests-windows-uv.yml index d49a93cef..721868597 100644 --- a/.github/workflows/basic-tests-windows-uv.yml +++ b/.github/workflows/basic-tests-windows-uv.yml @@ -32,28 +32,30 @@ jobs: python-version: '3.11' - name: Install dependencies - shell: pwsh + shell: bash run: | - powershell -ExecutionPolicy ByPass -c "irm https://astral.sh/uv/install.ps1 | iex" - $env:Path = "C:\Users\runneradmin\.local\bin;$env:Path" + curl -fsSL https://astral.sh/uv/install.sh | bash + export PATH=$HOME/.local/bin:$PATH uv python install 3.11 - $env:UV_PIP_OPTS="--no-binary tensorflow-io-gcs-filesystem" + export UV_PIP_OPTS="--no-binary tensorflow-io-gcs-filesystem" uv sync --dev uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv pip install pytest-ruff nbval - - name: Run Python Tests - shell: pwsh + - name: Test Selected Python Scripts (uv) + shell: bash run: | - .venv\Scripts\python -m pytest --ruff setup/02_installing-python-libraries/tests.py - .venv\Scripts\python -m pytest --ruff ch04/01_main-chapter-code/tests.py - .venv\Scripts\python -m pytest --ruff ch05/01_main-chapter-code/tests.py - .venv\Scripts\python -m pytest --ruff ch05/07_gpt_to_llama/tests/tests.py - .venv\Scripts\python -m pytest --ruff ch06/01_main-chapter-code/tests.py + source .venv/bin/activate + pytest --ruff setup/02_installing-python-libraries/tests.py + pytest --ruff ch04/01_main-chapter-code/tests.py + pytest --ruff ch05/01_main-chapter-code/tests.py + pytest --ruff ch05/07_gpt_to_llama/tests/tests.py + pytest --ruff ch06/01_main-chapter-code/tests.py - - name: Run Jupyter Notebook Tests - shell: pwsh + - name: Validate Selected Jupyter Notebooks (uv) + shell: bash run: | - .venv\Scripts\python -m pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb - .venv\Scripts\python -m pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb - .venv\Scripts\python -m pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb + source .venv/bin/activate + pytest --ruff --nbval ch02/01_main-chapter-code/dataloader.ipynb + pytest --ruff --nbval ch03/01_main-chapter-code/multihead-attention.ipynb + pytest --ruff --nbval ch02/04_bonus_dataloader-intuition/dataloader-intuition.ipynb From 361a75d9e2ae58d9b45fbd72db79400164ff6555 Mon Sep 17 00:00:00 2001 From: rasbt Date: Wed, 19 Feb 2025 19:35:51 -0600 Subject: [PATCH 5/6] powershell -> bash --- .github/workflows/basic-tests-windows-uv-pip.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/basic-tests-windows-uv-pip.yml b/.github/workflows/basic-tests-windows-uv-pip.yml index a89bd25f7..a698440d5 100644 --- a/.github/workflows/basic-tests-windows-uv-pip.yml +++ b/.github/workflows/basic-tests-windows-uv-pip.yml @@ -39,7 +39,7 @@ jobs: pip install uv uv venv --python=python3.11 source .venv/Scripts/activate - uv pip install --no-binary tensorflow-io-gcs-filesystem + export UV_PIP_OPTS="--no-binary tensorflow-io-gcs-filesystem" uv pip install -r requirements.txt uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv pip install pytest-ruff nbval From ca0eee4cf94a74fb69de59af3a74d367e9d1dc2e Mon Sep 17 00:00:00 2001 From: rasbt Date: Wed, 19 Feb 2025 21:01:15 -0600 Subject: [PATCH 6/6] simplify and use pythorch 3.12 --- .github/workflows/basic-tests-linux-uv.yml | 4 ++-- .github/workflows/basic-tests-macos-uv.yml | 5 ++--- .github/workflows/basic-tests-old-pytorch.yml | 4 ++-- .github/workflows/basic-tests-pip.yml | 2 +- .github/workflows/basic-tests-pytorch-rc.yml | 6 +++--- .github/workflows/basic-tests-windows-uv-pip.yml | 4 ++-- .github/workflows/basic-tests-windows-uv.yml | 4 ++-- .github/workflows/check-links.yml | 5 ++--- .github/workflows/check-spelling-errors.yml | 5 ++--- .github/workflows/pep8-linter.yml | 5 ++--- 10 files changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/basic-tests-linux-uv.yml b/.github/workflows/basic-tests-linux-uv.yml index bef697bd9..6196f1497 100644 --- a/.github/workflows/basic-tests-linux-uv.yml +++ b/.github/workflows/basic-tests-linux-uv.yml @@ -33,13 +33,13 @@ jobs: - name: Set up Python (uv) uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install uv and dependencies shell: bash run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv sync --dev --python 3.11 + uv sync --dev --python 3.12 uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv add pytest-ruff nbval diff --git a/.github/workflows/basic-tests-macos-uv.yml b/.github/workflows/basic-tests-macos-uv.yml index 3dec430f8..e6c694e08 100644 --- a/.github/workflows/basic-tests-macos-uv.yml +++ b/.github/workflows/basic-tests-macos-uv.yml @@ -33,14 +33,13 @@ jobs: - name: Set up Python (uv) uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install uv and dependencies shell: bash run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv sync --dev --python 3.11 - uv python install 3.11 + uv sync --dev --python 3.12 uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv add pytest-ruff nbval diff --git a/.github/workflows/basic-tests-old-pytorch.yml b/.github/workflows/basic-tests-old-pytorch.yml index 116db3c30..210d2972d 100644 --- a/.github/workflows/basic-tests-old-pytorch.yml +++ b/.github/workflows/basic-tests-old-pytorch.yml @@ -31,12 +31,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.10" - name: Install dependencies run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.11 + uv python install 3.10 uv add . --dev uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv add torch==${{ matrix.pytorch-version }} diff --git a/.github/workflows/basic-tests-pip.yml b/.github/workflows/basic-tests-pip.yml index 504690fda..619694414 100644 --- a/.github/workflows/basic-tests-pip.yml +++ b/.github/workflows/basic-tests-pip.yml @@ -33,7 +33,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Create Virtual Environment and Install Dependencies run: | diff --git a/.github/workflows/basic-tests-pytorch-rc.yml b/.github/workflows/basic-tests-pytorch-rc.yml index 742bb23a4..0d5cef6b1 100644 --- a/.github/workflows/basic-tests-pytorch-rc.yml +++ b/.github/workflows/basic-tests-pytorch-rc.yml @@ -27,13 +27,13 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: "3.11" + python-version: "3.12" - name: Install dependencies run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.11 - uv add . --dev + uv python install 3.12 + uv sync --dev --python 3.12 uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt uv add pytest-ruff nbval uv pip install --pre torch torchvision torchaudio --index-url https://download.pytorch.org/whl/nightly/cpu diff --git a/.github/workflows/basic-tests-windows-uv-pip.yml b/.github/workflows/basic-tests-windows-uv-pip.yml index a698440d5..fa2c5b35f 100644 --- a/.github/workflows/basic-tests-windows-uv-pip.yml +++ b/.github/workflows/basic-tests-windows-uv-pip.yml @@ -29,7 +29,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install dependencies shell: bash @@ -37,7 +37,7 @@ jobs: export PATH="$HOME/.local/bin:$PATH" pip install --upgrade pip pip install uv - uv venv --python=python3.11 + uv venv --python=python3.12 source .venv/Scripts/activate export UV_PIP_OPTS="--no-binary tensorflow-io-gcs-filesystem" uv pip install -r requirements.txt diff --git a/.github/workflows/basic-tests-windows-uv.yml b/.github/workflows/basic-tests-windows-uv.yml index 721868597..283c8c11c 100644 --- a/.github/workflows/basic-tests-windows-uv.yml +++ b/.github/workflows/basic-tests-windows-uv.yml @@ -29,14 +29,14 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install dependencies shell: bash run: | curl -fsSL https://astral.sh/uv/install.sh | bash export PATH=$HOME/.local/bin:$PATH - uv python install 3.11 + uv python install 3.12 export UV_PIP_OPTS="--no-binary tensorflow-io-gcs-filesystem" uv sync --dev uv pip install -r ch05/07_gpt_to_llama/tests/test-requirements-extra.txt diff --git a/.github/workflows/check-links.yml b/.github/workflows/check-links.yml index 52e41d6a2..79bcd5aaa 100644 --- a/.github/workflows/check-links.yml +++ b/.github/workflows/check-links.yml @@ -18,13 +18,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install dependencies run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.11 - uv add . --dev + uv sync --dev --python 3.12 uv add pytest-ruff pytest-check-links # Current version of retry doesn't work well if there are broken non-URL links # pip install pytest pytest-check-links pytest-retry diff --git a/.github/workflows/check-spelling-errors.yml b/.github/workflows/check-spelling-errors.yml index d347a2ee2..d256cc77b 100644 --- a/.github/workflows/check-spelling-errors.yml +++ b/.github/workflows/check-spelling-errors.yml @@ -18,13 +18,12 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install codespell run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.11 - uv add . --dev + uv sync --dev --python 3.12 uv add codespell - name: Run codespell diff --git a/.github/workflows/pep8-linter.yml b/.github/workflows/pep8-linter.yml index aa9de7f69..87e8b9ba3 100644 --- a/.github/workflows/pep8-linter.yml +++ b/.github/workflows/pep8-linter.yml @@ -14,12 +14,11 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.11' + python-version: '3.12' - name: Install ruff (a faster flake 8 equivalent) run: | curl -LsSf https://astral.sh/uv/install.sh | sh - uv python install 3.11 - uv add . --dev + uv sync --dev --python 3.12 uv add ruff - name: Run ruff with exceptions