Skip to content

Commit 5d3e385

Browse files
holly1238holly1238
authored
Merge branch 'master' into master
2 parents 85b994f + 515c64e commit 5d3e385

File tree

204 files changed

+21964
-4158
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

204 files changed

+21964
-4158
lines changed

‎.circleci/config.yml

Lines changed: 109 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
version: 2.1
22

3+
executors:
4+
windows-with-nvidia-gpu:
5+
machine:
6+
resource_class: windows.gpu.nvidia.medium
7+
image: windows-server-2019-nvidia:stable
8+
shell: bash.exe
9+
310
install_official_git_client: &install_official_git_client
411
name: Install Official Git Client
512
no_output_timeout: "1h"
@@ -99,7 +106,7 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
99106
sudo pip -q install awscli==1.16.35
100107
101108
if [ -n "${CUDA_VERSION}" ]; then
102-
DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
109+
DRIVER_FN="NVIDIA-Linux-x86_64-460.39.run"
103110
wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
104111
sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
105112
nvidia-smi
@@ -115,7 +122,7 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
115122
command: |
116123
set -e
117124
118-
export pyTorchDockerImageTag=291
125+
export pyTorchDockerImageTag=9de29bef4a5dc0dd1dd19428d83e5a66a44a1ed2
119126
echo "PyTorchDockerImageTag: "${pyTorchDockerImageTag}
120127
121128
cat >/home/circleci/project/ci_build_script.sh \<<EOL
@@ -164,17 +171,46 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
164171
165172
pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
166173
environment:
167-
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3"
174+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
168175
CUDA_VERSION: "9"
169176
resource_class: gpu.medium
170177
<<: *pytorch_tutorial_build_defaults
171178

172179
pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
173180
environment:
174-
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3"
181+
DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
175182
resource_class: medium
176183
<<: *pytorch_tutorial_build_defaults
177184

185+
pytorch_windows_build_worker: &pytorch_windows_build_worker
186+
executor: windows-with-nvidia-gpu
187+
steps:
188+
- checkout
189+
- run:
190+
name: Install Cuda
191+
no_output_timeout: 30m
192+
command: |
193+
.circleci/scripts/windows_cuda_install.sh
194+
- run:
195+
name: Generate cache key
196+
# This will refresh cache on Sundays, build should generate new cache.
197+
command: echo "$(date +"%Y-%U")" > .circleci-weekly
198+
- restore_cache:
199+
keys:
200+
- data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
201+
- run:
202+
name: test
203+
no_output_timeout: "1h"
204+
command: |
205+
.circleci/scripts/build_for_windows.sh
206+
- save_cache:
207+
key: data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
208+
paths:
209+
- advanced_source/data
210+
- beginner_source/data
211+
- intermediate_source/data
212+
- prototype_source/data
213+
178214
jobs:
179215
pytorch_tutorial_pr_build_worker_0:
180216
<<: *pytorch_tutorial_build_worker_defaults
@@ -302,6 +338,30 @@ jobs:
302338
pytorch_tutorial_master_build_manager:
303339
<<: *pytorch_tutorial_build_manager_defaults
304340

341+
pytorch_tutorial_windows_pr_build_worker_0:
342+
<<: *pytorch_windows_build_worker
343+
344+
pytorch_tutorial_windows_pr_build_worker_1:
345+
<<: *pytorch_windows_build_worker
346+
347+
pytorch_tutorial_windows_pr_build_worker_2:
348+
<<: *pytorch_windows_build_worker
349+
350+
pytorch_tutorial_windows_pr_build_worker_3:
351+
<<: *pytorch_windows_build_worker
352+
353+
pytorch_tutorial_windows_master_build_worker_0:
354+
<<: *pytorch_windows_build_worker
355+
356+
pytorch_tutorial_windows_master_build_worker_1:
357+
<<: *pytorch_windows_build_worker
358+
359+
pytorch_tutorial_windows_master_build_worker_2:
360+
<<: *pytorch_windows_build_worker
361+
362+
pytorch_tutorial_windows_master_build_worker_3:
363+
<<: *pytorch_windows_build_worker
364+
305365
workflows:
306366
build:
307367
jobs:
@@ -538,3 +598,48 @@ workflows:
538598
branches:
539599
only:
540600
- master
601+
- pytorch_tutorial_windows_pr_build_worker_0:
602+
filters:
603+
branches:
604+
ignore:
605+
- master
606+
- pytorch_tutorial_windows_pr_build_worker_1:
607+
filters:
608+
branches:
609+
ignore:
610+
- master
611+
- pytorch_tutorial_windows_pr_build_worker_2:
612+
filters:
613+
branches:
614+
ignore:
615+
- master
616+
- pytorch_tutorial_windows_pr_build_worker_3:
617+
filters:
618+
branches:
619+
ignore:
620+
- master
621+
- pytorch_tutorial_windows_master_build_worker_0:
622+
context: org-member
623+
filters:
624+
branches:
625+
only:
626+
- master
627+
- pytorch_tutorial_windows_master_build_worker_1:
628+
context: org-member
629+
filters:
630+
branches:
631+
only:
632+
- master
633+
- pytorch_tutorial_windows_master_build_worker_2:
634+
context: org-member
635+
filters:
636+
branches:
637+
only:
638+
- master
639+
- pytorch_tutorial_windows_master_build_worker_3:
640+
context: org-member
641+
filters:
642+
branches:
643+
only:
644+
- master
645+

‎.circleci/scripts/build_for_windows.sh

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
retry () {
5+
$* || (sleep 1 && $*) || (sleep 2 && $*)
6+
}
7+
8+
SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
9+
PROJECT_DIR="${SOURCE_DIR}/../.."
10+
pushd $SOURCE_DIR
11+
12+
#install wget and make
13+
curl -k https://ymu.dl.osdn.jp/mingw/68260/mingw-get-0.6.3-mingw32-pre-20170905-1-bin.zip -o mingw32.zip
14+
unzip mingw32.zip -d mingw32
15+
mingw32/bin/mingw-get.exe install mingw32-make
16+
mingw32/bin/mingw-get.exe install msys-findutils
17+
mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
18+
curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
19+
export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
20+
21+
#install anaconda3
22+
export CONDA_HOME="${SOURCE_DIR}/conda"
23+
export tmp_conda="${SOURCE_DIR}/conda"
24+
export miniconda_exe="${SOURCE_DIR}/miniconda.exe"
25+
rm -rf conda miniconda.exe
26+
curl -k https://repo.anaconda.com/miniconda/Miniconda3-latest-Windows-x86_64.exe -o miniconda.exe
27+
./install_conda.bat
28+
export PATH="${tmp_conda}:${tmp_conda}/Library/usr/bin:${tmp_conda}/Library/bin:${tmp_conda}/Scripts:${tmp_conda}/bin:$PATH"
29+
30+
eval "$(conda shell.bash hook)"
31+
conda create -qyn testenv python=3.7
32+
conda activate testenv
33+
34+
REQUIREMENTS="$(grep -v '^ *#\|^torch\|^torchaudio\|^torchvision|^torchtext' $PROJECT_DIR/requirements.txt | grep .)"
35+
echo $REQUIREMENTS > requirements.txt
36+
pip install -r requirements.txt
37+
pip install pySoundFile
38+
# Force uninstall torch & related packages, we'll install them using conda later.
39+
pip uninstall -y torch torchvision torchtext
40+
conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext torchaudio
41+
python -m spacy download de
42+
python -m spacy download en
43+
pushd ${PROJECT_DIR}
44+
DIR=.jenkins
45+
export NUM_WORKERS=4
46+
47+
if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
48+
python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
49+
python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
50+
python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
51+
python $DIR/remove_runnable_code.py beginner_source/audio_preprocessing_tutorial.py beginner_source/audio_preprocessing_tutorial.py || true
52+
# Temp remove for mnist download issue.
53+
python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py beginner_source/fgsm_tutorial.py || true
54+
55+
export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
56+
count=0
57+
FILES_TO_RUN=()
58+
for work_directory in beginner_source intermediate_source advanced_source recipes_source prototype_source; do
59+
for filename in $(find $work_directory -name '\*.py' -not -path '\*/data/\*'); do
60+
if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
61+
echo "Removing runnable code from "$filename
62+
python $DIR/remove_runnable_code.py $filename $filename
63+
else
64+
echo "Keeping "$filename
65+
FILES_TO_RUN+=($(basename $filename .py))
66+
fi
67+
count=$((count+1))
68+
done
69+
done
70+
echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
71+
fi
72+
73+
if [[ ! -d advanced_source/data || ! -d beginner_source/data || ! -d intermediate_source/data || ! -d prototype_source/data ]];then
74+
make download
75+
fi
76+
77+
make html

‎.circleci/scripts/install_conda.bat

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
start /wait "" "%miniconda_exe%" /S /InstallationType=JustMe /RegisterPython=0 /AddToPath=0 /D=%tmp_conda%

‎.circleci/scripts/windows_cuda_install.sh

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
#!/bin/bash
2+
set -eux -o pipefail
3+
4+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/cuda_10.1.243_426.00_win10.exe
5+
7z x cuda_10.1.243_426.00_win10.exe -ocuda_10.1.243_426.00_win10
6+
cd cuda_10.1.243_426.00_win10
7+
mkdir cuda_install_logs
8+
9+
set +e
10+
11+
./setup.exe -s nvcc_10.1 cuobjdump_10.1 nvprune_10.1 cupti_10.1 cublas_10.1 cublas_dev_10.1 cudart_10.1 cufft_10.1 cufft_dev_10.1 curand_10.1 curand_dev_10.1 cusolver_10.1 cusolver_dev_10.1 cusparse_10.1 cusparse_dev_10.1 nvgraph_10.1 nvgraph_dev_10.1 npp_10.1 npp_dev_10.1 nvrtc_10.1 nvrtc_dev_10.1 nvml_dev_10.1 -loglevel:6 -log:"$(pwd -W)/cuda_install_logs"
12+
13+
set -e
14+
15+
curl --retry 3 -kLO https://ossci-windows.s3.amazonaws.com/NvToolsExt.7z
16+
7z x NvToolsExt.7z -oNvToolsExt
17+
mkdir -p "C:/Program Files/NVIDIA Corporation/NvToolsExt"
18+
cp -r NvToolsExt/* "C:/Program Files/NVIDIA Corporation/NvToolsExt/"
19+
export NVTOOLSEXT_PATH="C:\\Program Files\\NVIDIA Corporation\\NvToolsExt\\"
20+
21+
if ! ls "/c/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v10.1/bin/nvcc.exe"
22+
then
23+
echo "CUDA installation failed"
24+
mkdir -p /c/w/build-results
25+
7z a "c:\\w\\build-results\\cuda_install_logs.7z" cuda_install_logs
26+
exit 1
27+
fi
28+
29+
cd ..
30+
rm -rf ./cuda_10.1.243_426.00_win10
31+
rm -f ./cuda_10.1.243_426.00_win10.exe

‎.devcontainer/Dockerfile

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
FROM mcr.microsoft.com/vscode/devcontainers/python:3.8
2+
3+
COPY requirements.txt /tmp/pip-tmp/
4+
5+
RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
6+
&& apt-get install git gcc unzip make -y \
7+
&& pip3 install --disable-pip-version-check --no-cache-dir -r /tmp/pip-tmp/requirements.txt \
8+
&& rm -rf /tmp/pip-tmp

‎.devcontainer/devcontainer.json

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
{
2+
"name": "PyTorch Tutorials",
3+
"build": {
4+
"context": "..",
5+
"dockerfile": "Dockerfile",
6+
"args": {}
7+
},
8+
"settings": {
9+
"terminal.integrated.shell.linux": "/bin/bash",
10+
"workbench.startupEditor": "none",
11+
"files.autoSave": "afterDelay",
12+
"python.dataScience.enabled": true,
13+
"python.dataScience.alwaysTrustNotebooks": true,
14+
"python.insidersChannel": "weekly",
15+
"python.showStartPage": false
16+
},
17+
"extensions": ["ms-python.python", "lextudio.restructuredtext"]
18+
}

‎.devcontainer/requirements.txt

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
# Refer to ./jenkins/build.sh for tutorial build instructions
2+
3+
sphinx==1.8.2
4+
sphinx-gallery==0.3.1
5+
tqdm
6+
numpy
7+
matplotlib
8+
torch
9+
torchvision
10+
torchtext
11+
torchaudio
12+
PyHamcrest
13+
bs4
14+
awscli==1.16.35
15+
flask
16+
spacy
17+
ray[tune]
18+
19+
# PyTorch Theme
20+
-e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
21+
22+
ipython
23+
24+
# to run examples
25+
pandas
26+
scikit-image
27+
pillow==8.1.1
28+
wget
29+
30+
# for codespaces env
31+
pylint

‎.gitignore

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@ beginner
33
intermediate
44
advanced
55
pytorch_basics
6+
recipes
7+
prototype
68

79
#data things
810
_data/
@@ -31,6 +33,7 @@ __pycache__/
3133
*.so
3234

3335
# Distribution / packaging
36+
src/
3437
.Python
3538
env/
3639
build/
@@ -115,3 +118,6 @@ ENV/
115118
.DS_Store
116119
cleanup.sh
117120
*.swp
121+
122+
# PyTorch things
123+
*.pt

0 commit comments

Comments
 (0)