Merge branch 'master' into automated_fixup_code_of_conduct_file_exists

Author: holly1238

.circleci/config.yml

@@ -106,7 +106,7 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
         sudo pip -q install awscli==1.16.35
 
         if [ -n "${CUDA_VERSION}" ]; then
-          DRIVER_FN="NVIDIA-Linux-x86_64-440.59.run"
+          DRIVER_FN="NVIDIA-Linux-x86_64-460.39.run"
           wget "https://s3.amazonaws.com/ossci-linux/nvidia_driver/$DRIVER_FN"
           sudo /bin/bash "$DRIVER_FN" -s --no-drm || (sudo cat /var/log/nvidia-installer.log && false)
           nvidia-smi
@@ -116,13 +116,15 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
         export AWS_ACCESS_KEY_ID=${CIRCLECI_AWS_ACCESS_KEY_FOR_ECR_READ_ONLY}
         export AWS_SECRET_ACCESS_KEY=${CIRCLECI_AWS_SECRET_KEY_FOR_ECR_READ_ONLY}
         eval $(aws ecr get-login --region us-east-1 --no-include-email)
+  - restore_cache:
+      key: v1.0-tutorial-{{ .Environment.CIRCLE_JOB }}
   - run:
       name: Build
       no_output_timeout: "20h"
       command: |
         set -e
 
-        export pyTorchDockerImageTag=291
+        export pyTorchDockerImageTag=9de29bef4a5dc0dd1dd19428d83e5a66a44a1ed2
         echo "PyTorchDockerImageTag: "${pyTorchDockerImageTag}
 
         cat >/home/circleci/project/ci_build_script.sh \<<EOL
@@ -164,25 +166,49 @@ pytorch_tutorial_build_defaults: &pytorch_tutorial_build_defaults
         fi
         set -x
 
+        # This also copies the cached build to docker.
         docker cp /home/circleci/project/. "$id:/var/lib/jenkins/workspace"
 
         export COMMAND='((echo "source ./workspace/env" && echo "sudo chown -R jenkins workspace && cd workspace && ./ci_build_script.sh") | docker exec -u jenkins -i "$id" bash) 2>&1'
         echo ${COMMAND} > ./command.sh && unbuffer bash ./command.sh | ts
 
+        # Copy the last build from docker
+        docker cp "$id:/var/lib/jenkins/workspace/_build" /home/circleci/project
+        docker cp "$id:/var/lib/jenkins/workspace/docs" /home/circleci/project
+        docker cp "$id:/var/lib/jenkins/workspace/advanced" /home/circleci/project
+        docker cp "$id:/var/lib/jenkins/workspace/beginner" /home/circleci/project
+        docker cp "$id:/var/lib/jenkins/workspace/intermediate" /home/circleci/project
+        docker cp "$id:/var/lib/jenkins/workspace/prototype" /home/circleci/project
+        docker cp "$id:/var/lib/jenkins/workspace/recipes" /home/circleci/project
+        docker cp "$id:/var/lib/jenkins/workspace/src" /home/circleci/project
+
+  - save_cache:
+      # Save to cache for incremental build
+      key: v1.0-tutorial-{{ .Environment.CIRCLE_JOB }}
+      paths:
+        - /home/circleci/project/_build
+        - /home/circleci/project/docs
+        - /home/circleci/project/advanced
+        - /home/circleci/project/beginner
+        - /home/circleci/project/intermediate
+        - /home/circleci/project/prototype
+        - /home/circleci/project/recipes
+        - /home/circleci/project/src
+
 pytorch_tutorial_build_worker_defaults: &pytorch_tutorial_build_worker_defaults
   environment:
-    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3"
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
     CUDA_VERSION: "9"
   resource_class: gpu.medium
   <<: *pytorch_tutorial_build_defaults
 
 pytorch_tutorial_build_manager_defaults: &pytorch_tutorial_build_manager_defaults
   environment:
-    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda9-cudnn7-py3"
+    DOCKER_IMAGE: "308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/pytorch-linux-xenial-cuda10.2-cudnn7-py3-gcc7"
   resource_class: medium
   <<: *pytorch_tutorial_build_defaults
 
-pytorch_windows_build: &pytorch_windows_build
+pytorch_windows_build_worker: &pytorch_windows_build_worker
   executor: windows-with-nvidia-gpu
   steps:
     - checkout
@@ -191,10 +217,25 @@ pytorch_windows_build: &pytorch_windows_build
         no_output_timeout: 30m
         command: |
           .circleci/scripts/windows_cuda_install.sh
+    - run:
+        name: Generate cache key
+        # This will refresh cache on Sundays, build should generate new cache.
+        command: echo "$(date +"%Y-%U")" > .circleci-weekly
+    - restore_cache:
+        keys:
+          - data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
     - run:
         name: test
+        no_output_timeout: "1h"
         command: |
           .circleci/scripts/build_for_windows.sh
+    - save_cache:
+        key: data-{{ checksum "Makefile" }}-{{ checksum ".circleci-weekly" }}
+        paths:
+          - advanced_source/data
+          - beginner_source/data
+          - intermediate_source/data
+          - prototype_source/data
 
 jobs:
   pytorch_tutorial_pr_build_worker_0:
@@ -323,8 +364,29 @@ jobs:
   pytorch_tutorial_master_build_manager:
     <<: *pytorch_tutorial_build_manager_defaults
 
-  pytorch_windows_build_worker:
-    <<: *pytorch_windows_build
+  pytorch_tutorial_windows_pr_build_worker_0:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_pr_build_worker_1:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_pr_build_worker_2:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_pr_build_worker_3:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_0:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_1:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_2:
+    <<: *pytorch_windows_build_worker
+
+  pytorch_tutorial_windows_master_build_worker_3:
+    <<: *pytorch_windows_build_worker
 
 workflows:
   build:
@@ -562,11 +624,48 @@ workflows:
             branches:
               only:
                 - master
-#      - pytorch_windows_build_worker:
-#          name: win_test_worker
-#          type: approval
-#          filters:
-#            branches:
-#              only:
-#                - master
+      - pytorch_tutorial_windows_pr_build_worker_0:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_pr_build_worker_1:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_pr_build_worker_2:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_pr_build_worker_3:
+          filters:
+            branches:
+              ignore:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_0:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_1:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_2:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
+      - pytorch_tutorial_windows_master_build_worker_3:
+          context: org-member
+          filters:
+            branches:
+              only:
+                - master
 

.circleci/scripts/build_for_windows.sh

@@ -16,7 +16,7 @@ mingw32/bin/mingw-get.exe install mingw32-make
 mingw32/bin/mingw-get.exe install msys-findutils
 mv mingw32/bin/mingw32-make.exe mingw32/bin/make.exe
 curl -k https://eternallybored.org/misc/wget/1.20.3/64/wget.exe -o mingw32/bin/wget.exe
-export PATH="${SOURCE_DIR}/mingw32/bin:$PATH"
+export PATH="${SOURCE_DIR}/mingw32/bin:${SOURCE_DIR}/mingw32/msys/1.0/bin:$PATH"
 
 #install anaconda3
 export CONDA_HOME="${SOURCE_DIR}/conda"
@@ -31,19 +31,49 @@ eval "$(conda shell.bash hook)"
 conda create -qyn testenv python=3.7
 conda activate testenv
 
-conda install sphinx
-pip install sphinx_gallery==0.3.1 flask pandas spacy ipython scipy pySoundFile scikit-image
-pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
-conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext
-conda install torchaudio -c pytorch-test
+REQUIREMENTS="$(grep -v '^ *#\|^torch\|^torchaudio\|^torchvision|^torchtext' $PROJECT_DIR/requirements.txt  | grep .)"
+echo $REQUIREMENTS > requirements.txt
+pip install -r requirements.txt
+pip install pySoundFile
+# Force uninstall torch & related packages, we'll install them using conda later.
+pip uninstall -y torch torchvision torchtext
+conda install -yq -c pytorch "cudatoolkit=10.1" pytorch torchvision torchtext torchaudio
 python -m spacy download de
 python -m spacy download en
 pushd ${PROJECT_DIR}
 DIR=.jenkins
-python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py || true
-python $DIR/remove_runnable_code.py beginner_source/data_loading_tutorial.py beginner_source/data_loading_tutorial.py || true
-python $DIR/remove_runnable_code.py beginner_source/dcgan_faces_tutorial.py beginner_source/dcgan_faces_tutorial.py || true
-python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
-python $DIR/remove_runnable_code.py intermediate_source/memory_format_tutorial.py intermediate_source/memory_format_tutorial.py || true
+export NUM_WORKERS=4
 
-make docs
+if [[ "${CIRCLE_JOB}" == *worker_* ]]; then
+  python $DIR/remove_runnable_code.py intermediate_source/model_parallel_tutorial.py intermediate_source/model_parallel_tutorial.py || true
+  python $DIR/remove_runnable_code.py advanced_source/static_quantization_tutorial.py advanced_source/static_quantization_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/hyperparameter_tuning_tutorial.py beginner_source/hyperparameter_tuning_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/audio_preprocessing_tutorial.py  beginner_source/audio_preprocessing_tutorial.py || true
+  python $DIR/remove_runnable_code.py beginner_source/dcgan_faces_tutorial.py  beginner_source/dcgan_faces_tutorial.py || true
+  python $DIR/remove_runnable_code.py intermediate_source/tensorboard_profiler_tutorial.py intermediate_source/tensorboard_profiler_tutorial.py || true
+  # Temp remove for mnist download issue. (Re-enabled for 1.8.1)
+  # python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py  beginner_source/fgsm_tutorial.py || true
+
+  export WORKER_ID=$(echo "${CIRCLE_JOB}" | tr -dc '0-9')
+  count=0
+  FILES_TO_RUN=()
+  for work_directory  in beginner_source intermediate_source advanced_source recipes_source prototype_source; do
+    for filename in $(find $work_directory -name '\*.py' -not -path '\*/data/\*'); do
+      if [ $(($count % $NUM_WORKERS)) != $WORKER_ID ]; then
+        echo "Removing runnable code from "$filename
+        python $DIR/remove_runnable_code.py $filename $filename
+      else
+        echo "Keeping "$filename
+        FILES_TO_RUN+=($(basename $filename .py))
+      fi
+      count=$((count+1))
+    done
+  done
+  echo "FILES_TO_RUN: " ${FILES_TO_RUN[@]}
+fi
+
+if [[ ! -d advanced_source/data || ! -d beginner_source/data || ! -d intermediate_source/data || ! -d prototype_source/data ]];then
+    make download
+fi
+
+make html

.devcontainer/Dockerfile

@@ -0,0 +1,8 @@
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.8
+
+COPY requirements.txt /tmp/pip-tmp/
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get install git gcc unzip make -y \
+    && pip3 install --disable-pip-version-check --no-cache-dir -r /tmp/pip-tmp/requirements.txt \
+    && rm -rf /tmp/pip-tmp

.devcontainer/devcontainer.json

@@ -0,0 +1,18 @@
+{
+  "name": "PyTorch Tutorials",
+  "build": {
+    "context": "..",
+    "dockerfile": "Dockerfile",
+    "args": {}
+  },
+  "settings": {
+    "terminal.integrated.shell.linux": "/bin/bash",
+    "workbench.startupEditor": "none",
+    "files.autoSave": "afterDelay",
+    "python.dataScience.enabled": true,
+    "python.dataScience.alwaysTrustNotebooks": true,
+    "python.insidersChannel": "weekly",
+    "python.showStartPage": false
+  },
+  "extensions": ["ms-python.python", "lextudio.restructuredtext"]
+}

.devcontainer/requirements.txt

@@ -0,0 +1,31 @@
+# Refer to ./jenkins/build.sh for tutorial build instructions
+
+sphinx==1.8.2
+sphinx-gallery==0.3.1
+tqdm
+numpy
+matplotlib
+torch
+torchvision
+torchtext
+torchaudio
+PyHamcrest
+bs4
+awscli==1.16.35
+flask
+spacy
+ray[tune]
+
+# PyTorch Theme
+-e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
+
+ipython
+
+# to run examples
+pandas
+scikit-image
+pillow==8.1.1
+wget
+
+# for codespaces env
+pylint

.gitignore

@@ -120,4 +120,4 @@ cleanup.sh
 *.swp
 
 # PyTorch things
-*.pt
+*.pt

.jenkins/build.sh

@@ -6,6 +6,10 @@ else
   export BUCKET_NAME=pytorch-tutorial-build-pull-request
 fi
 
+# set locale for click dependency in spacy
+export LC_ALL=C.UTF-8
+export LANG=C.UTF-8
+
 DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )"
 
 sudo apt-get update
@@ -18,6 +22,12 @@ pip install -r $DIR/../requirements.txt
 # export PATH=/opt/conda/bin:$PATH
 # pip install sphinx==1.8.2 pandas
 
+#Install PyTorch Nightly for test. 
+# Nightly - pip install --pre torch torchvision torchaudio -f https://download.pytorch.org/whl/nightly/cu102/torch_nightly.html
+# RC Link
+# pip uninstall -y torch torchvision torchaudio torchtext
+# pip install -f https://download.pytorch.org/whl/test/cu102/torch_test.html torch torchvision torchaudio torchtext
+
 # For Tensorboard. Until 1.14 moves to the release channel.
 pip install tb-nightly
 
@@ -28,9 +38,7 @@ python -m spacy download de
 # PyTorch Theme
 rm -rf src
 pip install -e git+git://github.com/pytorch/pytorch_sphinx_theme.git#egg=pytorch_sphinx_theme
-# pillow >= 4.2 will throw error when trying to write mode RGBA as JPEG,
-# this is a workaround to the issue.
-pip install sphinx-gallery==0.3.1 tqdm matplotlib ipython pillow==4.1.1
+pip install sphinx-gallery==0.3.1 tqdm matplotlib ipython pillow==8.1.0
 
 aws configure set default.s3.multipart_threshold 5120MB
 
@@ -39,6 +47,11 @@ export NUM_WORKERS=20
 if [[ "${JOB_BASE_NAME}" == *worker_* ]]; then
   # Step 1: Remove runnable code from tutorials that are not supposed to be run
   python $DIR/remove_runnable_code.py beginner_source/aws_distributed_training_tutorial.py beginner_source/aws_distributed_training_tutorial.py || true
+  # python $DIR/remove_runnable_code.py advanced_source/ddp_pipeline_tutorial.py advanced_source/ddp_pipeline_tutorial.py || true
+  # Temp remove for mnist download issue. (Re-enabled for 1.8.1)
+  # python $DIR/remove_runnable_code.py beginner_source/fgsm_tutorial.py beginner_source/fgsm_tutorial.py || true
+  # python $DIR/remove_runnable_code.py intermediate_source/spatial_transformer_tutorial.py intermediate_source/spatial_transformer_tutorial.py || true
+
   # TODO: Fix bugs in these tutorials to make them runnable again
   # python $DIR/remove_runnable_code.py beginner_source/audio_classifier_tutorial.py beginner_source/audio_classifier_tutorial.py || true
 

.jenkins/remove_runnable_code.py

@@ -16,9 +16,17 @@
             if line.startswith('#'):
                 ret_lines.append(line)
                 state = STATE_NORMAL
+            elif ((line.startswith('"""') or line.startswith('r"""')) and
+                    line.endswith('"""')):
+                ret_lines.append(line)
+                state = STATE_NORMAL
             elif line.startswith('"""') or line.startswith('r"""'):
                 ret_lines.append(line)
                 state = STATE_IN_MULTILINE_COMMENT_BLOCK_DOUBLE_QUOTE
+            elif ((line.startswith("'''") or line.startswith("r'''")) and
+                    line.endswith("'''")):
+                ret_lines.append(line)
+                state = STATE_NORMAL
             elif line.startswith("'''") or line.startswith("r'''"):
                 ret_lines.append(line)
                 state = STATE_IN_MULTILINE_COMMENT_BLOCK_SINGLE_QUOTE