-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathfaq.html
1181 lines (977 loc) · 67.1 KB
/
faq.html
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
<meta name="robots" content="noindex">
<meta charset="utf-8">
<meta name="generator" content="Docutils 0.18.1: http://docutils.sourceforge.net/" />
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Frequently Asked Questions — PyTorch 2.0 documentation</title>
<link rel="canonical" href="https://pytorch.org/docs/stable/torch.compiler_faq.html"/>
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<!-- <link rel="stylesheet" href="../_static/pygments.css" type="text/css" /> -->
<link rel="stylesheet" href="../_static/pygments.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/theme.css" type="text/css" />
<link rel="stylesheet" href="../_static/copybutton.css" type="text/css" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.css" type="text/css" />
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/katex.min.css" type="text/css" />
<link rel="stylesheet" href="../_static/katex-math.css" type="text/css" />
<link rel="stylesheet" href="../_static/sphinx-dropdown.css" type="text/css" />
<link rel="stylesheet" href="../_static/panels-bootstrap.min.css" type="text/css" />
<link rel="stylesheet" href="../_static/css/jit.css" type="text/css" />
<link rel="index" title="Index" href="../genindex.html" />
<link rel="search" title="Search" href="../search.html" />
<link rel="next" title="IRs" href="../ir.html" />
<link rel="prev" title="TorchDynamo Troubleshooting" href="troubleshooting.html" />
<!-- Google Analytics -->
<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>
<script>
window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-117752657-2');
</script>
<!-- End Google Analytics -->
<script src="../_static/js/modernizr.min.js"></script>
<!-- Preload the theme fonts -->
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<!-- Preload the katex fonts -->
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/[email protected]/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.15.2/css/all.css" integrity="sha384-vSIIfh2YWi9wW0r9iZe7RJPrKwp6bG+s9QZMoITbCckVJqGCCRhc+ccxNcdpHuYu" crossorigin="anonymous">
</head>
<div class="container-fluid header-holder tutorials-header" id="header-holder">
<div class="container">
<div class="header-container">
<a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
<div class="main-menu">
<ul>
<li>
<a href="https://pytorch.org/get-started">Get Started</a>
</li>
<li>
<a href="https://pytorch.org/ecosystem">Ecosystem</a>
</li>
<li>
<a href="https://pytorch.org/mobile">Mobile</a>
</li>
<li>
<a href="https://pytorch.org/blog/">Blog</a>
</li>
<li>
<a href="https://pytorch.org/tutorials">Tutorials</a>
</li>
<li class="active docs-active">
<div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
<a class="resource-option with-down-orange-arrow">
Docs
</a>
<div class="resources-dropdown-menu">
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/docs/stable/index.html">
<span class="dropdown-title">PyTorch</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/audio/stable/index.html">
<span class="dropdown-title">torchaudio</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/text/stable/index.html">
<span class="dropdown-title">torchtext</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/vision/stable/index.html">
<span class="dropdown-title">torchvision</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torcharrow">
<span class="dropdown-title">torcharrow</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/data">
<span class="dropdown-title">TorchData</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torchrec">
<span class="dropdown-title">TorchRec</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/serve/">
<span class="dropdown-title">TorchServe</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/torchx/">
<span class="dropdown-title">TorchX</span>
<p></p>
</a>
<a class="doc-dropdown-option nav-dropdown-item" href="https://pytorch.org/xla">
<span class="dropdown-title">PyTorch on XLA Devices</span>
<p></p>
</a>
</div>
</li>
<li>
<div id="resourcesDropdownButton" data-toggle="resources-dropdown" class="resources-dropdown">
<a class="resource-option with-down-arrow">
Resources
</a>
<div class="resources-dropdown-menu">
<a class="nav-dropdown-item" href="https://pytorch.org/features">
<span class="dropdown-title">About</span>
<p>Learn about PyTorch’s features and capabilities</p>
</a>
<a class="nav-dropdown-item" href="https://pytorch.org/foundation">
<span class="dropdown-title">PyTorch Foundation</span>
<p>Learn about the PyTorch foundation</p>
</a>
<a class="nav-dropdown-item" href="https://pytorch.org/#community-module">
<span class="dropdown-title">Community</span>
<p>Join the PyTorch developer community to contribute, learn, and get your questions answered.</p>
</a>
<a class="nav-dropdown-item" href="https://pytorch.org/community-stories">
<span class="dropdown-title">Community Stories</span>
<p>Learn how our community solves real, everyday machine learning problems with PyTorch.</p>
</a>
<a class="nav-dropdown-item" href="https://pytorch.org/resources">
<span class="dropdown-title">Developer Resources</span>
<p>Find resources and get questions answered</p>
</a>
<a class="nav-dropdown-item" href="https://pytorch.org/events">
<span class="dropdown-title">Events</span>
<p>Find events, webinars, and podcasts</p>
</a>
<a class="nav-dropdown-item" href="https://discuss.pytorch.org/" target="_blank">
<span class="dropdown-title">Forums</span>
<p>A place to discuss PyTorch code, issues, install, research</p>
</a>
<a class="nav-dropdown-item" href="https://pytorch.org/hub">
<span class="dropdown-title">Models (Beta)</span>
<p>Discover, publish, and reuse pre-trained models</p>
</a>
</div>
</div>
</li>
<li>
<a href="https://github.com/pytorch/pytorch">GitHub</a>
</li>
</ul>
</div>
<a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
</div>
</div>
</div>
<body class="pytorch-body">
<div class="table-of-contents-link-wrapper">
<span>Table of Contents</span>
<a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
</div>
<nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
<div class="pytorch-side-scroll">
<div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
<div class="pytorch-left-menu-search">
<div class="version">
<a href='https://pytorch.org/docs/versions.html'>2.0 ▼</a>
</div>
<div role="search">
<form id="rtd-search-form" class="wy-form" action="../search.html" method="get">
<input type="text" name="q" placeholder="Search Docs" />
<input type="hidden" name="check_keywords" value="yes" />
<input type="hidden" name="area" value="default" />
</form>
</div>
</div>
<p class="caption" role="heading"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../community/build_ci_governance.html">PyTorch Governance | Build + CI</a></li>
<li class="toctree-l1"><a class="reference internal" href="../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../community/design.html">PyTorch Design Philosophy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../community/governance.html">PyTorch Governance | Mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../community/persons_of_interest.html">PyTorch Governance | Maintainers</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Developer Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../notes/amp_examples.html">CUDA Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/extending.func.html">Extending torch.func with autograd.Function</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/gradcheck.html">Gradcheck mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/hip.html">HIP (ROCm) semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/modules.html">Modules</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/mps.html">MPS backend</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/numerical_accuracy.html">Numerical accuracy</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">torch.compile</span></p>
<ul class="current">
<li class="toctree-l1 current"><a class="reference internal" href="index.html">TorchDynamo Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="installation.html">Installing TorchDynamo</a></li>
<li class="toctree-l1"><a class="reference internal" href="get-started.html">Getting Started</a></li>
<li class="toctree-l1"><a class="reference internal" href="guards-overview.html">Guards Overview</a></li>
<li class="toctree-l1"><a class="reference internal" href="custom-backends.html">Custom Backends</a></li>
<li class="toctree-l1"><a class="reference internal" href="deep-dive.html">TorchDynamo Deeper Dive</a></li>
<li class="toctree-l1"><a class="reference internal" href="troubleshooting.html">TorchDynamo Troubleshooting</a></li>
<li class="toctree-l1 current"><a class="current reference internal" href="#">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ir.html">IRs</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../cpp_index.html">C++</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/javadoc/">Javadoc</a></li>
<li class="toctree-l1"><a class="reference internal" href="../deploy.html">torch::deploy</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../amp.html">torch.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../library.html">torch.library</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mps.html">torch.mps</a></li>
<li class="toctree-l1"><a class="reference internal" href="../backends.html">torch.backends</a></li>
<li class="toctree-l1"><a class="reference internal" href="../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../distributed.algorithms.join.html">torch.distributed.algorithms.join</a></li>
<li class="toctree-l1"><a class="reference internal" href="../distributed.elastic.html">torch.distributed.elastic</a></li>
<li class="toctree-l1"><a class="reference internal" href="../fsdp.html">torch.distributed.fsdp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../distributed.optim.html">torch.distributed.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../distributed.tensor.parallel.html">torch.distributed.tensor.parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../distributed.checkpoint.html">torch.distributed.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../_dynamo.html">torch._dynamo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../fft.html">torch.fft</a></li>
<li class="toctree-l1"><a class="reference internal" href="../func.html">torch.func</a></li>
<li class="toctree-l1"><a class="reference internal" href="../futures.html">torch.futures</a></li>
<li class="toctree-l1"><a class="reference internal" href="../fx.html">torch.fx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../linalg.html">torch.linalg</a></li>
<li class="toctree-l1"><a class="reference internal" href="../monitor.html">torch.monitor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../signal.html">torch.signal</a></li>
<li class="toctree-l1"><a class="reference internal" href="../special.html">torch.special</a></li>
<li class="toctree-l1"><a class="reference internal" href="../torch.overrides.html">torch.overrides</a></li>
<li class="toctree-l1"><a class="reference internal" href="../package.html">torch.package</a></li>
<li class="toctree-l1"><a class="reference internal" href="../profiler.html">torch.profiler</a></li>
<li class="toctree-l1"><a class="reference internal" href="../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../onnx_diagnostics.html">torch.onnx diagnostics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../complex_numbers.html">Complex Numbers</a></li>
<li class="toctree-l1"><a class="reference internal" href="../ddp_comm_hooks.html">DDP Communication Hooks</a></li>
<li class="toctree-l1"><a class="reference internal" href="../pipeline.html">Pipeline Parallelism</a></li>
<li class="toctree-l1"><a class="reference internal" href="../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../rpc.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../masked.html">torch.masked</a></li>
<li class="toctree-l1"><a class="reference internal" href="../nested.html">torch.nested</a></li>
<li class="toctree-l1"><a class="reference internal" href="../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../testing.html">torch.testing</a></li>
<li class="toctree-l1"><a class="reference internal" href="../benchmark_utils.html">torch.utils.benchmark</a></li>
<li class="toctree-l1"><a class="reference internal" href="../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../jit_utils.html">torch.utils.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../mobile_optimizer.html">torch.utils.mobile_optimizer</a></li>
<li class="toctree-l1"><a class="reference internal" href="../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../config_mod.html">torch.__config__</a></li>
</ul>
<p class="caption" role="heading"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio/stable">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/data">TorchData</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/torchrec">TorchRec</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text/stable">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/vision/stable">torchvision</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
</div>
</div>
</nav>
<div class="pytorch-container">
<div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
<div class="pytorch-breadcrumbs-wrapper">
<div role="navigation" aria-label="breadcrumbs navigation">
<ul class="pytorch-breadcrumbs">
<li>
<a href="../index.html">
Docs
</a> >
</li>
<li><a href="index.html">TorchDynamo Overview</a> ></li>
<li>Frequently Asked Questions</li>
<li class="pytorch-breadcrumbs-aside">
<a href="../_sources/dynamo/faq.rst.txt" rel="nofollow"><img src="../_static/images/view-page-source-icon.svg"></a>
</li>
</ul>
</div>
</div>
<div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
Shortcuts
</div>
</div>
<section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
<div class="pytorch-content-left">
<div class="rst-content">
<div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
<article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
<section id="frequently-asked-questions">
<h1>Frequently Asked Questions<a class="headerlink" href="#frequently-asked-questions" title="Permalink to this heading">¶</a></h1>
<p>At a high level, the TorchDynamo stack consists of a graph capture from
Python code using dynamo and a backend compiler. In this example the
backend compiler consists of backward graph tracing using AOTAutograd
and graph lowering using TorchInductor. There are of course many more
compilers available <a class="reference external" href="https://github.com/pytorch/torchdynamo/blob/0b8aaf340dad4777a080ef24bf09623f1aa6f3dd/README.md#existing-backend">here</a>
but for this document we will focus on inductor as a motivating example.</p>
<p>Torchdynamo supports training, using AotAutograd to capture backwards:</p>
<blockquote>
<div><ol class="arabic simple">
<li><p>the <code class="docutils literal notranslate"><span class="pre">.forward()</span></code> graph and <code class="docutils literal notranslate"><span class="pre">optimizer.step()</span></code> is captured by torchdynamo’s python evalframe frontend</p></li>
<li><p>for each segment of <code class="docutils literal notranslate"><span class="pre">.forward()</span></code> that torchdynamo captures, it uses AotAutograd to generate a backward graph segment</p></li>
<li><p>each pair of forward, backward graph are (optionally) min-cut partitioned to save the minimal state between forward/backward</p></li>
<li><p>the forward, backward pairs are wrapped in autograd.function modules 5. usercode calling<code class="docutils literal notranslate"><span class="pre">.backward()</span></code> still triggers eager’s autograd engine, which runs each ‘compiled backward’ graph as if it were one op, also running any non-compiled eager ops’ .backward() functions</p></li>
</ol>
</div></blockquote>
<section id="do-you-support-distributed-code">
<h2>Do you support Distributed code?<a class="headerlink" href="#do-you-support-distributed-code" title="Permalink to this heading">¶</a></h2>
<p>DDP has been tested and works, support for other distributed training
libraries is under discussion.</p>
<p>The main reason why Distributed code is challenging with dynamo is
because AOTAutograd unrolls both the forward and backward pass and
provides 2 graphs for backends to optimize. This is a problem for
distributed code because we’d like to ideally overlap communication
operations with computations. Eager pytorch accomplishes this in
different ways for DDP/FSDP- using autograd hooks, module hooks, and
modifications/mutations of module states. In a naive application of
dynamo, hooks that should run directly after an operation during
backwards may be delayed until after the entire compiled region of
backwards ops, due to how AOTAutograd compiled functions interact with
dispatcher hooks.</p>
<p>The basic strategy for optimizing DDP with Dynamo is outlined in
<a class="reference external" href="https://github.com/pytorch/pytorch/blob/master/torch/_dynamo/optimizations/distributed.py">distributed.py</a>
where the main idea will be to graph break on <a class="reference external" href="https://pytorch.org/docs/stable/notes/ddp.html#internal-design">DDP bucket
boundaries</a>.</p>
<p>When each node in DDP needs to synchronize its weights with the other
nodes it organizes its gradients and parameters into buckets which
reduces communication times and allows a node to broadcast a fraction of
its gradients to other waiting nodes.</p>
<p>Graph breaks in distributed code means you can expect dynamo and its
backends to optimize the compute overhead of a distributed program but
not its communication overhead. Graph-breaks may interfere with
compilation speedups, if the reduced graph-size robs the compiler of
fusion opportunities. However, there are diminishing returns with
increasing graph size since most of the current compute optimizations
are local fusions. So in practice this approach may be sufficient.</p>
</section>
<section id="do-i-still-need-to-export-whole-graphs">
<h2>Do I still need to export whole graphs?<a class="headerlink" href="#do-i-still-need-to-export-whole-graphs" title="Permalink to this heading">¶</a></h2>
<p>For the vast majority of models you probably don’t and you can use
<code class="docutils literal notranslate"><span class="pre">torch._dynamo()</span></code> optimize as is but there are a few situations where
full graphs are necessary and you can can ensure a full graph by simply
running <code class="docutils literal notranslate"><span class="pre">torch.dynamo(...,</span> <span class="pre">nopython=True)</span></code> * Large scale training
runs, think $250K+ that require pipeline parallelism and other advanced
sharding strategies * Inference optimizers like
<a class="reference external" href="https://github.com/pytorch/TensorRT">TensorRT</a> or
<a class="reference external" href="https://github.com/facebookincubator/AITemplate">AITemplate</a> that rely
on fusing much more aggressively than training optimizers * Mobile training or
inference.</p>
<p>Future work will include tracing communication operations into graphs,
coordinating these operations with compute optimizations, and optimizing
the communication operations.</p>
</section>
<section id="why-is-my-code-crashing">
<h2>Why is my code crashing?<a class="headerlink" href="#why-is-my-code-crashing" title="Permalink to this heading">¶</a></h2>
<p>If your code ran just fine without dynamo and started to crash with it
enabled then the most important first step is figuring out which part of
the stack your failure occurred in so try running things in the below
order and only try the next step if the previous step succeeded.</p>
<ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">torch.compile(...,</span> <span class="pre">backend="eager")</span></code> which only runs torchdynamo forward graph
capture and then runs the captured graph with PyTorch. If this fails
then there’s an issue with TorchDynamo.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">torch.compile(...,</span> <span class="pre">backend="aot_eager")</span></code>
which runs torchdynamo to capture a forward graph, and then AOTAutograd
to trace the backward graph without any additional backend compiler
steps. PyTorch eager will then be used to run the forward and backward
graphs. If this fails then there’s an issue with AOTAutograd.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">torch.compile(...,</span> <span class="pre">backend="inductor")</span></code> which runs torchdynamo to capture a
forward graph, and then AOTAutograd to trace the backward graph with the
TorchInductor compiler. If this fails then there’s an issue with TorchInductor</p></li>
</ol>
<section id="torchdynamo-errors">
<h3>TorchDynamo Errors<a class="headerlink" href="#torchdynamo-errors" title="Permalink to this heading">¶</a></h3>
<p>If the error that is generated occurs with the <code class="docutils literal notranslate"><span class="pre">"eager"</span></code> backend, then
torchdynamo is the most likely source of the error.</p>
<p>To debug these issues we recommend setting
<code class="docutils literal notranslate"><span class="pre">torch._dynamo.config.verbose=True</span></code> to get a full stack trace to both
the error in torchdynamo and the user code. In addition to this flag,
you can also set the <code class="docutils literal notranslate"><span class="pre">log_level</span></code> of torchdynamo through
<code class="docutils literal notranslate"><span class="pre">torch._dynamo.config.log_level</span></code>. The available levels are the
following: - <code class="docutils literal notranslate"><span class="pre">logging.DEBUG</span></code>: Print every instruction that is
encountered in addition to all below log levels - <code class="docutils literal notranslate"><span class="pre">logging.INFO</span></code>:
Print each function that is compiled (original and modified bytecode)
and the graph that is captured in addition to all below log levels -
<code class="docutils literal notranslate"><span class="pre">logging.WARNING</span></code> (default): Print graph breaks in addition to all
below log levels - <code class="docutils literal notranslate"><span class="pre">logging.ERROR</span></code>: Print errors only</p>
<p>If a model is sufficiently large, the logs can become overwhelming. If
an error occurs deep within a model’s python code, it can be useful to
execute only the frame in which the error occurs to enable easier
debugging. There are 2 tools available to enable this:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCHDYNAMO_DEBUG_FUNCTION=<desired_function_name></span></code> will only run TorchDynamo on functions with that name.</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">torch._dynamo.config.replay_record_enabled</span> <span class="pre">=</span> <span class="pre">True</span></code>) which dumps an execution record when an error is encountered. This record can then be replayed to run only the frame where an error occurred.</p></li>
</ul>
</section>
</section>
<section id="torchinductor-errors">
<h2>TorchInductor Errors<a class="headerlink" href="#torchinductor-errors" title="Permalink to this heading">¶</a></h2>
<p>With TorchInductor as the chosen backend, AOTAutograd is used to
generate the backward graph from the forward graph captured by
torchdynamo. It’s important to note that errors can occur during this
tracing and also while TorchInductor lowers the forward and backward
graphs to GPU code or C++.</p>
<p>A model can often consist of hundreds or thousands of FX nodes, so
narrowing the exact nodes where this problem occurred can be very
difficult which is why we highly recommend you use our minifier to
create tiny reproducible examples of failures you’re seeing. We can
minify errors that occur either at the AOTAutograd layer or Inductor
layer which you should try in the following order.</p>
<ol class="arabic simple">
<li><p><code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCHDYNAMO_REPRO_AFTER="aot"</span> <span class="pre">python</span> <span class="pre">your_model.py</span></code></p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCHDYNAMO_REPRO_AFTER="dynamo"</span> <span class="pre">python</span> <span class="pre">your_model.py</span></code></p></li>
</ol>
<p>Minifying your error is the quickest path to getting it fixed.</p>
<p>The minifier will actually create a <code class="docutils literal notranslate"><span class="pre">repro.py</span></code> for you at the location
set by <code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCHDYNAMO_REPRO_DIR</span></code> so make you have right access to
that directory. You can then run <code class="docutils literal notranslate"><span class="pre">python</span> <span class="pre">repro.py</span></code> and confirm that
you are getting the same error.</p>
<div class="admonition note">
<p class="admonition-title">Note</p>
<p>For other compilers such as nvfuser, the process is similar but
instead you would leverage <code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCHDYNAMO_REPRO_AFTER="dynamo"</span> <span class="pre">python</span> <span class="pre">your_model.py</span></code>.</p>
</div>
</section>
<section id="why-is-compilation-slow">
<h2>Why is compilation slow?<a class="headerlink" href="#why-is-compilation-slow" title="Permalink to this heading">¶</a></h2>
<section id="dynamo-compilation">
<h3>Dynamo Compilation<a class="headerlink" href="#dynamo-compilation" title="Permalink to this heading">¶</a></h3>
<p>TorchDynamo has a builtin stats function for collecting and displaying
the time spent in each compilation phase. These stats can be accessed by
calling <code class="docutils literal notranslate"><span class="pre">torch._dynamo.utils.compile_times()</span></code> after executing
<code class="docutils literal notranslate"><span class="pre">torch._dynamo</span></code>. By default, this returns a string representation of
the compile times spent in each TorchDynamo function by name.</p>
</section>
<section id="inductor-compilation">
<h3>Inductor Compilation<a class="headerlink" href="#inductor-compilation" title="Permalink to this heading">¶</a></h3>
<p>TorchInductor has a builtin stats and trace function for displaying time
spent in each compilation phase, output code, output graph visualization
and IR dump. <code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCH_COMPILE_DEBUG=1</span> <span class="pre">python</span> <span class="pre">repro.py</span></code>. This is a
debugging tool designed to make it easier to debug/understand the
internals of TorchInductor with an output that will look something like
<a class="reference external" href="https://gist.github.com/jansel/f4af078791ad681a0d4094adeb844396">this</a></p>
<p>Each file in that debug trace can be enabled/disabled via
<code class="docutils literal notranslate"><span class="pre">torch._inductor.config.trace.*</span></code>. The profile and the diagram are both
disabled by default since they are expensive to generate. See the
<a class="reference external" href="https://gist.github.com/jansel/f4af078791ad681a0d4094adeb844396">example debug directory
output</a>
for more examples.</p>
</section>
<section id="excessive-recompilation">
<h3>Excessive Recompilation<a class="headerlink" href="#excessive-recompilation" title="Permalink to this heading">¶</a></h3>
<p>When TorchDynamo compiles a function (or part of one), it makes certain
assumptions about locals and globals in order to allow compiler
optimizations, and expresses these assumptions as guards that check
particular values at runtime. If any of these guards fail, Dynamo will
recompile that function (or part) up to
<code class="docutils literal notranslate"><span class="pre">torch._dynamo.config.cache_size_limit</span></code> times. If your program is
hitting the cache limit, you will first need to determine which guard is
failing and what part of your program is triggering it.</p>
<p>The <a class="reference external" href="#recompilation-profiler">recompilation profiler</a> automates the
process of setting TorchDynamo’s cache limit to 1 and running your
program under an observation-only ‘compiler’ that records the causes of
any guard failures. You should be sure to run your program for at least
as long (as many iterations) as you were running when you ran into
trouble, and the profiler will accumulate statistics over this duration.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">from</span> <span class="nn">torch._dynamo.utils</span> <span class="kn">import</span> <span class="n">CompileProfiler</span>
<span class="n">prof</span> <span class="o">=</span> <span class="n">CompileProfiler</span><span class="p">()</span>
<span class="k">def</span> <span class="nf">my_model</span><span class="p">():</span>
<span class="o">...</span>
<span class="n">profiler_model</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">my_model</span><span class="p">,</span> <span class="n">backend</span><span class="o">=</span><span class="n">prof</span><span class="p">)</span>
<span class="n">profiler_model</span><span class="p">()</span>
<span class="nb">print</span><span class="p">(</span><span class="n">prof</span><span class="o">.</span><span class="n">report</span><span class="p">())</span>
</pre></div>
</div>
<p>Many of the reasons for graph breaks and excessive recompilation will be
fixed with upcoming support for <a class="reference external" href="https://docs.google.com/document/d/1QJB-GOnbv-9PygGlOMXwiO9K6vVNm8sNg_olixJ9koc/edit?usp=sharing">tracing dynamic tensor
shapes</a>,
more careful choices for guards and better tuned heuristics.</p>
</section>
<section id="why-are-you-recompiling-in-production">
<h3>Why are you recompiling in production?<a class="headerlink" href="#why-are-you-recompiling-in-production" title="Permalink to this heading">¶</a></h3>
<p>In some cases, you may not want unexpected compiles after a program has
warmed up. For example, if you are serving production traffic in a
latency critical application. For this, TorchDynamo provides an
alternate mode where prior compiled graphs are used, but no new ones are
generated:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="n">frozen_toy_example</span> <span class="o">=</span> <span class="n">dynamo</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">toy_example</span><span class="p">)</span>
<span class="n">frozen_toy_example</span><span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
</pre></div>
</div>
</section>
</section>
<section id="how-are-you-speeding-up-my-code">
<h2>How are you speeding up my code?<a class="headerlink" href="#how-are-you-speeding-up-my-code" title="Permalink to this heading">¶</a></h2>
<p>There are 3 major ways to accelerate PyTorch code:</p>
<ol class="arabic simple">
<li><p>Kernel fusion via vertical fusions which fuse sequential operations to avoid
excessive read/writes. For example, fuse 2 subsequent cosines means you
can can do 1 read 1 write instead 2 reads 2 writes 2. Horizontal fusion:
the simplest example being batching where a single matrix is multiplied
with a batch of examples but the more general scenario is a grouped GEMM
where a group of matrix multiplications are scheduled together</p></li>
<li><p>Out of order execution: A general optimization for compilers, by looking ahead
at the exact data dependencies within a graph we can decide on the most
opportune time to execute a node and which buffers can be reused</p></li>
<li><p>Automatic work placement: Similar of the out of order execution point,
but by matching nodes of a graph to resources like physical hardware or
memory we can design an appropriate schedule</p></li>
</ol>
<p>The above are general principles for accelerating PyTorch code but
different backends will each make different tradeoffs on what to
optimize. For example Inductor first takes care of fusing whatever it
can and only then generates <a class="reference external" href="https://openai.com/blog/triton/">Triton</a>
kernels. It can also</p>
<p>Triton in addition offers speedups because of automatic memory
coalescing, memory management and scheduling within each Streaming
Multiprocessor and has been designed to handle tiled computations.</p>
<p>However, regardless of the backend you use it’s best to use a benchmark
and see approach so try out the PyTorch profiler, visually inspect the
generated kernels and try to see what’s going on for yourself.</p>
</section>
<section id="why-am-i-not-seeing-speedups">
<h2>Why am I not seeing speedups?<a class="headerlink" href="#why-am-i-not-seeing-speedups" title="Permalink to this heading">¶</a></h2>
<section id="graph-breaks">
<h3>Graph Breaks<a class="headerlink" href="#graph-breaks" title="Permalink to this heading">¶</a></h3>
<p>The main reason you won’t see the speedups you’d like to by using dynamo
is excessive graph breaks. So what’s a graph break?</p>
<p>Given a program like:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">some_fun</span><span class="p">(</span><span class="n">x</span><span class="p">):</span>
<span class="o">...</span>
<span class="n">torch</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">some_fun</span><span class="p">)(</span><span class="n">x</span><span class="p">)</span>
<span class="o">...</span>
</pre></div>
</div>
<p>Torchdynamo will attempt to compile all of the torch/tensor operations
within <code class="docutils literal notranslate"><span class="pre">some_fun()</span></code> into a single FX graph, but it may fail to capture
everything into one graph.</p>
<p>Some graph break reasons are insurmountable to TorchDynamo like calling
into a C extension other than torch is invisible to torchdynamo, and
could do arbitrary things without TorchDynamo being able to introduce
necessary guards to ensure that the compiled program would be safe to reuse.</p>
<blockquote>
<div><p>To maximize performance, it’s important to have as few graph breaks
as possible.</p>
</div></blockquote>
</section>
<section id="identifying-the-cause-of-a-graph-break">
<h3>Identifying the cause of a graph break<a class="headerlink" href="#identifying-the-cause-of-a-graph-break" title="Permalink to this heading">¶</a></h3>
<p>To identify all graph breaks in a program and the associated reasons for
the breaks, <code class="docutils literal notranslate"><span class="pre">torch._dynamo.explain</span></code> can be used. This tool runs
TorchDynamo on the supplied function and aggregates the graph breaks
that are encountered. Here is an example usage:</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">import</span> <span class="nn">torch._dynamo</span> <span class="k">as</span> <span class="nn">dynamo</span>
<span class="k">def</span> <span class="nf">toy_example</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
<span class="n">x</span> <span class="o">=</span> <span class="n">a</span> <span class="o">/</span> <span class="p">(</span><span class="n">torch</span><span class="o">.</span><span class="n">abs</span><span class="p">(</span><span class="n">a</span><span class="p">)</span> <span class="o">+</span> <span class="mi">1</span><span class="p">)</span>
<span class="nb">print</span><span class="p">(</span><span class="s2">"woo"</span><span class="p">)</span>
<span class="k">if</span> <span class="n">b</span><span class="o">.</span><span class="n">sum</span><span class="p">()</span> <span class="o"><</span> <span class="mi">0</span><span class="p">:</span>
<span class="n">b</span> <span class="o">=</span> <span class="n">b</span> <span class="o">*</span> <span class="o">-</span><span class="mi">1</span>
<span class="k">return</span> <span class="n">x</span> <span class="o">*</span> <span class="n">b</span>
<span class="n">explanation</span><span class="p">,</span> <span class="n">out_guards</span><span class="p">,</span> <span class="n">graphs</span><span class="p">,</span> <span class="n">ops_per_graph</span> <span class="o">=</span> <span class="n">dynamo</span><span class="o">.</span><span class="n">explain</span><span class="p">(</span><span class="n">toy_example</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">10</span><span class="p">),</span> <span class="n">torch</span><span class="o">.</span><span class="n">randn</span><span class="p">(</span><span class="mi">10</span><span class="p">))</span>
<span class="nb">print</span><span class="p">(</span><span class="n">explanation</span><span class="p">)</span>
<span class="sd">"""</span>
<span class="sd">Dynamo produced 3 graphs, with 2 graph break and 6 ops.</span>
<span class="sd"> Break reasons:</span>
<span class="sd">1. call_function BuiltinVariable(print) [ConstantVariable(str)] {}</span>
<span class="sd"> File "t2.py", line 16, in toy_example</span>
<span class="sd"> print("woo")</span>
<span class="sd">2. generic_jump</span>
<span class="sd"> File "t2.py", line 17, in toy_example</span>
<span class="sd"> if b.sum() < 0:</span>
<span class="sd"> """</span>
</pre></div>
</div>
<p>To throw an error on the first graph break encountered you can use
disable python fallback by using <code class="docutils literal notranslate"><span class="pre">nopython=True</span></code>, this should be
familiar if you’ve worked with export based compilers.</p>
<div class="highlight-python notranslate"><div class="highlight"><pre><span></span><span class="k">def</span> <span class="nf">toy_example</span><span class="p">(</span><span class="n">a</span><span class="p">,</span> <span class="n">b</span><span class="p">):</span>
<span class="o">...</span>
<span class="n">torch</span><span class="o">.</span><span class="n">compile</span><span class="p">(</span><span class="n">toy_example</span><span class="p">,</span> <span class="n">fullgraph</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">backend</span><span class="o">=<</span><span class="n">compiler</span><span class="o">></span><span class="p">)</span>
</pre></div>
</div>
</section>
</section>
<section id="why-didnt-my-code-recompile-when-i-changed-it">
<h2>Why didn’t my code recompile when I changed it?<a class="headerlink" href="#why-didnt-my-code-recompile-when-i-changed-it" title="Permalink to this heading">¶</a></h2>
<p>If you went ahead and enabled dynamic shapes via
<code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCHDYNAMO_DYNAMIC_SHAPES=1</span> <span class="pre">python</span> <span class="pre">model.py</span></code> then your code
won’t recompile on shape changes. We’ve added support for dynamic shapes
which avoids recompilations in the case when shapes vary by less than a
factor of 2. This is especially useful in scenarios like varying image
sizes in CV or variable sequence length in NLP. In inference scenarios
it’s often not possible to know what a batch size will be beforehand
because you take what you can get from different client apps.</p>
<p>In general, TorchDynamo tries very hard not to recompile things
unnecessarily so if for example torchdynamo finds 3 graphs and your
change only modified one graph then only that graph will recompile. So
another tip to avoid potentially slow compilation times is to warmup a
model by compiling it once after which subsequent compilations will be
much faster. Cold start compile times is still a metric we track
visibly.</p>
</section>
<section id="why-am-i-getting-incorrect-results">
<h2>Why am I getting incorrect results?<a class="headerlink" href="#why-am-i-getting-incorrect-results" title="Permalink to this heading">¶</a></h2>
<p>Accuracy issues can also be minified if you set the environment variable
<code class="docutils literal notranslate"><span class="pre">TORCHDYNAMO_REPRO_LEVEL=4</span></code>, it operates with a similar git bisect
model and a full repro might be something like
<code class="docutils literal notranslate"><span class="pre">TORCHDYNAMO_REPRO_AFTER="aot"</span> <span class="pre">TORCHDYNAMO_REPRO_LEVEL=4</span></code> the reason
we need this is downstream compilers will codegen code whether it’s
Triton code or the C++ backend, the numerics from those downstream
compilers can be different in subtle ways yet have dramatic impact on
your training stability. So the accuracy debugger is very useful for us
to detect bugs in our codegen or with a backend compiler.</p>
</section>
<section id="why-am-i-getting-ooms">
<h2>Why am I getting OOMs?<a class="headerlink" href="#why-am-i-getting-ooms" title="Permalink to this heading">¶</a></h2>
<p>Dynamo is still an alpha product so there’s a few sources of OOMs and if
you’re seeing an OOM try disabling the following configurations in this
order and then open an issue on Github so we can solve the root problem
1. If you’re using dynamic shapes try disabling them, we’ve disabled
them by default: <code class="docutils literal notranslate"><span class="pre">env</span> <span class="pre">TORCHDYNAMO_DYNAMIC_SHAPES=0</span> <span class="pre">python</span> <span class="pre">model.py</span></code> 2.
CUDA graphs with Triton are enabled by default in inductor but removing
them may alleviate some OOM issues: <code class="docutils literal notranslate"><span class="pre">torch._inductor.config.triton.cudagraphs</span> <span class="pre">=</span> <span class="pre">False</span></code>.</p>
</section>
</section>
</article>
</div>
<footer>
<div class="rst-footer-buttons" role="navigation" aria-label="footer navigation">
<a href="../ir.html" class="btn btn-neutral float-right" title="IRs" accesskey="n" rel="next">Next <img src="../_static/images/chevron-right-orange.svg" class="next-page"></a>
<a href="troubleshooting.html" class="btn btn-neutral" title="TorchDynamo Troubleshooting" accesskey="p" rel="prev"><img src="../_static/images/chevron-right-orange.svg" class="previous-page"> Previous</a>
</div>
<hr>
<div role="contentinfo">
<p>
© Copyright 2023, PyTorch Contributors.
</p>
</div>
<div>
Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
</div>
</footer>
</div>
<script>
var match = window.location.href.match(/\/_[a-zA-Z0-9_]*.html|_dynamo/gi);
var url = window.location.href.lastIndexOf(match[match.length-1]);
if (url)
{
var div = '<div class="admonition note"><p class="admonition-title">Note</p><p><i class="fa fa-exclamation-circle" aria-hidden="true"> </i> This page describes an internal API which is not intended to be used outside of the PyTorch codebase and can be modified or removed without notice.</p></div>'
document.getElementById("pytorch-article").insertAdjacentHTML('afterBegin', div)
}
</script>
</div>
<div class="pytorch-content-right" id="pytorch-content-right">
<div class="pytorch-right-menu" id="pytorch-right-menu">
<div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
<ul>
<li><a class="reference internal" href="#">Frequently Asked Questions</a><ul>
<li><a class="reference internal" href="#do-you-support-distributed-code">Do you support Distributed code?</a></li>
<li><a class="reference internal" href="#do-i-still-need-to-export-whole-graphs">Do I still need to export whole graphs?</a></li>
<li><a class="reference internal" href="#why-is-my-code-crashing">Why is my code crashing?</a><ul>
<li><a class="reference internal" href="#torchdynamo-errors">TorchDynamo Errors</a></li>
</ul>
</li>
<li><a class="reference internal" href="#torchinductor-errors">TorchInductor Errors</a></li>
<li><a class="reference internal" href="#why-is-compilation-slow">Why is compilation slow?</a><ul>
<li><a class="reference internal" href="#dynamo-compilation">Dynamo Compilation</a></li>
<li><a class="reference internal" href="#inductor-compilation">Inductor Compilation</a></li>
<li><a class="reference internal" href="#excessive-recompilation">Excessive Recompilation</a></li>
<li><a class="reference internal" href="#why-are-you-recompiling-in-production">Why are you recompiling in production?</a></li>
</ul>
</li>
<li><a class="reference internal" href="#how-are-you-speeding-up-my-code">How are you speeding up my code?</a></li>
<li><a class="reference internal" href="#why-am-i-not-seeing-speedups">Why am I not seeing speedups?</a><ul>
<li><a class="reference internal" href="#graph-breaks">Graph Breaks</a></li>
<li><a class="reference internal" href="#identifying-the-cause-of-a-graph-break">Identifying the cause of a graph break</a></li>
</ul>
</li>
<li><a class="reference internal" href="#why-didnt-my-code-recompile-when-i-changed-it">Why didn’t my code recompile when I changed it?</a></li>
<li><a class="reference internal" href="#why-am-i-getting-incorrect-results">Why am I getting incorrect results?</a></li>
<li><a class="reference internal" href="#why-am-i-getting-ooms">Why am I getting OOMs?</a></li>
</ul>
</li>
</ul>
</div>
</div>
</div>
</section>
</div>
<script type="text/javascript" id="documentation_options" data-url_root="../" src="../_static/documentation_options.js"></script>
<script data-url_root="../" id="documentation_options" src="../_static/documentation_options.js"></script>
<script src="../_static/jquery.js"></script>
<script src="../_static/underscore.js"></script>
<script src="../_static/_sphinx_javascript_frameworks_compat.js"></script>
<script src="../_static/doctools.js"></script>
<script src="../_static/clipboard.min.js"></script>
<script src="../_static/copybutton.js"></script>
<script type="text/javascript" src="../_static/js/vendor/popper.min.js"></script>
<script type="text/javascript" src="../_static/js/vendor/bootstrap.min.js"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
<script type="text/javascript" src="../_static/js/theme.js"></script>
<script type="text/javascript">
jQuery(function () {
SphinxRtdTheme.Navigation.enable(true);
});
</script>
<script script type="text/javascript">
var collapsedSections = ['Developer Notes', 'Language Bindings', 'Libraries', 'Community'];
</script>
<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&guid=ON&script=0"/>
<!-- Begin Footer -->
<div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
<div class="container">
<div class="row">
<div class="col-md-4 text-center">
<h2>Docs</h2>
<p>Access comprehensive developer documentation for PyTorch</p>
<a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
</div>
<div class="col-md-4 text-center">
<h2>Tutorials</h2>
<p>Get in-depth tutorials for beginners and advanced developers</p>
<a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
</div>
<div class="col-md-4 text-center">
<h2>Resources</h2>
<p>Find development resources and get your questions answered</p>
<a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
</div>
</div>
</div>
</div>
<footer class="site-footer">
<div class="container footer-container">
<div class="footer-logo-wrapper">
<a href="https://pytorch.org/" class="footer-logo"></a>
</div>
<div class="footer-links-wrapper">
<div class="footer-links-col">
<ul>
<li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
<li><a href="https://pytorch.org/get-started">Get Started</a></li>
<li><a href="https://pytorch.org/features">Features</a></li>
<li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
<li><a href="https://pytorch.org/blog/">Blog</a></li>
<li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
</ul>
</div>
<div class="footer-links-col">
<ul>
<li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
<li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
<li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
<li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
<li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
<li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
</ul>
</div>
<div class="footer-links-col">
<ul>
<li class="list-title">Stay up to date</li>
<li><a href="https://www.facebook.com/pytorch" target="_blank">Facebook</a></li>
<li><a href="https://twitter.com/pytorch" target="_blank">Twitter</a></li>
<li><a href="https://www.youtube.com/pytorch" target="_blank">YouTube</a></li>
<li><a href="https://www.linkedin.com/company/pytorch" target="_blank">LinkedIn</a></li>
</ul>
</div>
<div class="footer-links-col">
<ul>
<li class="list-title">PyTorch Podcasts</li>
<li><a href="https://open.spotify.com/show/6UzHKeiy368jKfQMKKvJY5" target="_blank">Spotify</a></li>
<li><a href="https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008" target="_blank">Apple</a></li>
<li><a href="https://www.google.com/podcasts?feed=aHR0cHM6Ly9mZWVkcy5zaW1wbGVjYXN0LmNvbS9PQjVGa0lsOA%3D%3D" target="_blank">Google</a></li>
<li><a href="https://music.amazon.com/podcasts/7a4e6f0e-26c2-49e9-a478-41bd244197d0/PyTorch-Developer-Podcast?" target="_blank">Amazon</a></li>
</ul>