,QWURGXFWLRQ

0HVVDJH3DVVLQJ3URJUDPPLQJ
03,8VHUV*XLGHLQ
)2575$1



   
  


03,8VHU*XLGHLQ)2575$1
'U 3HWHU 6 3DFKHFR
'HSDUWPHQW RI 0DWKHPDWLFV
8QLYHUVLW\ RI 6DQ )UDQFLVFR
6DQ )UDQFLVFR &$ 
0DUFK  
:RR &KDW 0LQJ
&RPSXWHU &HQWUH
8QLYHUVLW\ RI +RQJ .RQJ
+RQJ .RQJ
0DUFK  
1. Introduction

2. Greetings !

2.1 General MPI Programs

2.2 Finding out About the Rest of the World

2.3 Message : Data + Envelope

2.4 MPI_Send and MPI_Recv

3. An Application

3.1 Serial program

3.2 Parallelizing the Trapezoid Rule

3.3 I/O on Parallel Processors

4. Collective Communication

4.1 Tree-Structured Communication

4.2 Broadcast

4.3 Reduce

4.4 Other Collective Communication Functions

5. Grouping Data for Communication

5.1 The Count Parameter

5.2 Derived Types and MPI_Type_struct

5.3 Other Derived Datatype Constructors

5.4 Pack/Unpack

5.5 Deciding Which Method to Use

6. Communicators and Topologies

6.1 Fox's Algorithm

6.2 Communicators

6.3 Working with Groups, Contexts, and Communicators

6.4 MPI_Comm_split

6.5 Topologies

6.6 MPI_Cart_sub

6.7 Implementation of Fox's Algorithm

7. Where To Go From Here

7.1 What We Haven't Discussed

7.2 Implementations of MPI

7.3 More Information on MPI

7.4 The Future of MPI

8. Compiling and Running MPI Programs

9. Reference
  ,QWURGXFWLRQ
KH0HVVDJH3DVVLQJ,QWHUIDFHRU03,LVDOLEUDU\RIIXQFWLRQVDQGPDFURVWKDWFDQEHXVHGLQ
& )2575$1 DQG & SURJUDPV $V LWV QDPH LPSOLHV 03, LV LQWHQGHG IRU XVH LQ
SURJUDPVWKDWH[SORLWWKHH[LVWHQFHRIPXOWLSOHSURFHVVRUVE\PHVVDJHSDVVLQJ
03,ZDVGHYHORSHGLQE\DJURXSRIUHVHDUFKHUVIURPLQGXVWU\JRYHUQPHQWDQG
DFDGHPLD$VVXFKLWLVRQHRIWKHILUVWVWDQGDUGVIRUSURJUDPPLQJSDUDOOHOSURFHVVRUVDQGLW
LVWKHILUVWWKDWLVEDVHGRQPHVVDJHSDVVLQJ
,Q  $ 8VHUV *XLGH WR 03, KDV EHHQ ZULWWHQ E\ 'U 3HWHU 6 3DFKHFR 7KLV LV D EULHI
WXWRULDOLQWURGXFWLRQWRVRPHRIWKHPRUHLPSRUWDQWIHDWXUHRIWKH03,IRU&SURJUDPPHUV
,WLVDQLFHO\ZULWWHQGRFXPHQWDWLRQDQGXVHUVLQRXUXQLYHUVLW\ILQGLWYHU\FRQFLVHDQGHDV\
WRUHDG
+RZHYHUPDQ\XVHUVRISDUDOOHOFRPSXWHUDUHLQWKHVFLHQWLILFDQGHQJLQHHUVFRPPXQLW\DQG
PRVWRIWKHPXVH)2575$1DVWKHLUSULPDU\FRPSXWHUODQJXDJH0RVWRIWKHPGRQWXVH
&ODQJXDJHSURILFLHQWO\7KLVVLWXDWLRQRFFXUVYHU\IUHTXHQWO\LQ+RQJ.RQJ$DUHVXOWWKH
$ 8VHUV *XLGH WR 03, LV WUDQVODWHG WR WKLV JXLGH LQ )RUWUDQ WR DGGUHVV IRU WKH QHHG RI
VFLHQWLILFSURJUDPPHUV
$FNQRZOHGJPHQWV,JUDWHIXOO\DFNQRZOHGJH'U3HWHU63DFKHFRIRUWKHXVHRI&YHUVLRQRIWKH
XVHU JXLGH RQ ZKLFK WKLV JXLGH LV EDVHG , ZRXOG DOVR JUDWHIXOO\ WKDQNV WR WKH &RPSXWHU
&HQWUHRIWKH8QLYHUVLW\RI+RQJ.RQJIRUWKHLUKXPDQUHVRXUFHVXSSRUWRIWKLVZRUN$QG
, DOVR WKDQNV WR DOO WKH UHVHDUFK LQVWLWXWLRQ ZKLFK VXSSRUWHG WKH RULJLQDO ZRUN E\ 'U
3DFKHFR

 *UHHWLQJV
7KH ILUVW SURJUDP WKDW PRVW RI XV VDZ ZDV WKH +HOOR ZRUOG SURJUDP LQ PRVW RI LQWURGXFWRU\
SURJUDPPLQJERRNV,WVLPSO\SULQWVWKHPHVVDJH+HOORZRUOG$YDULDQWWKDWPDNHVVRPHXVHRI
PXOWLSOHSURFHVVHVLVWRKDYHHDFKSURFHVVVHQGDJUHHWLQJWRDQRWKHUSURFHVV
,Q03,WKHSURFHVVLQYROYHGLQWKHH[HFXWLRQRIDSDUDOOHOSURJUDPDUHLGHQWLILHGE\DVHTXHQFHRI
QRQQHJDWLYHLQWHJHUV,IWKHUHDUHSSURFHVVHVH[HFXWLQJDSURJUDPWKH\ZLOOKDYHUDQNVS
7KHIROORZLQJSURJUDPKDVHDFKSURFHVVRWKHUWKDQVHQGDPHVVDJHWRSURFHVVDQGSURFHVV
SULQWVRXWWKHPHVVDJHVLWUHFHLYHG
program greetings
include 'mpif.h'
integer my_rank
integer p
integer source
integer dest
integer tag
character*100 message
character*10 digit_string
integer size
integer status(MPI_STATUS_SIZE)
integer ierr
call MPI_Init(ierr)
call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
call MPI_Comm_size(MPI_COMM_WORLD, p, ierr)
if (my_rank .NE. 0) then
write(digit_string,FMT="(I3)") my_rank
message = 'Greetings from process '
+ // trim(digit_string) // ' !'
dest = 0
tag = 0
call MPI_Send(message, len_trim(message),
+ MPI_CHARACTER, dest, tag, MPI_COMM_WORLD, ierr)
else
do source = 1, p-1
tag = 0
call MPI_Recv(message, 100, MPI_CHARACTER,
+ source, tag, MPI_COMM_WORLD, status, ierr)
write(6,FMT="(A)") message
enddo
endif
call MPI_Finalize(ierr)
end program greetings
7KHGHWDLOVRIFRPSLOLQJDQGH[HFXWLQJWKLVSURJUDPLVLQFKDSWHU
:KHQWKHSURJUDPLVFRPSLOHGDQGUXQZLWKWZRSURFHVVHVWKHRXWSXWVKRXOGEH
Greetings from process 1!
,ILWVUXQZLWKIRXUSURFHVVHVWKHRXWSXWVKRXOGEH
Greetings from process 1!
Greetings from process 2!
Greetings from process 3!
$OWKRXJKWKHGHWDLOVRIZKDWKDSSHQVZKHQWKHSURJUDPLVH[HFXWHGYDU\IURPPDFKLQHWRPDFKLQH
WKHHVVHQWLDOVDUHWKHVDPHRQDOOPDFKLQHV3URYLGHGZHUXQRQHSURFHVVRQHDFKSURFHVVRU
 7KHXVHULVVXHVDGLUHFWLYHWRWKHRSHUDWLQJV\VWHPZKLFKKDVWKHHIIHFWRISODFLQJD
FRS\RIWKHH[HFXWDEOHSURJUDPRQHDFKSURFHVVRU
 (DFKSURFHVVRUEHJLQVH[HFXWLRQRILWVFRS\RIWKHH[HFXWDEOH
 'LIIHUHQW SURFHVVHV FDQ H[HFXWH GLIIHUHQW VWDWHPHQWV E\ EUDQFKLQJ ZLWKLQ WKH
SURJUDP7\SLFDOO\WKHEUDQFKLQJZLOOEHEDVHGRQSURFHVVUDQNV

6R WKH Greetings SURJUDP XVHV WKH 6LQJOH 3URJUDP 0XOWLSOH 'DWD RU 630' SDUDGLJP 7KDW LV ZH
REWDLQWKH HIIHFWRIGLIIHUHQWSURJUDPVUXQQLQJRQGLIIHUHQWSURFHVVRUVE\WDNLQJEUDQFKHVZLWKLQD
VLQJOH SURJUDP RQ WKH EDVLV RI SURFHVV UDQN  WKH VWDWHPHQWV H[HFXWHG E\ SURFHVV  DUH GLIIHUHQW
IURP WKRVH H[HFXWHG E\ WKH RWKHU SURFHVVHV HYHQ WKRXJK DOO SURFHVVHV DUH UXQQLQJ WKH VDPH
SURJUDP 7KLV LV WKH PRVW FRPPRQO\ XVHG PHWKRG IRU ZULWLQJ 0,0' SURJUDPV DQG ZHOO XVH LW
H[FOXVLYHO\LQWKLV*XLGH

*HQHUDO03,3URJUDPV
(YHU\03,SURJUDPPXVWFRQWDLQWKHSUHSUHFHVVRUGLUHFWLYH
include mpif.h
7KLVILOHPSLIKFRQWDLQVWKHGHILQLWLRQVPDFURVDQGIXQFWLRQSURWRW\SHVQHFHVVDU\
IRUFRPSLOLQJDQ03,SURJUDP

%HIRUHDQ\RWKHU03,IXQFWLRQVFDQEHFDOOHGWKHIXQFWLRQ MPI_InitPXVWEHFDOOHG
DQGLWVKRXOGRQO\EHFDOOHGRQFH)RUWUDQ03,URXWLQHVKDYHDQ IERROR DUJXPHQW
WKLVFRQWDLQVWKHHUURUFRGH$IWHUDSURJUDPKDVILQLVKHGXVLQJ03,OLEUDU\LWPXVW
FDOO MPI_Finialize 7KLV FOHDQV XS DQ\ XQILQLVKHG EXVLQHVV OHIW E\ 03,  HJ
SHQGLQJ UHFHLYHV WKDW ZHUH QHYHU FRPSOHWHG 6R D W\SLFDO 03, SURJUDP KDV WKH
IROORZLQJOD\RXW
.
.
.
include 'mpif.h'
.
.
.
call MPI_Init(ierr)
.
.
 .
call MPI_Finialize(ierr)
.
.
.
end program

)LQGLQJRXW$ERXWWKH5HVWRIWKH:RUOG
03,SURYLGHVWKHIXQFWLRQ MPI_Comm_rankZKLFKUHWXUQVWKHUDQNRIDSURFHVV
LQLWVVHFRQGLQLWVVHFRQGDUJXPHQW,WVV\QWD[LV
call MPI_Comm_rank(COMM, RANK, IERROR)
integer COMM, RANK, IERROR
7KH ILUVW DUJXPHQW LV D FRPPXQLFDWRU (VVHQWLDOO\ D FRPPXQLFDWRU LV D FROOHFWLRQ RI
SURFHVVHV WKDW FDQ VHQG PHVVDJH WR HDFK RWKHU )RU EDVLF SURJUDPV WKH RQO\
FRPPXQLFDWRU QHHGHG LV MPI_COMM_WORLD ,W LV SUHGHILQHG LQ 03, DQG
FRQVLVWVRIDOOWKHSURFHVVHVUXQQLQJZKHQSURJUDPH[HFXWLRQEHJLQV
0DQ\ RI WKH FRQVWUXFWV LQ RXU SURJUDPV DOVR GHSHQG RQ WKH QXPEHU RI SURFHVVHV
H[HFXWLQJ WKH SURJUDP 6R 03, SURYLGHV WKH IXQFWLRQV MPI_Comm_size IRU
GHWHUPLQLQJ WKLV ,WV ILUVW DUJXPHQW LV D FRPPXQLFDWRU ,W UHWXUQV WKH QXPEHU RI
SURFHVVHVLQDFRPPXQLFDWRULQLWVVHFRQGDUJXPHQW,WVV\QWD[LV
call MPI_Comm_size(COMM, P, IERROR)
integer COMM, P, IERROR

0HVVDJH'DWD(QYHORSH
7KH DFWXDO PHVVDJHSDVVLQJ LQ RXU SURJUDP LV FDUULHG RXW E\ WKH 03, IXQFWLRQV MPI_Send DQG
MPI_Recv 7KH ILUVW FRPPDQG VHQGV D PHVVDJH WR D GHVLJQDWHG SURFHVV 7KH VHFRQG UHFHLYHV D
PHVVDJHIURPDSURFHVV7KHVHDUHWKHPRVWEDVLFPHVVDJHSDVVLQJFRPPDQGVLQ03,,QRUGHUIRU
WKHPHVVDJHWREHVXFFHVVIXOO\FRPPXQLFDWHGWKHV\VWHPPXVWDSSHQGVRPHLQIRUPDWLRQWRWKHGDWD
WKDWWKHDSSOLFDWLRQSURJUDPZLVKHVWRWUDQVPLW7KLVDGGLWLRQDOLQIRUPDWLRQIRUPVWKH HQYHORSHRIWKH
PHVVDJH,Q03,LWFRQWDLQVWKHIROORZLQJLQIRUPDWLRQ
 7KHUDQNRIWKHUHFHLYHU
 7KHUDQNRIWKHVHQGHU
 $WDJ
 $FRPPXQLFDWRU
7KHVH LWHPV FDQ EH XVHG E\ WKH UHFHLYHU WR GLVWLQJXLVK DPRQJ LQFRPLQJ PHVVDJHV 7KH VRXUFH
DUJXPHQWFDQEHXVHGWRGLVWLQJXLVKPHVVDJHVUHFHLYHGIURPGLIIHUHQWSURFHVVHV7KH WDJLVDXVHU
VSHFLILHG integer WKDW FDQ EH XVHG WR GLVWLQJXLVK PHVVDJHV UHFHLYHG IRUP D VLQJOH SURFHVV )RU
H[DPSOHVXSSRVHSURFHVV $LVVHQGLQJWZRPHVVDJHVWRSURFHVV %ERWKPHVVDJHVFRQWDLQVDVLQJOH
UHDOQXPEHU2QHRIWKHUHDOQXPEHULVWREHXVHGLQDFDOFXODWLRQZKLOHWKHRWKHULVWREHSULQWHG
,QRUGHUWRGHWHUPLQHZKLFKLVZKLFK $FDQXVHGLIIHUHQWWDJVIRUWKHWZRPHVVDJHV,I %XVHVWKH
VDPHWZRWDJVLQWKHFRUUHVSRQGLQJUHFHLYHVZKHQLWUHFHLYHVWKHPHVVDJHVLWZLOONQRZZKDWWR
GRZLWKWKHP03,JXDUDQWHHVWKDWWKHLQWHJHUVFDQEHXVHGDVWDJV0RVWLPSOHPHQWDWLRQV
DOORZPXFKODUJHUYDOXHV
$VZHQRWHGDERYHDFRPPXQLFDWRULVEDVLFDOO\DFROOHFWLRQRISURFHVVHVWKDWFDQVHQGPHVVDJHVWR
HDFK RWKHU :KHQ WZR SURFHVVHV DUH FRPPXQLFDWLQJ XVLQJ MPI_Send DQG MPI_Recv LWV
LPSRUWDQFH DULVHV ZKHQ VHSDUDWH PRGXOHV RI D SURJUDP KDYH EHHQ ZULWWHQ LQGHSHQGHQWO\ RI HDFK
RWKHU)RUH[DPSOHVXSSRVHZHZLVKWRVROYHDV\VWHPRIGLIIHUHQWLDOHTXDWLRQVDQGLQWKHFRXUVH
RIVROYLQJWKHV\VWHPZHQHHGWRVROYHDV\VWHPRIOLQHDUHTXDWLRQ5DWKHUWKDQZULWLQJWKHOLQHDU
V\VWHPVROYHUIURPVFUDWFKZHPLJKWZDQWWRXVHD OLEUDU\RIIXQFWLRQVIRUVROYLQJOLQHDUV\VWHPVWKDW
ZDVZULWWHQE\VRPHRQHHOVHDQGWKDWKDVEHHQKLJKO\RSWLPL]HGIRUWKHV\VWHPZHUHXVLQJ+RZGR
ZHDYRLGFRQIXVLQJWKHPHVVDJHV ZHVHQGIURPSURFHVV $WRSURFHVV%ZLWKWKRVHVHQWE\WKHOLEUDU\
IXQFWLRQV " %HIRUH WKH DGYHQW RI FRPPXQLFDWRUV ZH ZRXOG SUREDEO\ KDYH WR SDUWLWLRQ WKH VHW RI
YDOLGWDJVVHWWLQJDVLGHVRPHRIWKHPIRUH[FOXVLYHXVHE\WKHOLEUDU\IXQFWLRQV7KLVLVWHGLRXVDQGLW
ZLOOFDXVHSUREOHPVLIZHWU\WRUXQRXUSURJUDPRQDQRWKHUV\VWHPWKHRWKHUV\VWHPVOLQHDUVROYHU
PD\ QRW SUREDEO\ ZRQW UHTXLUH WKH VDPH VHW RI WDJV :LWK WKH DGYHQW RI FRPPXQLFDWRUV ZH
VLPSO\ FUHDWH D FRPPXQLFDWRU WKDW FDQ EH XVHG H[FOXVLYHO\ E\ WKH OLQHDU VROYHU DQG SDVV LW DV DQ
DUJXPHQWLQFDOOVWRWKHVROYHU:HOOGLVFXVVWKHGHWDLOVRIWKLVODWHU)RUQRZZHFDQJHWDZD\ZLWK
XVLQJ WKH SUHGHILQHG FRPPXQLFDWRU MPI_COMM_WORLD ,W FRQVLVWV RI DOO WKH SURFHVVHV
UXQQLQJWKHSURJUDPZKHQH[HFXWLRQEHJLQV

03,B6HQGDQG03,B5HFY
7RVXPPDUL]HOHWVGHWDLOWKHV\QWD[RI MPI_SendDQGMPI_Recv
MPI_Send( Message, count, datatype, dest, tag, comm, ierror)
<type> message(*)
INTEGER count, datatype, dest, tag, comm, ierror
MPI_Recv( message, count, datatype, source, tag,
comm, status, ierror)
<type> message(*)
INTEGER count, datatype, dest, tag, comm
INTEGER status(MPI_STATUS_SIZE),ierror
0RVW03,IXQFWLRQVVWRUHVDQLQWHJHUHUURUFRGHLQWKH DUJXPHQW LHUURU +RZHYHU ZH ZLOO LJQRUH
WKHVHUHWXUQYDOXHVLQPRVWFDVHV
7KH FRQWHQWV RI WKH PHVVDJH DUH VWRUHG LQ D EORFN RI PHPRU\ UHIHUHQFHG E\ WKH DUJXPHQW
message7KHQH[WWZRDUJXPHQWV countDQGdatatypeDOORZWKHV\VWHPWRLGHQWLI\WKHHQGRI
WKHPHVVDJHLWFRQWDLQVDVHTXHQFHRI countYDOXHVHDFKKDYLQJ 03,W\SHdatatype7KLVW\SHLV
QRWD)RUWUDQW\SHDOWKRXJKPRVWRIWKHSUHGHILQHGW\SHVFRUUHVSRQG)RUWUDQW\SHV7KHSUHGHILQHG
03,W\SHVDQGWKHFRUUHVSRQGLQJ)2575$1W\SHVLIWKH\H[LVWDUHOLVWHGLQWKHIROORZLQJWDEOH

03,GDWDW\SH )2575$1GDWDW\SH
 03,B,17(*(5 ,17(*(5

03,B5($/ 5($/

03,B'28%/(B35(&,6,21 '28%/(35(&,6,21

03,B&203/(; &203/(;

03,B/2*,&$/ /2*,&$/

03,B&+$5$&7(5 &+$5$&7(5

03,B%<7(

03,B3$&.('

7KHODVWWZRW\SHV MPI_BYTEDQGMPI_PACKEDGRQWFRUUHVSRQGWRVWDQGDUG)RUWUDQW\SHV
7KH 03,B%<7( W\SH FDQ EH XVHG LI \RX ZLVK WR IRUFH WKH V\VWHP WR SHUIRUP QR FRQYHUVLRQ
EHWZHHQ GLIIHUHQW GDWD UHSUHVHQWDWLRQV  HJ RQ D KHWHURJHQHRXV QHWZRUN RI ZRUNVWDWLRQV XVLQJ
GLIIHUHQWUHSUHVHQWDWLRQVRIGDWD:HOOGLVFXVVWKHW\SH MPI_PACKEDODWHU
1RWHWKDWWKHDPRXQWRIVSDFHDOORFDWHGIRUWKHUHFHLYLQJEXIIHUGRHVQRWKDYHWRPDWFKWKHH[DFW
DPRXQWRIVSDFHLQWKHPHVVDJHEHLQJUHFHLYHG)RUH[DPSOHZKHQRXUSURJUDPLVUXQWKHVL]HRI
WKHPHVVDJHWKDWSURFHVVVHQGV len_trim(message)LVFKDUDFWHUVEXWSURFHVVUHFHLYHVWKH
PHVVDJHLQDEXIIHUWKDWKDVVWRUDJHIRUFKDUDFWHUV7KVLPDNHVVHQVH,QJHQHUDOWKHUHFHLYLQJ
SURFHVV PD\ QRW NQRZ WKH H[DFW VL]H RI WKH PHVVDJH EHLQJ VHQW 6R 03, DOORZV D PHVVDJH WR EH
UHFHLYHGDVORQJDVWKHUHLVVXIILFLHQWVWRUDJHDOORFDWHG,IWKHUHLVQWVXIILFLHQWVWRUDJHDQRYHUIORZ
HUURURFFXUV>@

7KH DUJXPHQWV dest DQG source DUH UHVSHFWLYHO\ WKH UDQNV RI WKH UHFHLYLQJ DQG WKH VHQGLQJ
SURFHVVHV 03, DOORZV source WR EH ZLOGFDUG 7KHUH LV D SUHGHILQHG FRQVWDQW
MPI_ANY_SOURCEWKDWFDQEHXVHGLIDSURFHVVLVUHDG\WRUHFHLYHDPHVVDJHIURP DQ\VHQGLQJ
SURFHVVUDWKHUWKDQDSDUWLFXODUVHQGLQJSURFHVV7KHUHLV QRWDZLOGFDUGIRUdest
$V ZH QRWHG HUOLHU 03, KDV WZR PHFKDQLVPV VSHFLILFDOO\ GHVLJQHG IRU SDUWLWLRQLQJ WKH PHVVDJH
VSDFH  WDJV DQG FRPPXQLFDWRUV 7KH DUJXPHQWV tag DQG comm DUH UHVSHFWLYHO\ WKH WDJ DQG
FRPPXQLFDWRU 7KH tag LV DQ LQWHJHU DQG IRU QRZ RXU RQO\ FRPPXQLFDWRU LV
MPI_COMM_WORLDZKLFKDVZHQRWHGHDUOLHULVSUHGHILQHGRQDOO03,V\VWHPVDQGFRQVLVWV
RI DOO WKH SURFHVVHV UXQQLQJ ZKHQ H[HFXWLRQ RI WKH SURJUDP EHJLQV 7KHUH LV D ZLOGFDUG
MPI_ANY_TAGWKDWMPI_RecvFDQXVHIRUWKHWDJ7KHUHLV QRZLOGFDUGIRUWKHFRPPXQLFDWRU
,QRWKHUZRUGVLQRUGHUIRUSURFHVV $WRVHQGDPHVVDJHWRSURFHVV %WKHDUJXPHQW commWKDW$
XVHVLQMPI_SendPXVWEHLGHQWLFDOWRWKHDUJXPHQWWKDW %XVHVLQMPI_Recv

7KHODVWDUJXPHQWRI MPI_RecvstatusUHWXUQVLQIRUPDWLRQRQWKHGDWDWKDWZDVDFWXDOO\UHFHLYHG
,WUHIHUHQFHVDDUUD\ZLWKWZRHOHPHQWVRQHIRUWKHVRXUFHDQGRQHIRUWKHWDJV6RLIIRUH[DPSOH
WKH source RI WKH UHFHLYH ZDV MPI_ANY_SOURCE WKHQ status ZLOO FRQWDLQ WKH UDQN RI WKH
SURFHVVWKDWVHQWWKHPHVVDJH

 $Q$SSOLFDWLRQ
1RZWKDWZHNQRZKRZWRVHQGPHVVDJHZLWK03,OHWVZULWHDSURJUDPWKDWXVHVPHVVDJH
SDVVLQJWRFDOFXODWHDGHILQLWHLQWHJUDOZLWKWKHWUDSH]RLGUXOH

6HULDOSURJUDP

5HFDOOWKDWWKHWUDSH]RLGUXOHHVWLPDWHV E\GLYLGLQJWKHLQWHUYDO>DE@LQWRQ
VHJPHQWVRIHTXDODQGFDOFXODWLQJWKHIROORZLQJVXP

+HUHK EDQDQG[  DLKL Q

L

%\ SXWWLQJ I[ LQWR D VXESURJUDP ZH FDQ ZULWH D VHULDO SURJUDP IRU FDOFXODWLQJ DQ
LQWHJUDOXVLQJWKHWUDSH]RLGUXOH
C serial.f -- calculate definite integral using
trapezoidal
C rule.
C
C The function f(x) is hardwired.
C Input: a, b, n.
C Output: estimate of integral from a to b of f(x)
C using n trapezoids.
PROGRAM serial
IMPLICIT NONE
real integral
real a
real b
integer n
real h
real x
integer i
real f
external f
print *, 'Enter a, b, and n'
read *, a, b, n
 h = (b-a)/n
integral = (f(a) + f(b))/2.0
x = a
do i = 1 , n-1
x = x + h
integral = integral + f(x)
enddo
integral = integral*h
print *,'With n =', n,' trapezoids, our estimate'
print *,'of the integral from ', a, ' to ',b, ' = ' ,
+integral
end
C*****************************************************
*
real function f(x)
IMPLICIT NONE
real x
C Calculate f(x).
f = x*x
return
end
C*****************************************************
*

3DUDOOHOL]LQJWKH7UDSH]RLG5XOH
2QH DSSURDFK WR SDUDOOHOL]LQJ WKLV SURJUDP LV WR VLPSO\ VSOLW WKH LQWHUYDO > DE@ XS DPRQJ WKH
SURFHVVHVDQGHDFKSURFHVVFDQHVWLPDWHWKHLQWHJUDORI I[RYHULWVVXELQWHUYDO,QRUGHUWRHVWLPDWH
WKHWRWDOLQWHJUDOWKHSURFHVVHVORFDOFDOFXODWLRQVDUHDGGHG
6XSSRVH WKHUH DUH S SURFHVVHV DQG Q WUDSH]RLGV DQG LQ RUGHU WR VLPSOLI\ WKH GLVFXVVLRQ DOVR
VXSSRVHWKDWQLVHYHQO\GLYLVLEOHE\ S7KHQLWLVQDWXUDOIRUWKHILUVWSURFHVVWRFDOFXODWHWKHDUHDRI
WKHILUVW QSWUDSH]RLGVWKHVHFRQGSURFHVVWRFDOFXODWHWKHDUHDRIWKHQH[W QSHWF6RSURFHVVT
ZLOOHVWLPDWHWKHLQWHJUDORYHUWKHLQWHUYDO
7KXVHDFKSURFHVVQHHGVWKHIROORZLQJLQIRUPDWLRQ

7KHQXPEHURISURFHVVHV S

,WVUDQN

7KHHQWLUHLQWHUYDORILQWHJUDWLRQ> DE@

7KHQXPEHURIVXELQWHUYDOV Q

5HFDOOWKDWWKHILUVWWZRLWHPVFDQEHIRXQGE\FDOOLQJWKH03,IXQFWLRQV MPI_Comm_sizeDQG
MPI_Comm_Rank7KHODWWHUWZRLWHPVVKRXOGSUREDEO\EHLQSXWE\WKHXVHU%XWWKLVFDQUDLVH
VRPH GLIILFXOW SUREOHPV 6R IRU RXU ILUVW DWWHPSW DW FDOFXODWLQJ WKH LQWHJUDO OHWV KDUGZLUH WKHVH
YDOXHVE\VLPSO\VHWWLQJWKHLUYDOXHVZLWKDVVLJQPHQWVWDWHPHQWV
$ VWUDLJKWIRUZDUG DSSURDFK WR VXPPLQJ WKH SURFHVVHV LQGLYLGXDO FDOFXODWLRQV LV WR KDYH HDFK
SURFHVVVHQGLWVORFDOFDOFXODWLRQWRSURFHVVDQGKDYHSURFHVVGRWKHILQDODGGLWLRQ
:LWKWKHVHDVVXPSWLRQVZHFDQZULWHDSDUDOOHOWUDSH]RLGUXOHSURJUDP
c trap.f -- Parallel Trapezoidal Rule, first version
c
c Input: None.
c Output: Estimate of the integral from a to b of f(x)
c using the trapezoidal rule and n trapezoids.
c
c Algorithm:
c 1. Each process calculates "its" interval of
c integration.
c 2. Each process estimates the integral of f(x)
c over its interval using the trapezoidal rule.
c 3a. Each process != 0 sends its integral to 0.
c 3b. Process 0 sums the calculations received from
c the individual processes and prints the result.
c
c Note: f(x), a, b, and n are all hardwired.
c
program trapezoidal
c
IMPLICIT NONE
include 'mpif.h'
c
integer my_rank ! My process rank.
integer p ! The number of processes.
real a ! Left endpoint.
real b ! Right endpoint.
integer n ! Number of trapezoids.
real h ! Trapezoid base length.
real local_a ! Left endpoint for my process.
real local_b ! Right endpoint my process.
integer local_n ! Number of trapezoids for my
! calculation.
real integral ! Integral over my interval.
real total ! Total integral.
integer source ! Process sending integal.
integer dest ! All messages go to 0.
integer tag
integer status(MPI_STATUS_SIZE)
integer ierr
real Trap
data a, b, n, dest, tag /0.0, 1.0, 1024, 0, 50/
C Let the system do what it needs to start up MPI.
call MPI_INIT(ierr)
C Get my process rank.
call MPI_COMM_RANK(MPI_COMM_WORLD, my_rank, ierr)
C Find out how many processes are being used.
call MPI_COMM_SIZE(MPI_COMM_WORLD, p, ierr)
h = (b-a)/n ! h is the same for all processes.
local_n = n/p ! So is the number of trapezoids.
C Length of each process' interval of integration = local_n*h.
C So my interval starts at :
local_a = a + my_rank*local_n*h
local_b = local_a + local_n*h
integral = Trap(local_a, local_b, local_n, h)
C Add up the integals calculated by each process.
if (my_rank .EQ. 0) then
total = integral
do source = 1, p-1
call MPI_RECV(integral, 1, MPI_REAL, source, tag,
+ MPI_COMM_WORLD, status, ierr)
total = total + integral
enddo
else
call MPI_SEND(integral, 1, MPI_REAL, dest,
+ tag, MPI_COMM_WORLD, ierr)
endif
C Print the result.
if (my_rank .EQ. 0) then
write(6,200) n
200 format(' ','With n = ',I4,' trapezoids, our estimate')
write(6,300) a, b, total
300 format(' ','of the integral from ',f6.2,' to ',f6.2,
+ ' = ',f11.5)
endif
C Shut down MPI.
call MPI_FINALIZE(ierr)
end program trapezoidal
 real function Trap(local_a, local_b, local_n, h)
IMPLICIT NONE
real local_a
real local_b
integer local_n
real h
real integral ! Store result in integal.
real x
real i
real f
integral = (f(local_a) + f(local_b))/2.0
x = local_a
do i = 1, local_n - 1
x = x + h
integral = integral + f(x)
enddo
integal = integral*h
Trap = integral
return
end

real function f(x)

IMPLICIT NONE
real x
f = x*x
end

2EVHUYH WKDW WKLV SURJUDP DOVR XVHV WKH 630' SDUDGLJP (YHQ WKRXJK SURFHVV  H[HFXWHV DQ
HVVHQWLDOO\GLIIHUHQWVHWRIFRPPDQGVIURPWKHUHPDLQLQJSURFHVVHVLWVWLOOUXQVWKHVDPHSURJUDP
7KHGLIIHUHQWFRPPDQGVDUHH[HFXWHGE\EUDQFKLQJEDVHGRQWKHSURFHVVUDQN
 ,2RQ3DUDOOHO3URFHVVRUV
2QHREYLRXVSUREOHPZLWKRXUSURJUDPLVLWVODFNRIJHQHUDOLW\WKHGDWD DEDQGQDUHKDUGZLUHG
7KH XVHU VKRXOG EH DEOH WR HQWHU WKHVH YDOXHV GXULQJ H[HFXWLRQ /HWV ORRN PRUH FDUHIXOO\ DW WKH
SUREOHPRI,2RQSDUDOOHOPDFKLQHV

,Q RXU greetings DQG trapezoid SURJUDPV ZH DVVXPHG WKDW SURFHVV  FRXOG ZULWH WR VWDQGDUG
RXWSXWWKHWHUPLQDOVFUHHQ0RVWSDUDOOHOSURFHVVRUVSURYLGHWKLVPXFK,2,QIDFWPRVWSDUDOOHO
SURFHVVRUV DOORZ DOO SURFHVVRUV WR ERWK UHDG IURP VWDQGDUG LQSXW DQG ZULWH WR VWDQGDUG RXWSXW
+RZHYHUGLIILFXOWDULVHZKHQVHYHUDOSURFHVVHVDUHVLPXOWDQHRXVO\WU\LQJWRH[HFXWH,2IXQFWLRQV
,QRUGHUWRXQGHUVWDQGWKLVOHWVORRNDWDQH[DPSOH

6XSSRVHZHPRGLI\WKH trapezoidSURJUDPVRWKDWHDFKSURFHVVDWWHPSWVWRUHDGWKHYDOXHV D E

DQGQE\DGGLQJWKHVWDWHPHQW
read *, a , b, n
6XSSRVHDOVRWKDWZHUXQWKHSURJUDPZLWKWZRSURFHVVHVDQGWKHXVHUW\SHVLQ
0 1 1024
:KDWKDSSHQ"'RERWKSURFHVVHVJHWWKHGDWD"'RHVRQO\RQH"2UHYHQZRUVHGRHVVD\SURFHVV
JHWWKHDQGZKLOHSURFHVVJHWVWKH" ,IDOOWKHSURFHVVHV JHW WKH GDWD ZKDW KDSSHQV
ZKHQZHZULWHDSURJUDPZKHUHZHZDQWSURFHVVJHWVWKHGDWDZKDWKDSSHQVWRWKHRWKHUV",VLW
HYHQUHDVRQDEOHWRKDYHPXOWLSOHSURFHVVHVUHDGLQJGDWDIURPDVLQJOHWHUPLQDO"
2QWKHRWKHUKDQGZKDWKDSSHQVLIVHYHUDOSURFHVVHVDWWHPSWWRVLPXOWDQHRXVO\ZULWHGDWDWRWKH
WHUPLQDOVFUHHQ'RHVWKHGDWDIURPSURFHVVJHWSULQWHGILUVWWKHQWKHGDWDIRUPSURFHVVHWF"
2UGRHVWKHGDWDDSSHDULQVRPHUDQGRPRUGHU"2UHYHQZRUVHGRHVWKHGDWDIURPWKHGLIIHUHQW
SURFHVVHVJHWDOOPL[HGXSVD\KDOIDOLQHIURPWZRFKDUDFWHUVIURPFKDUDFWHUVIURPWZR
OLQHVIURPHWF"
7KHVWDQGDUG,2FRPPDQGVDYDLODEOHLQ)RUWUDQDQGPRVWRWKHUODQJXDJHVGRQWSURYLGHVLPSOH
VROXWLRQV WR WKHVH SUREOHPV DQG ,2 FRQWLQXHV WR EH WKH VXEMHFW RI FRQVLGHUDEOH UHVHDUFK LQ WKH
SDUDOOHOSURFHVVLQJFRPPXQLW\6ROHWVORRNDWVRPHQRWVRVLPSOHVROXWLRQVWRWKHVHSUREOHPV
7KXVIDUZHKDYHDVVXPHGWKDWSURFHVVFDQDWOHDVWZULWHWRVWDQGDUGRXWSXW:HZLOODOVRDVVXPH
WKDWLWFDQUHDGIURPVWDQGDUGLQSXW,QPRVWFDVHVZHZLOORQO\DVVXPHWKDWSURFHVVFDQGR,2
,WVKRXOGEHQRWHGWKDWWKLVLVDYHU\ZHDNDVVXPSWLRQVLQFHDVZHQRWHGPRVWSDUDOOHOPDFKLQHV
DOORZPXOWLSOHSURFHVVHVWRFDUU\RXW,2<RXPLJKWZDQWWRDVN\RXUORFDOH[SHUWZKHWKHUWKHUH
DUHDQ\UHVWULFWLRQVRQZKLFKSURFHVVHVFDQGR,2
,I RQO\ SURFHVV  FDQ GR ,2 WKHQ ZH QHHG IRU SURFHVV  WR VHQG WKH XVHU LQSXW WR WKH RWKHU
SURFHVVHV 7KLV LV UHDGLO\ DFFRPSOLVKHG ZLWK D VKRUW ,2 IXQFWLRQ WKDW XVHV MPI_Send DQG
MPI_Recv
C
****************************************************************
*
C Function Get_data
C Reads in the user input a, b, and n.
C Input arguments:
C 1. integer my_rank: rank of current process.
C 2. integer p: number of processes.
C Output parameters:
C 1. real a: left endpoint a.
C 2. real b: right endpoint b.
C 3. integer n: number of trapezoids.
C Algorithm:
C 1. Process 0 prompts user for input and
C reads in the values.
C 2. Process 0 sends input values to other
C processes.
C
subroutine Get_data(a, b, n, my_rank, p)
IMPLICIT NONE
real a
real b
integer n
integer my_rank
integer p
INCLUDE 'mpif.h'
C
integer source
integer dest
integer tag
integer status(MPI_STATUS_SIZE)
integer ierr
data source /0/
C
if (my_rank == 0) then
print *, 'Enter a, b and n'
read *, a, b, n
C
C
do dest = 1 , p-1
tag = 0
call MPI_SEND(a, 1, MPI_REAL , dest, tag,
+ MPI_COMM_WORLD, ierr )
tag = 1
call MPI_SEND(b, 1, MPI_REAL , dest, tag,
+ MPI_COMM_WORLD, ierr )
tag = 2
call MPI_SEND(n, 1, MPI_INTEGER, dest,
+ tag, MPI_COMM_WORLD, ierr )
enddo
else
tag = 0
call MPI_RECV(a, 1, MPI_REAL , source, tag,
+ MPI_COMM_WORLD, status, ierr )
tag = 1
call MPI_RECV(b, 1, MPI_REAL , source, tag,
+ MPI_COMM_WORLD, status, ierr )
tag = 2
call MPI_RECV(n, 1, MPI_INTEGER, source, tag,
+ MPI_COMM_WORLD, status, ierr )
endif
return
end
C
C
C
****************************************************************
**


 &ROOHFWLYH&RPPXQLFDWLRQ
7KHUHDUHSUREDEO\DIHZWKLQJVLQWKHWUDSH]RLGUXOHSURJUDPWKDWZHFDQLPSURYHRQ)RU
H[DPSOHWKHUHLVWKH,2LVVXH7KHUHDUHDOVRDFRXSOHRISUREOHPVZHKDYHQWGLVFXVVHG
\HW/HWVORRNDWZKDWKDSSHQVZKHQWKHSURJUDPLVUXQZLWKHLJKWSURFHVVHV
$OOWKHSURFHVVHVEHJLQ H[HFXWLQJ WKH SURJUDP  PRUH RU OHVV  VLPXODWDQHRXVO\ +RZHYHU
DIWHU FDUU\LQJ RXW WKH EDVLF VHWXS WDVNV  FDOOV WR 03,B,QLW 03,B&RPPBVL]H DQG
03,B&RPPBUDQNSURFHVVHVDUHLGOHZKLOHSURFHVVFROOHFWVWKHLQSXWGDWD:HGRQW
ZDQW WR KDYH LGOH SURFHVVHV EXW LQ YLHZ RI RXU UHVWULFWLRQV RQ ZKLFK SURFHVVHV FDQ UHDG
LQSXWGDWDWKHKLJKHUUDQNSURFHVVHVPXVWFRQWLQXHWRZDLWZKLOHVHQGVWKHQSXWGDWDWR
WKH ORZHU UDQN SURFHVVHV 7KLV LVQW MXVW DQ ,2 LVVXH 1RWLFH WKDW WKHUH LV D VLPLODU
LQHIILFLHQF\DWWKHHQGRIWKHSURJUDPZKHQSURFHVVGRHVDOOWKHZRUNRIFROOHFWLQJDQGG
DGGLQJWKHORFDOLQWHJUDOV
2IFRXUVHWKLVLVKLJKO\XQGHVLUDEOHWKHPDLQSRLQWRISDUDOOHOSURFHVVLQJLVWRJHWPXOWLSOH
SURFHVVHVWRFROODERUDWHRQVROYLQJDSUREOHP,IRQHRIWKHSURFHVVHVLVGRLQJPRVWRIWKH
ZRUNZHPLJKWDVZHOOXVHDFRQYHQWLRQDOVLQJOHSURFHVVRUPDFKLQH

7UHH6WUXFWXUHG&RPPXQLFDWLRQ
/HWVWU\WRLPSURYHRXUFRGH:HOOEHJLQE\IRFXVLQJRQWKHGLVWULEXWLRQRIWKHLQSXWGDWD+RZ
FDQZHGLYLGHWKHZRUNPRUHHYHQO\DPRQJWKHSURFHVVHV"$QDWXUDOVROXWLRQLVWRLPDJLQHWKDWZH
KDYHDWUHHRISURFHVVHVZLWKDWWKHURRW
'XULQJWKHILUVWVWDJHRIGDWDGLVWULEXWLRQVHQGVWKHGDWDWRVD\'XULQJWKHQH[WVWDJHVHQGV
WKHGDWDWRZKLOHVHQGVLWWR'XULQJWKHODVWVWDJHVHQGVWRZKLOHVHQGVWRVHQGVWR
DQGVHQGVWRVHHILJXUH6RZHKDYHUHGXFHGRXULQSXWGLVWULEXWLRQORRSIURPVWDJHVWR
VWDJHV0RUHJHQHUDOO\LIZHKDYHSSURFHVVHVWKLVSURFHGXUHDOORZVXVWRGLVWULEXWHWKHLQSXWGDWD
LQ VWDJHVUDWKHUWKDQSVWDJHVZKLFKLISLVODUJHLVDKXJHVDYLQJV
 )LJXUH  3URFHVVRUV FRQILJXUHG DV D WUHH

,QRUGHUWRPRGLI\WKH*HWBGDWDIXQFWLRQWRXVHDWUHHVWUXFWXUHGGLVWULEXWLRQVFKHPHZHQHHGWR
LQWURGXFH D ORRS ZLWK VWDJHV ,Q RUGHU WR LPSOHPHQW WKH ORRS HDFK SURFHVV QHHGV WR
FDOFXODWHDWHDFKVWDJH

ZKHWKHULWUHFHLYHVDQGLIVRWKHVRXUFHDQG

ZKHWKHULWVHQGVDQGLIVRWKHGHVWLQDWLRQ
$V\RXFDQSUREDEO\JXHVVWKHVHFDOFXODWLRQVFDQEHDELWFRPSOLFDWHGHVSHFLDOO\VLQFHWKHUHLVQR
FDQRQLFDOFKRLFHRIRUGHULQJ,QRXUH[DPSOHZHFKRVH
VHQGVWR
VHQGVWRVHQGVWR
VHQGVWRVHQGVWRVHQGVWRVHQGVWR
:HPLJKWDOVRKDYHFKRVHQIRUH[DPSOH
VHQGVWR
VHQGVWRVHQGVWR
VHQGVWRVHQGVWRVHQGVWRVHQGVWR
,QGHHGXQOHVVZHNQRZVRPHWKLQJDERXWWKHXQGHUO\LQJWRSRORJ\RIRXUPDFKLQHZHFDQWUHDOO\
GHFLGHZKLFKVFKHPHLVEHWWHU
6RLGHDOO\ZHZRXOGSUHIHUWRXVHDIXQFWLRQWKDWKDVEHHQVSHFLILFDOO\WDLORUHGWRWKHPDFKLQHZHUH
XVLQJVRWKDWZHZRQWKDYHWRZRUU\DERXWDOOWKHVHWHGLRXVGHWDLOVDQGZHZRQWKDYHWRPRGLI\
RXUFRGHHYHU\WLPHZHFKDQJHPDFKLQHV$V\RXPD\KDYHJXHVV03,SURYLGHVVXFKDIXQFWLRQ
%URDGFDVW
$ FRPPXQLFDWLRQ SDWWHUQ WKDW LQYROYHV DOO WKH SURFHVVHV LQ D FRPPXQLFDWRU LV D
FROOHFWLYHFRPPXQLFDWLRQ$VDFRQVHTXHQFHDFROOHFWLYHFRPPXQLFDWLRQXVXDOO\LQYROYHV
PRUHWKDQWZRSURFHVVHV$ EURDGFDVWLVDFROOHFWLYHFRPPXQLFDWLRQLQZKLFKDVLQJOH
SURFHVVVHQGVWKHVDPHGDWDWRSHYHU\SURFHVV,Q03,WKHIXQFWLRQIRUEURDGFDVWLQJ
GDWDLV03,B%FDVW
MPI_BCAST( BUFFER, COUNT, DATATYPE, ROOT, COMM, IERROR
)
<type> BUFFER(*)
INTEGER COUNT, DATA, ROOT, COMM, IERROR
,WVLPSO\VHQGVDFRS\RIWKHGDWDLQ%8))(5RQSURFHVV5227WRHDFKSURFHVVLQ
WKH FRPPXQLFDWRU &200 ,7 VKRXOG EH FDOOHG E\ DOO WKH SURFHVVHV LQ WKH
FRPPXQLFDWRUZLWKWKHVDPHDUJXPHQWVIRU5227DQG&200+HQFHDEURDGFDVW
PHVVDJH FDQQRW EH UHFHLYHG ZLWK 03,B5HFY 7KH SDUDPHWHUV &2817 DQG
'$7$7<3(KDYHWKHVDPHIXQFWLRQWKDWWKH\KDYHLQ03,B6HQGDQG03B5HFY 
WKH\VSHFLI\WKHH[WHQWRIWKHPHVVDJH+RZHYHUXQOLNHWKHSRLQWWRSRLQWIXQFWLRQV
03, LQVLVWV WKDW LQ FROOHFWLYH FRPPXQLFDWLRQ &2817 DQG '$7$7<3( EH WKH
VDPH RQ DOO WKH SURFHVVHV LQ WKH FRPPXQLFDWRU >@ 7KH UHDVRQ IRU WKLV LV WKDW LQ
VRPHFROOHFWLYHRSHUDWLRQVVHHEHORZDVLQJOHSURFHVVZLOOUHFHLYHGDWDIURPPDQ\
RWKHUSURFHVVHVDQGLQRUGHUIRUDSURJUDPWRGHWHUPLQHKRZPXFKGDWDKDVEHHQ
UHFHLYHGLWZRXOGQHHGDQHQWLUH DUUD\RIUHWXUQVWDWXVHV
:HFDQUHZULWHWKH*HWBGDWDIXQFWLRQXVLQJ03,B%FDVWDVIROORZV
subroutine Get_data2(a, b, n, my_rank)
real a
real b
integer n
integer my_rank
integer ierr
include 'mpif.h'
C
C
if (my_rank .EQ. 0) then
print *, 'Enter a, b, and n'
read *, a, b, n
endif
C
 call MPI_BCAST(a, 1, MPI_REAL , 0, MPI_COMM_WORLD,
ierr )
call MPI_BCAST(b, 1, MPI_REAL , 0, MPI_COMM_WORLD,
ierr )
call MPI_BCAST(n, 1, MPI_INTEGER, 0, MPI_COMM_WORLD,
ierr )
end subroutine Get_data2
C
C
C
******************************************************
************
&HUWDLQO\ WKLV YHUVLRQ RI *HWBGDWD LV PXFK PRUH FRPSDFW DQG UHDGLO\
FRPSUHKHQVLEOH WKDQ WKH RULJLQDO DQG LI 03,B%FDVW KDV EHHQ RSWLPL]HG IRU \RXU
V\VWHPLWZLOODOVREHDJRRGGHDOIDVWHU

5HGXFH
,QWKHWUDSH]RLGUXOHSURJUDPDIWHUWKHLQSXWSKDVHHYHU\SURFHVVRUH[HFXWHVHVVHQWLDOO\ WKH VDPH
FRPPDQGVXQWLOWKHILQDOVXPPDWLRQSKDVH6RXQOHVVRXUIXQFWLRQI[LVIDLUO\FRPSOLFDWHGLHLW
UHTXLUHV FRQVLGHUDEO\ PRUH ZRUN WR HYDOXDWH RYHU FHUWDLQ SDUWV RI >DE@ WKLV SDUW RI WKH SURJUDP
GLVWULEXWHVWKHZRUNHTXDOO\DPRQJWKHSURFHVVRUV$VZH KDYH DOUHDG\ QRWHG WKLV LV QRW WKH FDVH
ZLWKWKHILQDOVXPPDWLRQSKDVHZKHQRQFHDJDLQSURFHVVJHWVDGLVSURSRUWLRQDWHDPRXQWRIWKH
ZRUN+RZHYHU\RXKDYHSUREDEO\DOUHDG\QRWLFHGWKDWE\UHYHUVLQJWKHDUURZVLQILJXUHZHFDQ
XVHWKHVDPHLGHDZHXVHGLQVHFWLRQ7KDWLVZHFDQGLVWULEXWHWKHZRUNRIFDOFXODWLQJWKHVXP
DPRQJWKHSURFHVVRUVDVIROORZV
 DVHQGVWRVHQGVWRVHQGVWRVHQGVWR
EDGGVLWVLQWHJUDOWRWKDWRIDGGVLWVLQWHJUDOWRWKDWRIHWF
 DVHQGVWRVHQGVWR
EDGGVDGGV
 DVHQGVWR
D DGGV
2IFRXUVHZHUXQLQWRWKHVDPHTXHVWLRQWKDWRFFXUUHGZKHQZHZHUHZULWLQJRXURZQEURDGFDVW
LVWKLVWUHHVWUXFWXUHPDNLQJRSWLPDOXVHRIWKHWRSRORJ\RIRXUPDFKLQH"2QFHDJDLQZHKDYHWR
DQVZHUWKDWWKLVGHSHQGVRQWKHPDFKLQH6RDVEHIRUHZHVKRXOGOHW03,GRWKHZRUNE\XVLQJDQ
RSWLPL]HGIXQFWLRQ
7KH JOREDO VXP WKDW ZH ZLVK WR FDOFXODWH LV DQ H[DPSOH RI D JHQHUDO FODVV RI FROOHFWLYH
FRPPXQLFDWLRQRSHUDWLRQVFDOOHG UHGXFWLRQRSHUDWLRQV,QDJOREDOUHGXFWLRQRSHUDWLRQDOOWKHSURFHVVHV
LQDFRPPXQLFDWRUFRQWULEXWHGDWDZKLFKLVFRPELQHGXVLQJDELQDU\RSHUDWLRQ7\SLFDOELQDU\
RSHUDWLRQV DUH DGGLWLRQ PD[ PLQ ORJLFDODQG HWF 7KH 03, IXQFWLRQ IRU SHUIRUPLQJ D UHGXFWLRQ
RSHUDWLRQLV
MPI_Reduce(OPERAND, RESULT, COUNT, DATATYEP, OP, ROOT, COMM,
IERROR)
<type> OPERAND(*), RESULT(*)
INTEGER COUNT, DATATYPE, OP, ROOT, COMM, IERROR
03,B5HGXFHFRPELQHVWKHRSHUDQGVVWRUHGLQ23(5$1'XVLQJRSHUDWLRQ23DQGVWRUHVWKH
UHVXOWLQ5(68/7RQSURFHVV5227%RWK23(5$1'DQG5(68/7UHIHUWR&2817PHPRU\
ORFDWLRQVZLWKW\SH'$7$7<3(03,B5HGXFHPXVWEHFDOOHGE\DOOSURFHVVHVLQWKHFRPPXQLFDWRU
&200DQG&2817'$7$7<3(DQG23PXVWEHWKHVDPHRQHDFKSURFHVV
7KHDUJXPHQW23FDQWDNHRQRQHRIWKHIROORZLQJSUHGHILQHGYDOXHV

2SHUDWLRQ1DPH 0HDQLQJ

03,B0$; 0D[LPXP

03,B0,1 0LQLPXP

03,B680 6XP

03,B352' 3URGXFW

03,B/$1' /RJLFDO$QG

03,B%$1' %LWZLVH$QG

03,B/25 /RJLFDO2U

03,B%25 %LWZLVW2U

03,B/;25 /RJLFDO([FOXVLYH2U

03,B%;25 %LWZLVH([FOXVLYH2U

03,B0$;/2& 0D[LPXPDQG/RFDWLRQRI0D[LPXP

03,B0,1/2& 0LQLPXPDQG/RFDWLRQRI0LQLPXP

,WLVDOVRSRVVLEOHWRGHILQHDGGLWLRQDORSHUDWLRQV)RUGHWDLOVVHH>@
$VDQH[DPSOHOHWVUHZULWHWKHODVWIHZOLQHVRIWKHWUDSH]RLGUXOHSURJUDP
C Add up the integrals calculated by each process.
MPI_Reduce( INTEGRAL, TOTAL, 1, MPI_REAL, MPI_SUM, 0,
+ MPI_COMM_WORLD, ierr)
C Print the result.
1RWHWKDWHDFKSURFHVVRUFDOOV03,B5HGXFHZLWKWKHVDPHDUJXPHQWV,QSDUWLFXODUHYHQWKRXJK
WRWDORQO\KDVVLJQLILFDQFHRQSURFHVVHDFKSURFHVVPXVWVXSSO\DQDUJXPHQW
 2WKHU&ROOHFWLYH&RPPXQLFDWLRQ
)XQFWLRQV
03,VXSSOLHVPDQ\RWKHUFROOHFWLYHFRPPXQLFDWLRQIXQFWLRQV:HEULHIO\HQXPHUDWHVRPHRIWKHVH
KHUH)RUIXOOGHWDLOVVHH>@

MPI_Barrier( COMM, IERROR)

INTEGER COMM, IERROR
03,B%DUULHUSURYLGHVDPHFKDQLVPIRUV\QFKURQL]LQJDOOWKHSURFHVVHVLQWKHFRPPXQLFDWRUFRPP
(DFKSURFHVVEORFNVLHSDXVHVXQWLOHYHU\SURFHVVLQFRPPKDVFDOOHG03,B%DUULHU
MPI_Gather(SEND_BUF, SEND_COUNT, SEND_TYPE, RECV_BUF,
RECV_COUNT, RECV_TYPE, ROOT, COMM, IERROR )
<type>SEND_BUF(*), RECV_BUF(*)
INTEGER SEND_COUNT,SEND_TYPE,RECV_COUNT,
RECV_TYPE,ROOT,COMM,IERROR

(DFKSURFHVVLQFRPPVHQGVWKHFRQWHQWVRIVHQGBEXIWRSURFHVVZLWKUDQNURRW7KHUSRFHVVURRW
FRQFDWHQDWHVWKHUHFHLYHGGDWDLQSURFHVVUDQNRUGHULQUHFYBEXI7KDWLVWKHGDWDIURPSURFHVVLV
IROORZHGE\WKHGDWDIURPSURFHVVZKLFKLVIROORZHGE\WKHGDWDIURPSURFHVVHWF7KHUHFY
DUJXPHQWVDUHVLJQLILFDQWRQO\RQWKHSURFHVVZLWKUDQNURRW7KHDUJXPHQWUHFYBFRXQWLQGLFDWHVWKH
QXPEHURILWHPVUHFHLYHGIURPHDFKSURFHVVQRWWKHWRWDOQXPEHUUHFHLYHG
MPI_Scatter(SEND_BUF, SEND_COUNT, SEND_TYPE, RECV_BUF,
RECV_COUNT, RECV_TYPE, ROOT, COMM, IERROR )
<type>SEND_BUF(*), RECV_BUF(*)
INTEGER SEND_COUNT,SEND_TYPE,RECV_COUNT,
RECV_TYPE,ROOT,COMM,IERROR
7KHSURFHVVZLWKUDQNURRWGLVWULEXWHVWKHFRQWHQWVRIVHQGBEXIDPRQJWKHSURFHVVHV7KHFRQWHQWV
RIVHQGBEXIDUHVSOLWLQWRSVHJPHQWVHDFKFRQVLVWLQJRIVHQGBFRXQWLWHPV7KHILUVWVHJPHQWJRHVWR
SURFHVVWKHVHFRQGWRSURFHVVHWF7KHVHQGDUJXPHQWVDUHVLJQLILFDQWRQO\RQSURFHVVURRW
MPI_Allgather(SEND_BUF, SEND_COUNT, SEND_TYPE, RECV_BUF,
RECV_COUNT, RECV_TYPE, ROOT, COMM, IERROR )
<type>SEND_BUF(*), RECV_BUF(*)
INTEGER SEND_COUNT,SEND_TYPE,RECV_COUNT,
RECV_TYPE,ROOT,COMM,IERROR

03,B$OOJDWKHUJDWKHUVWKHFRQWHQWVRIHDFKVHQGBEXIRQHDFKSURFHVV,WVHIIHFWLVWKHVDPHDVLI
WKHUH ZHUH D VHTXHQFH RI S FDOOV WR 03,B*DWKHU HDFK RI ZKLFK KDV D GLIIHUHQW SURFHVV DFWLQJ DV
URRW
MPI_AllReduce(OPERAND, RESULT, COUNT, DATATYPE, OP, COMM, IERROR
)
<type>OPERAND(*), RESULT(*)
INTEGER COUNT, DATATYPE, OP ,COMM,IERROR
03,B$OOUHGXFHVWRUHVWKHUHVXOWRIWKHUHGXFHRSHUDWLRQ23LQHDFKSURFHVVUHVXOWEXIIHU

 *URXSLQJ'DWDIRU
&RPPXQLFDWLRQ
:LWKFXUUHQWJHQHUDWLRQPDFKLQHVVHQGLQJDPHVVDJHLVDQH[SHQVLYHRSHUDWLRQ6RDVDUXOH
RI WKXPE WKH IHZHU PHVVDJHV VHQW WKH EHWWHU WKH RYHUDOO SHUIRUPDQFH RI WKH SURJUDP
+RZHYHU LQ HDFK RI RXU WUDSH]RLG UXOH SURJUDPV ZKHQ ZH GLVWULEXWHG WKH LQSXW GDWD ZH
VHQW D E DQG Q LQ VHSDUDWH PHVVDJHV  ZKHWKHU ZH XVHG 03,B6HQG DQG 03,B5HFY RU
03,B%FDVW6RZHVKRXOGEHDEOHWRLPSURYHWKHSHUIRUPDQFHRIWKHSURJUDPE\VHQGLQJ
WKH WKUHH LQSXW YDOXHV LQ D VLQJOH PHVVDJH 03, SURYLGHV WKUHH PHFKDQLVPV IRU JURXSLQJ
LQGLYLGXDO GDWD LWHPV LQWR D VLQJOH PHVVDJH  WKH FRXQW SDUDPHWHU WR WKH YDULRXV
FRPPXQLFDWLRQ URXWLQHV GHULYHG GDWDW\SHV DQG 03,B3DFN  03,B8QSDFN :H H[DPLQH
HDFKRIWKHVHRSWLRQVLQWXUQ

7KH&RXQW3DUDPHWHU
5HFDOO WKDW 03,B6HQG 03,B5HFY 03,B%FDVW DQG 03,B5HGXFH DOO KDYH D FRXQW
DQGDdatatypeDUJXPHQW7KHVHWZRSDUDPHWHUVDOORZWKHXVHUWRJURXSGDWDLWHPV
KDYLQJWKHVDPHEDVLFW\SHLQWRDVLQJOHPHVVDJH,QRUGHUWRXVHWKLVWKHJURXSHG
GDWD LWHPV PXVW EH VWRUHG LQ FRQWLJXRXV PHPRU\ ORFDWLRQV 6LQFH )RUWUDQ JXDUDQWHHV
WKDW DUUD\ HOHPHQWV DUH VWRUHG LQ FRQWLJXRXV PHPRU\ ORFDWLRQV LI ZH ZLVK WR VHQG
WKHHOHPHQWVRIDQDUUD\RUDVXEVHWRIDQDUUD\ZHFDQGRVRLQDVLQJOHPHVVDJH,Q
IDFWZHYHDOUHDG\GRQHWKLVLQVHFWLRQZKHQZHVHQWDQDUUD\RI character
$VDQRWKHUH[DPSOHVXSSRVHZHZLVKWRVHQGWKHVHFRQGKDOIRIDYHFWRUFRQWDLQLQJ
UHDOQXPEHUVIURPSURFHVVWRSURFHVV
real vector(100)
integer status(MPI_STATUS_SIZE)
integer p, my_rank, ierr
integer i
.
.
.
if (my_rank == 0) then
C Initialize vector and send.
tag = 47
count = 50
dest = 1
call MPI_SEND(vector(51), count, MPI_REAL, dest, tag,
+ MPI_COMM_WORLD, ierr)
else
tag = 47
count = 50
source = 0
call MPI_RECV(vector(51), count, MPI_REAL, source,
tag,
+ MPI_COMM_WORLD, status, ierr)
endif
8QIRUWXQDWHO\ WKLV GRHVQW KHOS XV ZLWK WKH WUDSH]RLG UXOH SURJUDP 7KH GDWD ZH
ZLVKWRGLVWULEXWHWRWKHRWKHUSURFHVVHVDEDQGQDUHVWRUHGLQDQDUUD\6RHYHQLI
ZHGHFODUHGWKHPRQHDIWHUWKHRWKHULQRXUSURJUDP
real a
real b
integer n
)RUWUDQGRHVQRWJXDUDQWHHWKDWWKH\DUHVWRUHGLQFRQWLJXRXVPHPRU\ORFDWLRQV2QH
PLJKWEHWHPSWHGWRVWRUHQDVDIORDWDQGSXWWKHWKUHHYDOXHVLQDQDUUD\EXWWKLV
ZRXOGEHSRRUSURJUDPPLQJVW\OHDQGLWZRXOGQWDGGUHVVWKHIXQGDPHQWDOLVVXH,Q
RUGHUWRVROYHWKHSUREOHPZHQHHGWRXVHRQHRI03,VRWKHUIDFLOLWLHVIRUJURXSLQJ
GDWD
'HULYHG7\SHVDQG03,B7\SHBVWUXFW
,WPLJKWVHHPWKDWDQRWKHURSWLRQZRXOGEHWRVWRUH DEDQGQLQDGHULYHGW\SHZLWK
WKUHHPHPEHUVWZRUHDOVDQGDQLQWHJHUDQGWU\WRXVHWKHGDWDW\SHDUJXPHQWWR
03,B%FDVW7KHGLIILFXOW\KHUHLVWKDWWKHW\SHRIGDWDW\SHLV03,B'DWDW\SHZKLFKLV
DQDFWXDOW\SHLWVHOIQRWWKHVDPH WKLQJ DV D XVHUGHILQHG W\SH LQ )RUWUDQ  )RU
H[DPSOHVXSSRVHZHLQFOXGHGWKHW\SHGHILQLWLRQ
type INDATA_TYPE
real a
real b
integer n
end type
DQGWKHYDULDEOHGHILQLWLRQ
type (INDATA_TYPE) indata
1RZLIZHFDOO03,B%FDVW
call MPI_Bcast(indata, 1, INDATA_TYPE, 0,
MPI_COMM_WORLD,
+ ierror)
WKHSURJUDPZLOOIDLO7KHGHWDLOGHSHQGRQWKHLPSOHPHQWDWLRQRI03,WKDW\RXUH
XVLQJ7KHSUREOHPKHUHLVWKDW03,LVD SUHH[LVWLQJOLEUDU\RIIXQFWLRQV7KDWLVWKH
03,IXQFWLRQVZHUHZULWWHQZLWKRXWNQRZOHGJHRIWKHGDWDW\SHVWKDW\RXGHILQHLQ
\RXU SURJUDP ,Q SDUWLFXODU QRQH RI WKH 03, IXQFWLRQV NQRZV DERXW
,1'$7$B7<3(
03,SURYLGHVDSDUWLDOVROXWLRQWRWKLV SUREOHPE\DOORZLQJWKHXVHUWREXLOG03,
GDWDW\SHV DW H[HFXWLRQ WLPH ,Q RUGHU WR EXLOG DQ 03, GDWDW\SH RQH HVVHQWLDOO\
VSHFLILHV WKH OD\RXW RI WKH GDWD LQ WKH W\SH  WKH PHPEHU W\SHV DQG WKHLU UHODWLYH
ORFDWLRQVLQPHPRU\6XFKDW\SHLVFDOOHGD 03,GHULYHGGDWDW\SH,QRUGHUWRVHHKRZ
WKLVZRUNVOHWVZULWHDIXQFWLRQWKDWZLOOEXLOGD03,GHULYHGW\SH
MODULE GLOBAL
type INDATA_TYPE
real a
real b
integer n
end type INDATA_TYPE
END MODULE GLOBAL
subroutine Build_derived_type(indata, mesg_mpi_t)
use GLOBAL
INCLUDE 'mpif.h'
IMPLICIT NONE
type(INDATA_TYPE) indata
integer mesg_mpi_t
integer ierr
integer block_lengths(3)
integer displacements(3)
integer address(4)
integer typelist(3)
C Build a derived datatype consisting of two real and
an integer.
C First specify the types.
typelist(1) = MPI_REAL
typelist(2) = MPI_REAL
typelist(3) = MPI_INTEGER
C Specify the number of elements of each type.
block_lengths(1) = 1
block_lengths(2) = 1
block_lengths(3) = 1
C Calculate the displacements of the members relative
to indata.
call MPI_Address(indata, address(1), ierr)
call MPI_Address(indata%a, address(2), ierr)
call MPI_Address(indata%b, address(3), ierr)
call MPI_Address(indata%n, address(4), ierr)
displacements(1) = address(2) - address(1)
displacements(2) = address(3) - address(1)
displacements(3) = address(4) - address(1)
C Build the derived datatype
call MPI_TYPE_STRUCT(3, block_lengths, displacements,
+ typelist, mesg_mpi_t, ierr)
C Commit it -- tell system we'll be using it for
communication.
call MPI_TYPE_COMMIT(mesg_mpi_t, ierr)
return
end

7KHILUVWWKUHHVWDWHPHQWVVSHFLI\WKHW\SHVRIWKHPHPEHUVRIWKH03,GHULYHGW\SH
DQG WKH QH[W WKUHH VSHFLILHV WKH QXPEHU RI HOHPHQWV RI HDFK W\SH 7KH QH[W IRXU
FDOFXODWHWKHDGGUHVVHVRIWKHWKUHHPHPEHUVRI indata7KHQH[WWKUHHVWDWHPHQWV
XVH WKH FDOFXODWHG DGGUHVVHV WR GHWHUPLQH WKH GLVSODFHPHQWV RI WKH WKUHH PHPEHUV
UHODWLYH WR WKH DGGUHVV RI WKH ILUVW  ZKLFK LV JLYHQ GLVSODFHPHQW  :LWK WKLV
LQIRUPDWLRQ ZH NQRZ WKH W\SHV VL]HV DQG UHODWLYH ORFDWLRQV RI WKH PHPEHUV RI D
YDULDEOHKDYLQJ)RUWUDQW\SH,1'$7$B7<3(DQGKHQFHZHFDQGHILQHDGHULYHG
GDWDW\SHWKDWFRUUHVSRQGVWRWKH)RUWUDQW\SH7KLVLVGRQHE\FDOOLQJWKHIXQFWLRQV
MPI_Type_structDQGMPI_Type_commit
7KH QHZO\ FUHDWHG 03, GDWDW\SH FDQ EH XVHG LQ DQ\ RI WKH 03, FRPPXQLFDWLRQ
IXQFWLRQV,QRUGHUWRXVHLWZHVLPSO\XVHWKHVWDUWLQJDGGUHVVRIDYDULDEOHRIW\SH
,1'$7$B7<3( DV WKH ILUVW DUJXPHQW DQG WKH GHULYHG W\SH LQ WKH GDWDW\SH
DUJXPHQW)RUH[DPSOHZHFRXOGUHZULWHWKH*HWBGDWDIXQFWLRQDVIROORZV
subroutine Get_data3(indata, my_rank)
use global
type(INDATA_TYPE) indata
integer my_rank
integer mesg_mpi_t
integer ierr
include 'mpif.h'
if (my_rank == 0) then
print *, 'Enter a, b, and n'
read *, indata%a, indata%b, indata%n
endif
call Build_derived_type(indata, mesg_mpi_t)
call MPI_BCAST(indata, 1, mesg_mpi_t, 0,
MPI_COMM_WORLD,
+ierr )
 return
end
7R VXPPDUL]H WKHQ ZH FDQ EXLOG JHQHUDO 03, GHULYHG GDWDW\SHV E\ FDOOLQJ
MPI_Type_struct7KHV\QWD[LV
call MPI_TYPE_STRUCT(count, array_of_block_lengths,
+array_of_displacements, array_of_types, newtype,
ierror)
integer count, array_of_block_lengths(*),
integer array_of_displacements(*) , array_of_types(*)
integer array_of_types(*), newtype, ierror

7KHDUJXPHQWcountLVWKHQXPEHURIHOHPHQWVLQWKHGHULYHGW\SH,WLVDOVRWKHVL]H
RI WKH WKUHH DUUD\V array_of_block_lengths array_of_displacements DQG
array_of_types 7KH DUUD\ array_of_block_lengths FRQWDLQV WKH QXPEHU RI
HQWULHV LQ HDFK HOHPHQW RI WKH W\SH 6R LI DQ HOHPHQW RI WKH W\SH LV DQ DUUD\ RI P
YDOXHV WKHQ WKH FRUUHVSRQGLQJ HQWU\ LQ array_of_block_lengths LV P 7KH DUUD\
array_of_displacements FRQWDLQV WKH GLVSODFHPHQW RI HDFK HOHPHQW IURP WKH
EHJLQQLQJRIWKHPHVVDJHDQGWKHDUUD\ array_of_typesFRQWDLQVWKH03,GDWDW\SH
RIHDFKHQWU\7KHDUJXPHQW newtypeUHWXUQVDSRLQWHUWRWKH03,GDWDW\SHFUHDWHG
E\WKHFDOOWRMPI_Type_struct

1RWH DOVR WKDW newtype DQG WKH HQWULHV LQ array_of_types DOO KDYH W\SH
MPI_Datatype 6R MPI_Type_struct FDQ EH FDOOHG UHFXUVLYHO\ WR EXLOG PRUH
FRPSOH[GHULYHGGDWDW\SHV

2WKHU'HULYHG'DWDW\SH&RQVWUXFWRUV
MPI_Type_struct LV WKH PRVW JHQHUDO GDWDW\SH FRQVWUXFWRU LQ 03, DQG DV D
FRQVHTXHQFH WKH XVHU PXVW SURYLGH D FRPSOHWH GHVFULSWLRQ RI HDFK HOHPHQW RI WKH
W\SH,IWKHGDWDWREHWUDQVPLWWHGFRQVLVWVRIDVXEVHWRIWKHHQWULHVLQDQDUUD\ZH
VKRXOGQ
WQHHGWRSURYLGHVXFKGHWDLOHGLQIRUPDWLRQVLQFHDOOWKHHOHPHQWVKDYHWKH
VDPHEDVLFW\SH03,SURYLGHVWKUHHGHULYHGGDWDW\SHFRQVWUXFWRUVIRUGHDOLQJZLWK
WKLV VLWXDWLRQ MPI_Type_Contiguous MPI_Type_vector DQG
MPI_Type_indexed 7KH ILUVW FRQVWUXFWRU EXLOGV D GHULYHG W\SH ZKRVH HOHPHQWV
DUH FRQWLJXRXV HQWULHV LQ DQ DUUD\ 7KH VHFRQG EXLOGV D W\SH ZKRVH HOHPHQWV DUH
HTXDOO\ VSDFHG HQWULHV RI DQ DUUD\ DQG WKH WKLUG EXLOGV D W\SH ZKRVH HOHPHQWV DUH
DUELWUDU\ HQWULHV RI DQ DUUD\ 1RWH WKDW EHIRUH DQ\ GHULYHG W\SH FDQ EH XVHG LQ
FRPPXQLFDWLRQLWPXVWEH FRPPLWWHGZLWKDFDOOWRMPI_Type_commit
'HWDLOVRIWKHV\QWD[RIWKHDGGLWLRQDOW\SHFRQVWUXFWRUVIROORZ
MPI_Type_contiguous(count, oldtype, newtype, ierror)
integer count, oldtype, newtype, ierror

MPI_Type_contiguousFUHDWHVDGHULYHGGDWDW\SHFRQVLVWLQJRI countHOHPHQWVRI
W\SHoldtype7KHHOHPHQWVEHORQJWRFRQWLJXRXVPHPRU\ORFDWLRQV
 MPI_Type_vector(count, block_length, stride,
+element_type, newtype, ierror)
integer count, blocklength, stride oldtype, newtype,
ierror

MPI_Type_vector FUHDWHV D GHULYHG W\SH FRQVLVWLQJ RI count HOHPHQWV (DFK

HOHPHQW FRQWDLQV block_length HQWULHV RI W\SH element_type Stride LV WKH
QXPEHU RI HOHPHQWV RI W\SH element_type EHWZHHQ VXFFHVVLYH HOHPHQWV RI
new_type
MPI_Type_indexed(count, array_of_block_lengths,
+array_of_displacements, element_type, newtype, ierror
integer count, array_of_block_lengths(*),
+array_of_displacements, element_type, newtype, ierror

MPI_Type_indexedFUHDWHVDGHULYHGW\SHFRQVLVWLQJRI countHOHPHQWV7KH LWK

HOHPHQW L    count FRQVLVWV RI array_of_block_lengths[i] HQWULHV RI W\SH
element_type DQG LW LV GLVSODFHG array_of_displacements[i] XQLWV RI W\SH
element_typeIURPWKHEHJLQQLQJRI newtype

3DFN8QSDFN
$Q DOWHUQDWLYH DSSURDFK WR JURXSLQJ GDWD LV SURYLGHG E\ WKH 03, IXQFWLRQV
MPI_Pack DQG MPI_Unpack MPI_Pack DOORZV RQH WR H[SOLFLWO\ VWRUH
QRQFRQWLJXRXVGDWDLQFRQWLJXRXVPHPRU\ORFDWLRQVDQG MPI_UnpackFDQEHXVHG
WR FRS\ GDWD IURP D FRQWLJXRXV EXIIHU LQWR QRQFRQWLJXRXV PHPRU\ ORFDWLRQV ,Q
RUGHUWRVHHKRZWKH\DUHXVHGOHW
VUHZULWH Get_dataRQHODVWWLPH
subroutine Get_data4(a, b, n, my_rank)
real a
real b
integer n
integer my_rank
C
INCLUDE 'mpif.h'
integer ierr
character buffer(100)
integer position
C in the buffer
C
if (my_rank .EQ. 0) then
print *,'Enter a, b, and n'
read *, a, b, n
C
C Now pack the data into buffer. Position = 0
C says start at beginning of buffer.
position = 0
C
C Position is in/out
call MPI_PACK(a, 1, MPI_REAL , buffer, 100,
+ position, MPI_COMM_WORLD, ierr )
C Position has been incremented: it now refer-
C ences the first free location in buffer.
C
call MPI_PACK(b, 1, MPI_REAL , buffer, 100,
+ position, MPI_COMM_WORLD, ierr )
C Position has been incremented again.
C
call MPI_PACK(n, 1, MPI_INTEGER, buffer, 100,
+ position, MPI_COMM_WORLD, ierr )
C Position has been incremented again.
C
C Now broadcast contents of buffer
call MPI_BCAST(buffer, 100, MPI_PACKED, 0,
+ MPI_COMM_WORLD, ierr )
else
call MPI_BCAST(buffer, 100, MPI_PACKED, 0,
+ MPI_COMM_WORLD, ierr )
C
C Now unpack the contents of buffer
position = 0
call MPI_UNPACK(buffer, 100, position, a, 1,
+ MPI_REAL , MPI_COMM_WORLD, ierr )
C Once again position has been incremented:
C it now references the beginning of b.
C
call MPI_UNPACK(buffer, 100, position, b, 1,
+ MPI_REAL , MPI_COMM_WORLD, ierr )
call MPI_UNPACK(buffer, 100, position, n, 1,
+ MPI_INTEGER, MPI_COMM_WORLD, ierr )
endif
return
end
C

,Q WKLV YHUVLRQ RI Get_data SURFHVV  XVHV MPI_Pack WR FRS\ a WR buffer DQG
WKHQ DSSHQG b DQG n $IWHU WKH EURDGFDVW RI buffer WKH UHPDLQLQJ SURFHVVHV XVH
MPI_Unpack WR VXFFHVVLYHO\ H[WUDFW a b DQG n IURP buffer 1RWH WKDW WKH
GDWDW\SHIRUWKHFDOOVWR MPI_BcastLVMPI_PACKED

7KHV\QWD[RIMPI_PackLV
MPI_Pack( pack_data, in_count, datatype, buffer, size,
+position_ptr, comm, ierror)
<type> pack_data(*), buffer(*)
integer in_count, datatype, size, position_ptr, comm,
ierror

7KHSDUDPHWHU pack_dataUHIHUHQFHVWKHGDWDWREHEXIIHUHG,WVKRXOGFRQVLVWRI
in_countHOHPHQWVHDFKKDYLQJW\SH datatype7KHSDUDPHWHU position_ptr LVDQ
LQRXW SDUDPHWHU 2Q LQSXW WKH GDWD UHIHUHQFHG E\ pack_data LV FRSLHG LQWR
PHPRU\ VWDUWLQJ DW DGGUHVV buffer + position_ptr 2Q UHWXUQ position_ptr
UHIHUHQFHVWKHILUVWORFDWLRQLQ bufferDIWHUWKHGDWDWKDWZDVFRSLHG7KHSDUDPHWHU
sizeFRQWDLQVWKHVL]H LQE\WHVRIWKHPHPRU\UHIHUHQFHGE\ bufferDQGcommLVWKH
FRPPXQLFDWRUWKDWZLOOEHXVLQJ buffer

7KHV\QWD[RIMPI_UnpackLV
MPI_Unpack(buffer, size, position_ptr, unpack_data,
+count, datatype, comm, ierror)
 <type> inbuf(*), outbuf(*)
integer insize, position, outcount, datatype, comm,
ierror

7KHSDUDPHWHU bufferUHIHUHQFHVWKHGDWDWREHXQSDFNHG,WFRQVLVWVRI sizeE\WHV

7KHSDUDPHWHUposition_ptrLVRQFHDJDLQDQ LQRXWSDUDPHWHU:KHQMPI_Unpack
LV FDOOHG WKH GDWD VWDUWLQJ DW DGGUHVV buffer + position_ptr LV FRSLHG LQWR WKH
PHPRU\UHIHUHQFHGE\ unpack_data2QUHWXUQ position_ptrUHIHUHQFHVWKHILUVW
ORFDWLRQLQbufferDIWHUWKHGDWDWKDWZDVMXVWFRSLHG MPI_UnpackZLOOFRS\count
HOHPHQWV KDYLQJ W\SH datatype LQWR unpack_data 7KH FRPPXQLFDWRU DVVRFLDWHG
ZLWKbufferLVcomm

'HFLGLQJ:KLFK0HWKRGWR8VH
,IWKHGDWDWREHVHQWLVVWRUHGLQFRQVHFXWLYHHQWULHVRIDQDUUD\WKHQRQHVKRXOGVLPSO\XVH
WKH count DQG datatype DUJXPHQWV WR WKH FRPPXQLFDWLRQ IXQFWLRQV 7KLV DSSURDFK
LQYROYHVQRDGGLWLRQDORYHUKHDGLQWKHIRUPRIFDOOVWRGHULYHGGDWDW\SHFUHDWLRQIXQFWLRQVRU
FDOOVWRMPI_Pack/MPI_Unpack
,IWKHUHDUHDODUJHQXPEHURIHOHPHQWVWKDWDUHQRWLQFRQWLJXRXVPHPRU\ORFDWLRQVWKHQ
EXLOGLQJDGHULYHGW\SHZLOOSUREDEO\LQYROYHOHVVRYHUKHDGWKDQDODUJHQXPEHURIFDOOVWR
MPI_Pack/MPI_Unpack
,I WKH GDWD DOO KDYH WKH VDPH W\SH DQG DUH VWRUHG DW UHJXODU LQWHUYDOV LQ PHPRU\ HJ D
FROXPQRIDPDWUL[WKHQLWZLOODOPRVWFHUWDLQO\EHPXFKHDVLHUDQGIDVWHUWRXVHDGHULYHG
GDWDW\SHWKDQLWZLOOEHWRXVH MPI_Pack/MPI_Unpack)XUWKHUPRUHLIWKHGDWDDOOKDYH
WKHVDPHW\SHEXWDUHVWRUHGLQLUUHJXODUO\VSDFHGORFDWLRQVLQPHPRU\LWZLOOVWLOOSUREDEO\
EHHDVLHUDQGPRUHHIILFLHQWWRFUHDWHDGHULYHGW\SHXVLQJ MPI_Type_indexed)LQDOO\LI
WKHGDWDDUHKHWHURJHQHRXVDQGRQHLVUHSHDWHGO\VHQGLQJWKHVDPHFROOHFWLRQRIGDWDHJ
URZQXPEHUFROXPQQXPEHUPDWUL[HQWU\WKHQLWZLOOEHEHWWHUWRXVHDGHULYHGW\SHVLQFH
WKHRYHUKHDGRIFUHDWLQJWKHGHULYHGW\SHLVLQFXUUHGRQO\RQFHZKLOHWKHRYHUKHDGRIFDOOLQJ
MPI_Pack/MPI_UnpackPXVWEHLQFXUUHGHYHU\WLPHWKHGDWDLVFRPPXQLFDWHG
7KLVOHDYHVWKHFDVHZKHUHRQHLVVHQGLQJKHWHURJHQHRXVGDWDRQO\RQFHRUYHU\IHZWLPHV
,QWKLVFDVHLWPD\EHDJRRGLGHDWRFROOHFWVRPHLQIRUPDWLRQRQWKHFRVWRIGHULYHGW\SH
FUHDWLRQ DQG SDFNLQJXQSDFNLQJ WKH GDWD )RU H[DPSOH RQ DQ Q&8%(  UXQQLQJ WKH
03,&+ LPSOHPHQWDWLRQ RI 03, LW WDNHV DERXW  PLOOLVHFRQGV WR FUHDWH WKH GHULYHG W\SH
XVHGLQ Get_data3ZKLOHLWRQO\WDNHVDERXWPLOOLVHFRQGVWRSDFNRUXQSDFNWKHGDWDLQ
Get_data42IFRXUVHWKHVDYLQJLVQ
WDVJUHDWDVLWVHHPVEHFDXVHRIWKHDV\PPHWU\LQWKH
SDFNXQSDFNSURFHGXUH7KDWLVZKLOHSURFHVVSDFNVWKHGDWDWKHRWKHUSURFHVVHVDUHLGOH
DQGWKHHQWLUHIXQFWLRQZRQ
WFRPSOHWHXQWLOERWKWKHSDFNDQGXQSDFNDUHH[HFXWHG6RWKH
FRVWUDWLRLVSUREDEO\PRUHOLNHWKDQ

7KHUHDUHDOVRDFRXSOHRIVLWXDWLRQVLQZKLFKWKHXVHRI MPI_PackDQG MPI_UnpackLV

SUHIHUUHG1RWHILUVWWKDWLWPD\EHSRVVLEOHWRDYRLGWKHXVHRI V\VWHPEXIIHULQJZLWKSDFN
VLQFH WKH GDWD LV H[SOLFLWO\ VWRUHG LQ D XVHUGHILQHG EXIIHU 7KH V\VWHP FDQ H[SORLW WKLV E\
QRWLQJ WKDW WKH PHVVDJH GDWDW\SH LV MPI_PACKED $OVR QRWH WKDW WKH XVHU FDQ VHQG
YDULDEOHOHQJWK
 PHVVDJHV E\ SDFNLQJ WKH QXPEHU RI HOHPHQWV DW WKH EHJLQQLQJ RI WKH
EXIIHU)RUH[DPSOHVXSSRVHZHZDQWWRVHQGURZVRIDVSDUVHPDWUL[,IZHKDYHVWRUHGD
URZ DV D SDLU RI DUUD\V  RQH FRQWDLQLQJ WKH FROXPQ VXEVFULSWV DQG RQH FRQWDLQLQJ WKH
FRUUHVSRQGLQJPDWUL[HQWULHVZHFRXOGVHQGDURZIURPSURFHVVWRSURFHVVDVIROORZV
PROGRAM SpaRow
INCLUDE 'mpif.h'
integer HUGE
parameter (HUGE = 100)
integer p
integer my_rank
real entries(10)
integer column_subscripts(10)
integer nonzeroes
integer position
integer row_number
character buffer *100
integer status(MPI_STATUS_SIZE)
integer ierr
integer i
data nonzeroes /10/
C
call MPI_INIT( ierr)
call MPI_COMM_SIZE(MPI_COMM_WORLD, p, ierr )
call MPI_COMM_RANK(MPI_COMM_WORLD, my_rank, ierr )
C
if (my_rank .EQ. 0) then
C Get the number of nonzeros in the row.
C Initialize entries and column_subscripts
do i = 1, nonzeroes
entries(i) = 2*i
column_subscripts(i) = 3*i
enddo
C
C Now pack the data and send
position = 1
call MPI_PACK( nonzeroes, 1, MPI_INTEGER, buffer, HUGE,
+ position, MPI_COMM_WORLD, ierr )
call MPI_PACK( row_number, 1, MPI_INTEGER, buffer, HUGE,
+ position, MPI_COMM_WORLD, ierr )
call MPI_PACK(entries, nonzeroes, MPI_REAL , buffer,
+ HUGE, position, MPI_COMM_WORLD, ierr )
call MPI_PACK(column_subscripts,nonzeroes,MPI_INTEGER,
+ buffer, HUGE, position, MPI_COMM_WORLD, ierr )
call MPI_SEND(buffer, position, MPI_PACKED, 1, 0,
+ MPI_COMM_WORLD, ierr )
else
call MPI_RECV(buffer, HUGE, MPI_PACKED, 0, 0,
+ MPI_COMM_WORLD, status, ierr )
position = 1
call MPI_UNPACK(buffer, HUGE, position, nonzeroes,
+ 1, MPI_INTEGER, MPI_COMM_WORLD, ierr )
call MPI_UNPACK(buffer, HUGE, position, row_number,
+ 1, MPI_INTEGER, MPI_COMM_WORLD, ierr )
call MPI_UNPACK(buffer,HUGE, position, entries,
+ nonzeroes, MPI_REAL , MPI_COMM_WORLD, ierr )
call MPI_UNPACK(buffer, HUGE, position,
+ column_subscripts,
+ nonzeroes, MPI_INTEGER, MPI_COMM_WORLD, ierr )
do i = 1, nonzeroes
print *, entries(i), column_subscripts(i)
enddo
endif
C
call MPI_FINALIZE(ierr)
 end


 &RPPXQLFDWRUVDQG
7RSRORJLHV
7KHXVHRIFRPPXQLFDWRUVDQGWRSRORJLHVPDNHV03,GLIIHUHQWIURPPRVWRWKHUPHVVDJH
SDVVLQJV\VWHPV5HFROOHFWWKDWORRVHO\VSHDNLQJDFRPPXQLFDWRULVDFROOHFWLRQRISURFHVVHV
WKDWFDQVHQGPHVVDJHVWRHDFKRWKHU$WRSRORJ\LVDVWUXFWXUHLPSRVHGRQWKHSURFHVVHVLQ
D FRPPXQLFDWRU WKDW DOORZV WKH SURFHVVHV WR EH DGGUHVVHG LQ GLIIHUHQW ZD\V ,Q RUGHU WR
LOOXVWUDWHWKHVHLGHDVZHZLOOGHYHORSFRGHWRLPSOHPHQW)R[
VDOJRULWKPIRUPXOWLSO\LQJWZR
VTXDUHPDWULFHV

)R[
V$OJRULWKP
:HDVVXPHWKDWWKHIDFWRUPDWULFHV$ D DQG %  E  KDYH RUGHU Q :HDOVR DVVXPH WKDW WKH
LM LM

QXPEHURISURFHVVHV SLVDSHUIHFWVTXDUHZKRVHVTXDUHURRWHYHQO\GLYLGHV Q6D\S T DQG

QT,Q)R[
VDOJRULWKPWKHIDFWRUPDWULFHVDUHSDUWLWLRQHGDPRQJWKHSURFHVVHVLQD EORFNFKHFNHUERDUG
IDVKLRQ 6R ZH YLHZ RXU SURFHVVHV DV D YLUWXDO WZRGLPHQVLRQDO T [ T JULG DQG HDFK SURFHVV LV
DVVLJQHGDQ [ VXEPDWUL[RIHDFKRIWKHIDFWRUPDWULFHV0RUHIRUPDOO\ZHKDYHDPDSSLQJ

WKDWLVERWKRQHWRRQHDQGRQWR7KLVGHILQHVRXUJULGRISURFHVVHVSURFHVV LEHORQJVWRWKHURZ
DQGFROXPQJLYHQE\ )XUWKHUWKHSURFHVVZLWKUDQN LVDVVLJQHGWKHVXEPDWULFHV


DQG

)RUH[DPSOHLIS  [[PRGDQGQ WKHQ$ZRXOGEHSDUWLWLRQHGDVIROORZV

3URFHVV 3URFHVV 3URFHVV

3URFHVV 3URFHVV 3URFHVV

 3URFHVV 3URFHVV 3URFHVV

,Q )R[
V DOJRULWKP WKH EORFN VXEPDWULFHV $  DQG %  V      T   DUH PXOWLSOLHG DQG
UV VW

DFFXPXODWHGRQSURFHVV 7KHEDVLFDOJRULWKPLV
do step = 0, q - 1
1. Choose a submatrix of A from each row of processes.
2. In each row of processes broadcast the submatrix
chosen in that row to the other processes in that row.
3. On each process, multiply the newly received submatrix
of A by the submatrix of B currently residing on the
process.
4. On each process, send the submatrix of B to the
process directly above. (On processes in the first
row, send the submatrix to the last row.)
enddo
7KHVXEPDWUL[FKRVHQLQWKHU URZLV$ ZKHUH
WK

UX

X UstepPRGT

&RPPXQLFDWRUV
,IZHWU\WRLPSOHPHQW)R[
VDOJRULWKPLWEHFRPHVDSSDUHQWWKDWRXUZRUNZLOOEHJUHDWO\IDFLOLWDWHG
LIZHFDQWUHDWFHUWDLQVXEVHWVRISURFHVVHVDVDFRPPXQLFDWLRQXQLYHUVHDWOHDVWRQDWHPSRUDU\
EDVLV)RUH[DPSOHLQWKHSVHXGRFRGH
1. In each row of processes broadcast the submatrix
chosen in that row to the other processes in that row.
LW ZRXOG EH XVHIXO WR WUHDW HDFK URZ RI SURFHVVHV DV D FRPPXQLFDWLRQ XQLYHUVH ZKLOH LQ WKH
VWDWHPHQW
1. On each process, send the submatrix of B to the
process directly above. (On processes in the first
row, send the submatrix to the last row.)
LWZRXOGEHXVHIXOWRWUHDWHDFKFROXPQRISURFHVVHVDVDFRPPXQLFDWLRQXQLYHUVH
7KHPHFKDQLVPWKDW03,SURYLGHVIRUWUHDWLQJDVXEVHWRISURFHVVHVDVDFRPPXQLFDWLRQ
XQLYHUVH
LV WKH FRPPXQLFDWRU 8S WR QRZ ZH
YH EHHQ ORRVHO\ GHILQLQJ D FRPPXQLFDWRU DV D FROOHFWLRQ RI
SURFHVVHVWKDWFDQVHQGPHVVDJHVWRHDFKRWKHU+RZHYHUQRZWKDWZHZDQWWRFRQVWUXFWRXURZQ
FRPPXQLFDWRUVZHZLOOQHHGDPRUHFDUHIXOGLVFXVVLRQ
,Q03,WKHUHDUHWZRW\SHVRIFRPPXQLFDWRUV LQWUDFRPPXQLFDWRUVDQGLQWHUFRPPXQLFDWRUV
,QWUDFRPPXQLFDWRUVDUHHVVHQWLDOO\DFROOHFWLRQRISURFHVVHVWKDWFDQVHQGPHVVDJHVWRHDFKRWKHU
DQG HQJDJH LQ FROOHFWLYH FRPPXQLFDWLRQ RSHUDWLRQV )RU H[DPSOH MPI_COMM_WORLD LV DQ
LQWUDFRPPXQLFDWRU DQG ZH ZRXOG OLNH IRU HDFK URZ DQG HDFK FROXPQ RI SURFHVVHV LQ )R[
V
DOJRULWKPWRIRUPDQLQWUDFRPPXQLFDWRU,QWHUFRPPXQLFDWRUVDVWKHQDPHLPSOLHVDUHXVHGIRU
VHQGLQJ PHVVDJHV EHWZHHQ SURFHVVHV EHORQJLQJ WR GLVMRLQW LQWUDFRPPXQLFDWRUV )RU H[DPSOH DQ
LQWHUFRPPXQLFDWRU ZRXOG EH XVHIXO LQ DQ HQYLURQPHQW WKDW DOORZHG RQH WR G\QDPLFDOO\ FUHDWH
SURFHVVHVDQHZO\FUHDWHGVHWRISURFHVVHVWKDWIRUPHGDQLQWUDFRPPXQLFDWRUFRXOGEHOLQNHGWR
WKHRULJLQDOVHWRISURFHVVHVHJ MPI_COMM_WORLDE\DQLQWHUFRPPXQLFDWRU:HZLOORQO\
GLVFXVVLQWUDFRPPXQLFDWRUV7KHLQWHUHVWHGUHDGHULVUHIHUUHGWR>@IRUGHWDLOVRQWKHXVHRILQWHU
FRPPXQLFDWRUV
$PLQLPDOLQWUDFRPPXQLFDWRULVFRPSRVHGRI

D*URXSDQG

D&RQWH[W
$JURXSLVDQRUGHUHGFROOHFWLRQRISURFHVVHV,IDJURXSFRQVLVWVRI SSURFHVVHVHDFKSURFHVVLQWKH
JURXSLVDVVLJQHGDXQLTXH UDQNZKLFKLVMXVWDQRQQHJDWLYHLQWHJHULQWKHUDQJH S$
FRQWH[WFDQEHWKRXJKWRIDVDV\VWHPGHILQHGWDJWKDWLVDWWDFKHGWRDJURXS6RWZRSURFHVVHVWKDW
EHORQJWRWKHVDPHJURXSDQGWKDWXVHWKHVDPHFRQWH[WFDQFRPPXQLFDWH7KLVSDLULQJRIDJURXS
ZLWK D FRQWH[W LV WKH PRVW EDVLF IRUP RI D FRPPXQLFDWRU 2WKHU GDWD FDQ EH DVVRFLDWHG WR D
FRPPXQLFDWRU ,Q SDUWLFXODU D VWUXFWXUH RU WRSRORJ\ FDQ EH LPSRVHG RQ WKH SURFHVVHV LQ D
FRPPXQLFDWRUDOORZLQJDPRUHQDWXUDODGGUHVVLQJVFKHPH:H
OOGLVFXVVWRSRORJLHVLQVHFWLRQ

:RUNLQJ ZLWK *URXSV &RQWH[WV DQG

&RPPXQLFDWRUV
7R LOOXVWUDWH WKH EDVLFV RI ZRUNLQJ ZLWK FRPPXQLFDWRUV OHW
V FUHDWH D FRPPXQLFDWRU ZKRVH
XQGHUO\LQJ JURXS FRQVLVWV RI WKH SURFHVVHV LQ WKH ILUVW URZ RI RXU YLUWXDO JULG 6XSSRVH WKDW
MPI_COMM_WORLDFRQVLVWVRISSURFHVVHVZKHUHT  S/HW
VDOVRVXSSRVHWKDW

 [T
[PRGT6RWKHILUVWURZRISURFHVVHVFRQVLVWVRIWKHSURFHVVHVZLWKUDQNV T+HUHWKH
UDQNVDUHLQ03,B&200B:25/',QRUGHUWRFUHDWHWKHJURXSRIRXUQHZFRPPXQLFDWRUZH
FDQH[HFXWHWKHIROORZLQJFRGH
PROGRAM ComCrt
INCLUDE 'mpif.h'
IMPLICIT NONE
integer, parameter :: MAX_PROCS = 100
integer p
real p_real
integer q
integer my_rank
integer MPI_GROUP_WORLD
integer first_row_group
integer first_row_comm
integer process_ranks(0:MAX_PROCS-1)
integer proc
integer test
integer sum
integer my_rank_in_first_row
integer ierr
C
C
test = 0
call MPI_INIT( ierr)
call MPI_COMM_SIZE(MPI_COMM_WORLD, p, ierr )
call MPI_COMM_RANK(MPI_COMM_WORLD, my_rank, ierr )
C
p_real = p
q = sqrt(p_real)
C
C Make a list of the processes in the new communicator.
do proc = 0, q-1
process_ranks(proc) = proc
enddo
C
C Get the group underlying MPI_COMM_WORLD
call MPI_COMM_GROUP(MPI_COMM_WORLD, MPI_GROUP_WORLD, ierr )
C
C Create the new group
call MPI_GROUP_INCL(MPI_GROUP_WORLD, q, process_ranks,
+ first_row_group, ierr)
C
C Create the new communicator
call MPI_COMM_CREATE(MPI_COMM_WORLD, first_row_group,
+ first_row_comm, ierr)
C
7KLV FRGH SURFHHGV LQ D IDLUO\ VWUDLJKWIRUZDUG IDVKLRQ WR EXLOG WKH QHZ FRPPXQLFDWRU )LUVW LW
FUHDWHV D OLVW RI WKH SURFHVVHV WR EH DVVLJQHG WR WKH QHZ FRPPXQLFDWRU 7KHQ LW FUHDWHV D JURXS
FRQVLVWLQJ RI WKHVH SURFHVVHV 7KLV UHTXLUHG WZR FRPPDQGV ILUVW JHW WKH JURXS DVVRFLDWHG ZLWK
MPI_COMM_WORLDVLQFHWKLVLVWKHJURXSIURPZKLFKWKHSURFHVVHVLQWKHQHZJURXSZLOOEH
WDNHQ WKHQ FUHDWH WKH JURXS ZLWK MPI_Group_incl )LQDOO\ WKH DFWXDO FRPPXQLFDWRU LV FUHDWHG
ZLWKDFDOOWRMPI_Comm_create7KHFDOOWRMPI_Comm_createLPSOLFLWO\DVVRFLDWHVDFRQWH[W
ZLWK WKH QHZ JURXS 7KH UHVXOW LV WKH FRPPXQLFDWRU first_row_comm 1RZ WKH SURFHVVHV LQ
first_row_comm FDQ SHUIRUP FROOHFWLYH FRPPXQLFDWLRQ RSHUDWLRQV )RU H[DPSOH SURFHVV  LQ
first_row_groupFDQEURDGFDVW$ WRWKHRWKHUSURFHVVHVLQ first_row_group


integer my_rank_in_first_row
real, allocatable , dimension( :,: ) :: A_00
if (my_rank < q) then
call MPI_COMM_RANK(first_row_comm,
+ my_rank_in_first_row, ierr)
! Allocate space for A_00, order n_bar.
allocate (A_00(n_bar,n_bar))
if (my_rank_in_first_row == 0) then
! initialize A_00
endif
call MPI_BCAST( A_00, n_bar*n_bar, MPI_INTEGER, 0,
+ first_row_comm, ierr)
endif
*URXSVDQGFRPPXQLFDWRUVDUH RSDTXHREMHFWV)URPDSUDFWLFDOVWDQGSRLQWWKLVPHDQVWKDWWKHGHWDLOV
RI WKHLU LQWHUQDO UHSUHVHQWDWLRQ GHSHQG RQ WKH SDUWLFXODU LPSOHPHQWDWLRQ RI 03, DQG DV D
FRQVHTXHQFH WKH\ FDQQRW EH GLUHFWO\ DFFHVVHG E\ WKH XVHU 5DWKHU WKH XVHU DFFHVVHV D KDQGOH WKDW
UHIHUHQFHVWKHRSDTXHREMHFWDQGWKHRSDTXHREMHFWVDUHPDQLSXODWHGE\VSHFLDO03,IXQFWLRQVIRU
H[DPSOHMPI_Comm_createMPI_Group_inclDQGMPI_Comm_group
&RQWH[WV DUH QRW H[SOLFLWO\ XVHG LQ DQ\ 03, IXQFWLRQV 5DWKHU WKH\ DUH LPSOLFLWO\ DVVRFLDWHG ZLWK
JURXSV ZKHQ FRPPXQLFDWRUV DUH FUHDWHG 7KH V\QWD[ RI WKH FRPPDQGV ZH XVHG WR FUHDWH
first_row_commLVIDLUO\VHOIH[SODQDWRU\7KHILUVWFRPPDQG
MPI_Comm_group(comm, group, ierror)
integer comm, group, ierror

VLPSO\UHWXUQVWKHJURXSXQGHUO\LQJWKHFRPPXQLFDWRU comm
7KHVHFRQGFRPPDQG
MPI_Group_incl(old_group, new_group_size,
+ranks_in_old_group, new_group, ierror)
integer old_group, new_group_size,
integer ranks_in_old_group(*), new_group, ierror

FUHDWHV D QHZ JURXS IURP D OLVW RI SURFHVVHV LQ WKH H[LVWLQJ JURXS old_group 7KH QXPEHU RI
SURFHVVHV LQ WKH QHZ JURXS LV new_group_size DQG WKH SURFHVVHV WR EH LQFOXGHG DUH OLVWHG LQ
ranks_in_old_group3URFHVVLQnew_groupKDVUDQNranks_in_old_group(0)LQold_group
SURFHVVLQnew_groupKDVUDQNranks_in_old_group(1)LQold_groupHWF
7KHILQDOFRPPDQG
MPI_Comm_create(old_comm, new_group, new_comm, ierror)
integer old_comm, new_group, new_comm, ierror

DVVRFLDWHVDFRQWH[WZLWKWKHJURXS new_groupDQGFUHDWHVWKHFRPPXQLFDWRU new_comm$OORI

WKHSURFHVVHVLQnew_groupEHORQJWRWKHJURXSXQGHUO\LQJ old_comm
7KHUH LV DQ H[WUHPHO\ LPSRUWDQW GLVWLQFWLRQ EHWZHHQ WKH ILUVW WZR IXQFWLRQV DQG WKH WKLUG
MPI_Comm_group DQG MPI_Group_incl DUH ERWK ORFDO RSHUDWLRQV 7KDW LV WKHUH LV QR
FRPPXQLFDWLRQDPRQJSURFHVVHVLQYROYHGLQWKHLUH[HFXWLRQ+RZHYHU MPI_Comm_create LVD
FROOHFWLYHRSHUDWLRQ $OOWKHSURFHVVHVLQ old_commPXVWFDOOMPI_Comm_createZLWKWKHVDPH
DUJXPHQWV7KH6WDQGDUG>@JLYHVWKUHHUHDVRQVIRUWKLV

 ,W DOORZV WKH LPSOHPHQWDWLRQ WR OD\HU MPI_Comm_create RQ WRS RI UHJXODU FROOHFWLYH
FRPPXQLFDWLRQV
 ,WSURYLGHVDGGLWLRQDOVDIHW\
 ,WSHUPLWVLPSOHPHQWDWLRQVWRDYRLGFRPPXQLFDWLRQUHODWHGWRFRQWH[WFUHDWLRQ

1RWHWKDWVLQFH MPI_Comm_createLVFROOHFWLYHLWZLOOEHKDYHLQWHUPVRIWKHGDWDWUDQVPLWWHG
DVLILWV\QFKURQL]HV,QSDUWLFXODULIVHYHUDOFRPPXQLFDWRUVDUHEHLQJFUHDWHGWKH\PXVWEHFUHDWHG
LQWKHVDPHRUGHURQDOOWKHSURFHVVHV

03,B&RPPBVSOLW
,QRXUPDWUL[PXOWLSOLFDWLRQSURJUDPZHQHHGWRFUHDWHPXOWLSOHFRPPXQLFDWRUV
RQHIRUHDFKURZRISURFHVVHVDQGRQHIRUHDFKFROXPQ7KLVZRXOGEHDQH[WUHPHO\
WHGLRXVSURFHVVLI SZHUHODUJH DQG ZH KDG WR FUHDWH HDFK FRPPXQLFDWRU XVLQJ WKH
WKUHH IXQFWLRQV GLVFXVVHG LQ WKH SUHYLRXV VHFWLRQ )RUWXQDWHO\ 03, SURYLGHV D
IXQFWLRQ MPI_Comm_splitWKDWFDQFUHDWHVHYHUDOFRPPXQLFDWRUVVLPXOWDQHRXVO\
$VDQH[DPSOHRILWVXVHZH
OOFUHDWHRQHFRPPXQLFDWRUIRUHDFKURZRISURFHVVHV
integer my_row_comm
 integer my_row
C my_rank is rank in MPI_COMM_WORLD.
C q*q = p
my_row = my_rank/q
call MPI_COMM_SPLIT(MPI_COMM_WORLD, my_row, my_rank,
+ my_row_comm, ierr)


7KH VLQJOH FDOO WR MPI_Comm_split FUHDWHV T QHZ FRPPXQLFDWRUV DOO RI WKHP
KDYLQJ WKH VDPH QDPH my_row_comm )RU H[DPSOH LI S   WKH JURXS
XQGHUO\LQJmy_row_commZLOOFRQVLVWRIWKHSURFHVVHVDQGRQSURFHVVHV
 DQG  2Q SURFHVVHV   DQG  WKH JURXS XQGHUO\LQJ my_row_comm ZLOO
FRQVLVWRIWKHSURFHVVHVDQGDQG RQSURFHVVHVDQGLWZLOOFRQVLVWRI
SURFHVVHVDQG

7KHV\QWD[RIMPI_Comm_splitLV
MPI_COMM_SPLIT(old_comm, split_key, rank_key,
+ new_comm, ierror)
integer old_comm, split_key, rank_key, new_comm,
ierror

,WFUHDWHVDQHZFRPPXQLFDWRUIRUHDFKYDOXHRI split_key3URFHVVHVZLWKWKHVDPH
YDOXHRI split_keyIRUPDQHZJURXS7KHUDQNLQWKHQHZJURXSLVGHWHUPLQHGE\
WKHYDOXHRIrank_key,ISURFHVV$DQGSURFHVV %FDOOMPI_Comm_splitZLWKWKH
VDPHYDOXHRI split_keyDQGWKH rank_keyDUJXPHQWSDVVHG E\ SURFHVV $ LV OHVV
WKDQ WKDW SDVVHG E\ SURFHVV % WKHQ WKH UDQN RI $ LQ WKH JURXS XQGHUO\LQJ
new_commZLOOEHOHVVWKDQWKHUDQNRISURFHVV %,IWKH\FDOOWKHIXQFWLRQZLWKWKH
VDPH YDOXH RI rank_key WKH V\VWHP ZLOO DUELWUDULO\ DVVLJQ RQH RI WKH SURFHVVHV D
ORZHUUDQN

MPI_Comm_splitLVDFROOHFWLYHFDOODQGLWPXVWEHFDOOHGE\DOOWKHSURFHVVHVLQ
old_comm7KHIXQFWLRQFDQEHXVHGHYHQLIWKHXVHUGRHVQ
WZLVKWRDVVLJQHYHU\
SURFHVVWRDQHZFRPPXQLFDWRU7KLVFDQEHDFFRPSOLVKHGE\SDVVLQJWKHSUHGHILQHG
FRQVWDQW MPI_UNDEFINEDDVWKH split_keyDUJXPHQW3URFHVVHVGRLQJWKLVZLOO
KDYHWKHSUHGHILQHGYDOXH MPI_COMM_NULLUHWXUQHGLQnew_comm

7RSRORJLHV
5HFROOHFWWKDWLWLVSRVVLEOHWRDVVRFLDWHDGGLWLRQDOLQIRUPDWLRQLQIRUPDWLRQEH\RQGWKHJURXSDQG
FRQWH[W  ZLWK D FRPPXQLFDWRU 7KLV DGGLWLRQDO LQIRUPDWLRQ LV VDLG WR EH FDFKHG ZLWK WKH
FRPPXQLFDWRU DQG RQH RI WKH PRVW LPSRUWDQW SLHFHV RI LQIRUPDWLRQ WKDW FDQ EH FDFKHG ZLWK D
FRPPXQLFDWRU LV D WRSRORJ\ ,Q 03, D WRSRORJ\ LV MXVW D PHFKDQLVP IRU DVVRFLDWLQJ GLIIHUHQW
DGGUHVVLQJVFKHPHVZLWKWKHSURFHVVHVEHORQJLQJWRDJURXS 1RWHWKDW03, WRSRORJLHV DUH YLUWXDO
WRSRORJLHV  WKHUH PD\ EH QR VLPSOH UHODWLRQ EHWZHHQ WKH SURFHVV VWUXFWXUH GHILQHG E\ D YLUWXDO
WRSRORJ\DQGWKHDFWXDOXQGHUO\LQJSK\VLFDOVWUXFWXUHRIWKHSDUDOOHOPDFKLQH
7KHUHDUHHVVHQWLDOO\WZRW\SHVRIYLUWXDOWRSRORJLHVWKDWFDQEHFUHDWHGLQ03,D FDUWHVLDQRUJULG
WRSRORJ\ DQG D JUDSK WRSRORJ\ &RQFHSWXDOO\ WKH IRUPHU LV VXEVXPHG E\ WKH ODWWHU +RZHYHU
EHFDXVHRIWKHLPSRUWDQFHRIJULGVLQDSSOLFDWLRQVWKHUHLVDVHSDUDWHFROOHFWLRQRIIXQFWLRQVLQ03,
ZKRVHSXUSRVHLVWKHPDQLSXODWLRQRIYLUWXDOJULGV

,Q )R[
V DOJRULWKP ZH ZLVK WR LGHQWLI\ WKH SURFHVVHV LQ MPI_COMM_WORLD ZLWK WKH
FRRUGLQDWHV RI D VTXDUH JULG DQG HDFK URZ DQG HDFK FROXPQ RI WKH JULG QHHGV WR IRUP LWV RZQ
FRPPXQLFDWRU/HW
VORRNDWRQHPHWKRGIRUEXLOGLQJWKLVVWUXFWXUH

:HEHJLQE\DVVRFLDWLQJDVTXDUHJULGVWUXFWXUHZLWK MPI_COMM_WORLD,QRUGHUWRGRWKLV
ZHQHHGWRVSHFLI\WKHIROORZLQJLQIRUPDWLRQ
 7KHQXPEHURIGLPHQVLRQVLQWKHJULG:HKDYH
 7KHVL]HRIHDFKGLPHQVLRQ,QRXUFDVHWKLVLVMXVWWKHQXPEHURIURZVDQGWKHQXPEHURI
FROXPQV:HKDYHTURZVDQGTFROXPQV
 7KHSHULRGLFLW\RIHDFKGLPHQVLRQ,QRXUFDVHWKLVLQIRUPDWLRQVSHFLILHVZKHWKHUWKHILUVW
HQWU\ LQ HDFK URZ RU FROXPQ LV DGMDFHQW WR WKH ODVW HQWU\ LQ WKDW URZ RU FROXPQ
UHVSHFWLYHO\6LQFHZHZDQWDFLUFXODU
VKLIWRIWKHVXEPDWULFHVLQHDFKFROXPQZHZDQWWKH
VHFRQGGLPHQVLRQWREHSHULRGLF,W
VXQLPSRUWDQWZKHWKHUWKHILUVWGLPHQVLRQLVSHULRGLF
 )LQDOO\03,JLYHVWKHXVHUWKHRSWLRQRIDOORZLQJWKHV\VWHPWRRSWLPL]HWKHPDSSLQJRIWKH
JULGRISURFHVVHVWRWKHXQGHUO\LQJSK\VLFDOSURFHVVRUVE\SRVVLEO\UHRUGHULQJWKHSURFHVVHV
LQWKHJURXSXQGHUO\LQJWKHFRPPXQLFDWRU6LQFHZHGRQ
WQHHGWRSUHVHUYHWKHRUGHULQJRI
WKHSURFHVVHVLQMPI_COMM_WORLDZHVKRXOGDOORZWKHV\VWHPWRUHRUGHU
+DYLQJPDGHDOOWKHVHGHFLVLRQVZHVLPSO\H[HFXWHWKHIROORZLQJFRGH
integer grid_comm
integer dim_sizes(0:1)
logical wrap_around(0:1)
logical reorder = .TRUE.
dim_sizes(0) = q
dim_sizes(1) = q
wrap_around(0) = .TRUE.
wrap_around(1) = .TRUE.
call MPI_CART_CREATE(MPI_COMM_WORLD, 2, dim_sizes,
+ wrap_around, reorder, grid_comm, ierr)

$IWHU H[HFXWLQJ WKLV FRGH WKH FRPPXQLFDWRU grid_comm ZLOO FRQWDLQ DOO WKH SURFHVVHV LQ
MPI_COMM_WORLD SRVVLEO\ UHRUGHUHG DQG LW ZLOO KDYH D WZRGLPHQVLRQDO FDUWHVLDQ
FRRUGLQDWHV\VWHPDVVRFLDWHG,QRUGHUIRUDSURFHVVWRGHWHUPLQHLWVFRRUGLQDWHVLWVLPSO\FDOOVWKH
IXQFWLRQMPI_Cart_coords
integer coordinates(0:1)
integer my_grid_rank
call MPI_COMM_RANK(grid_comm, my_grid_rank, ierr)
call MPI_CART_COORDS(grid_comm, my_grid_rank, 2,
+ coordinates, ierr)

1RWLFHWKDWZHQHHGHGWRFDOO MPI_Comm_rankLQRUGHUWRJHWWKHSURFHVVUDQNLQ grid_comm

7KLVZDVQHFHVVDU\EHFDXVHLQRXUFDOOWR MPI_Cart_createZHVHWWKH reorder IODJWR .TRUE.
DQG KHQFH WKH RULJLQDO SURFHVV UDQNLQJ LQ MPI_COMM_WORLD PD\ KDYH EHHQ FKDQJHG LQ
grid_comm

7KHLQYHUVH
WRMPI_Cart_coordsLVMPI_Cart_rank
call MPI_CART_RANK(grid_comm, coordinates, grid_rank,
+ ierr)
integer grid_comm, coordinates(*), grid_rank, ierr

*LYHQ WKH FRRUGLQDWHV RI D SURFHVV MPI_Cart_rank UHWXUQV WKH UDQN RI WKH SURFHVV LQ LWV WKLUG
SDUDPHWHUprocess_rank

7KHV\QWD[RIMPI_Cart_createLV
call MPI_CART_CREATE(old_comm, number_of_dims, dim_sizes,
+ periods, reorder, cart_comm, ierror)
integer old_comm, number_of_dims, dim_sizes(*)
logical periods(*), reorder
integer cart_comm, ierror

MPI_Cart_createFUHDWHVDQHZFRPPXQLFDWRU cart_commE\FDFKLQJDFDUWHVLDQWRSRORJ\ZLWK
old_comm,QIRUPDWLRQRQWKHVWUXFWXUHRIWKHFDUWHVLDQWRSRORJ\LVFRQWDLQHGLQWKHSDUDPHWHUV
number_of_dimsdim_sizesDQG periods7KHILUVWRIWKHVH number_of_dimsFRQWDLQVWKH
QXPEHURIGLPHQVLRQVLQWKHFDUWHVLDQFRRUGLQDWHV\VWHP7KHQH[WWZR dim_sizesDQGperiods
DUHDUUD\VZLWKRUGHUHTXDOWR number_of_dims7KHDUUD\dim_sizesVSHFLILHVWKHRUGHURIHDFK
GLPHQVLRQDQGperiodsVSHFLILHVZKHWKHUHDFKGLPHQVLRQLVFLUFXODURUOLQHDU

7KH SURFHVVHV LQ cart_comm DUH UDQNHG LQ URZPDMRU RUGHU 7KDW LV WKH ILUVW URZ FRQVLVWV RI
SURFHVVHV     GLPBVL]HV WKH VHFRQG URZ FRQVLVWV RI SURFHVVHV GLPBVL]HV
GLPBVL]HV    GLPBVL]HV HWF 7KXV LW PD\ EH DGYDQWDJHRXV WR FKDQJH WKH UHODWLYH
UDQNLQJRIWKHSURFHVVHVLQ old_comm)RUH[DPSOHVXSSRVHWKH SK\VLFDOWRSRORJ\LVD[JULG
DQGWKHSURFHVVHVQXPEHUVLQ old_commDUHDVVLJQHGWRWKHSURFHVVRUVJULGVTXDUHVDVIROORZV

  

  

  

&OHDUO\WKHSHUIRUPDQFHRI)R[
VDOJRULWKPZRXOGEHLPSURYHGLIZHUHQXPEHUHGWKHSURFHVVHV
+RZHYHU VLQFH WKH XVHU GRHVQ
W NQRZ ZKDW WKH H[DFW PDSSLQJ RI SURFHVVHV WR SURFHVVRUV LV ZH
PXVWOHWWKHV\VWHPGRLWE\VHWWLQJWKH reorderSDUDPHWHUWR.TRUE. 
6LQFHMPI_Cart_createFRQVWUXFWVDQHZFRPPXQLFDWRULWLVDFROOHFWLYHRSHUDWLRQ
7KHV\QWD[RIWKHDGGUHVVLQIRUPDWLRQIXQFWLRQVLV
MPI_Cart_rank(comm, coordinates, rank, ierror)
integer comm, coordinates(*), rank, ierror
MPI_Cart_coords(comm, rank, number_of_dims, coordinates,
+ ierror)
integer comm, rank, number_of_dims, coordinates(*), ierror

MPI_Cart_rank UHWXUQV WKH UDQN LQ WKH FDUWHVLDQ FRPPXQLFDWRU comm RI WKH SURFHVV ZLWK
FDUWHVLDQFRRUGLQDWHV coordinates6Rcoordinates LVDQDUUD\ZLWKRUGHUHTXDOWRWKHQXPEHURI
GLPHQVLRQVLQWKHFDUWHVLDQWRSRORJ\DVVRFLDWHGZLWK comm MPI_Cart_coordsLVWKHLQYHUVHWR
MPI_Cart_rank LW UHWXUQV WKH FRRUGLQDWHV RI WKH SURFHVV ZLWK UDQN rank LQ WKH FDUWHVLDQ
FRPPXQLFDWRUcomm1RWHWKDWERWKRIWKHVHIXQFWLRQVDUHORFDO

03,B&DUWBVXE
:H FDQ DOVR SDUWLWLRQ D JULG LQWR JULGV RI ORZHU GLPHQVLRQ )RU H[DPSOH ZH FDQ
FUHDWHDFRPPXQLFDWRUIRUHDFKURZRIWKHJULGDVIROORZV
logical varying_coords(0:1)
integer row_comm
varying_coords(0) = .FALSE.
varying_coords(1) = .TRUE.
call MPI_CART_SUB(grid_comm, varying_coords, row_comm,
ierr)

7KH FDOO WR MPI_Cart_sub FUHDWHV T QHZ FRPPXQLFDWRUV 7KH varying_coords

DUJXPHQW LV DQ DUUD\ RI ERROHDQ ,W VSHFLILHV ZKHWKHU HDFK GLPHQVLRQ EHORQJV WR
WKHQHZFRPPXQLFDWRU6LQFHZH
UHFUHDWLQJFRPPXQLFDWRUVIRUWKHURZVRIWKHJULG
HDFK QHZ FRPPXQLFDWRU FRQVLVWV RI WKH SURFHVVHV REWDLQHG E\ IL[LQJ WKH URZ
FRRUGLQDWH DQG OHWWLQJ WKH FROXPQ FRRUGLQDWH YDU\ +HQFH ZH DVVLJQHG
varying_coords WKH YDOXH )$/6(  WKH ILUVW FRRUGLQDWH GRHVQ
W YDU\  DQG
ZHDVVLJQHG varying_coordsWKHYDOXH758(WKHVHFRQGFRRUGLQDWHYDULHV
2Q HDFK SURFHVV WKH QHZ FRPPXQLFDWRU LV UHWXUQHG LQ row_comm ,Q RUGHU WR
FUHDWHWKHFRPPXQLFDWRUVIRUWKHFROXPQVZHVLPSO\UHYHUVHWKHDVVLJQPHQWVWRWKH
HQWULHVLQvarying_coords
integer col_comm
varying_coords(0) = .TRUE.
varying_coords(1) = .FALSE.
call MPI_CART_SUB(grid_comm, varying_coords, row_comm,
ierr)
 1RWHWKHVLPLODULW\RI MPI_Cart_subWRMPI_Comm_split7KH\SHUIRUPVLPLODU
IXQFWLRQV  WKH\ ERWK SDUWLWLRQ D FRPPXQLFDWRU LQWR D FROOHFWLRQ RI QHZ
FRPPXQLFDWRUV+RZHYHU MPI_Cart_subFDQRQO\EHXVHGZLWKDFRPPXQLFDWRU
WKDWKDVDQDVVRFLDWHGFDUWHVLDQWRSRORJ\DQGWKHQHZFRPPXQLFDWRUVFDQRQO\EH
FUHDWHG E\ IL[LQJ RU YDU\LQJ RQH RU PRUH GLPHQVLRQV RI WKH ROG FRPPXQLFDWRUV
$OVRQRWHWKDWMPI_Cart_subLVOLNHMPI_Comm_splitDFROOHFWLYHRSHUDWLRQ

,PSOHPHQWDWLRQRI)R[
V$OJRULWKP
7RFRPSOHWHRXUGLVFXVVLRQOHW
VZULWHWKHFRGHWRLPSOHPHQW)R[
VDOJRULWKP)LUVWZH
OOZULWH D
IXQFWLRQ WKDW FUHDWHV WKH YDULRXV FRPPXQLFDWRUV DQG DVVRFLDWHG LQIRUPDWLRQ 6LQFH WKLV UHTXLUHV D
ODUJHQXPEHURIYDULDEOHVDQGZH
OOEHXVLQJWKLVLQIRUPDWLRQLQRWKHUIXQFWLRQVZH
OOSXWLWLQWRD
)RUWUDQGHULYHGW\SHWRIDFLOLWDWHSDVVLQJLWDPRQJWKHYDULRXVIXQFWLRQV
1RWLFHWKDWVLQFHHDFKRIRXUFRPPXQLFDWRUVKDVDQDVVRFLDWHGWRSRORJ\ZHFRQVWUXFWHGWKHPXVLQJ
WKHWRSRORJ\FRQVWUXFWLRQIXQFWLRQV MPI_Cart_createDQGMPI_Cart_subUDWKHUWKDQWKH
PRUHJHQHUDOFRPPXQLFDWRUFRQVWUXFWLRQIXQFWLRQV MPI_Comm_createDQGMPI_Comm_split
program myfox
include 'mpif.h'
IMPLICIT NONE
type GRID_INFO_TYPE
integer p ! Total number of processes.
integer comm ! Communicator for the entire grid.
integer row_comm ! Communicator for my row.
integer col_comm ! Communicator for my col.
integer q ! Order of grid.
integer my_row ! My row number.
integer my_col ! My column number.
integer my_rank ! My rank in the grid communicator.
end type GRID_INFO_TYPE
TYPE (GRID_INFO_TYPE) :: grid_info
integer my_rank, ierr
real, allocatable, dimension(:,:) :: A,B,C
integer n, n_bar
call MPI_INIT(ierr)
call Setup_grid(grid_info)
call MPI_Comm_rank(MPI_COMM_WORLD, my_rank, ierr)
if (my_rank == 0) then
print *, 'What is the order of the matrices?'
read *, n
endif
call MPI_BCAST(n,1,MPI_INTEGER, 0, MPI_COMM_WORLD,ierr)
n_bar = n/(grid_info%q)
! Allocate local storage for local matrix.
allocate( A(n_bar,n_bar) )
allocate( B(n_bar,n_bar) )
allocate( C(n_bar,n_bar) )
A = 1.0
B = 2.0
call Fox(n,grid_info,A,B,C,n_bar)
print *, C
contains
subroutine Setup_grid(grid)
TYPE (GRID_INFO_TYPE), intent(inout) :: grid
integer old_rank
integer dimensions(0:1)
logical periods(0:1)
integer coordinates(0:1)
logical varying_coords(0:1)
integer ierr
! Set up Global Grid Information.
call MPI_Comm_size(MPI_COMM_WORLD, grid%p, ierr)
call MPI_Comm_rank(MPI_COMM_WORLD, old_rank, ierr )
grid%q = int(sqrt(dble(grid%p)))
dimensions(0) = grid%q
dimensions(1) = grid%q
periods(0) = .TRUE.
periods(1) = .TRUE.
call MPI_Cart_create(MPI_COMM_WORLD, 2,
+ dimensions, periods, .TRUE. , grid%comm, ierr)
call MPI_Comm_rank (grid%comm, grid%my_rank, ierr )
call MPI_Cart_coords(grid%comm, grid%my_rank, 2,
+ coordinates, ierr )
grid%my_row = coordinates(0)
grid%my_col = coordinates(1)
! Set up row and column communicators.
varying_coords(0) = .FALSE.
varying_coords(1) = .TRUE.
call MPI_Cart_sub(grid%comm,varying_coords,
+ grid%row_comm,ierr)
varying_coords(0) = .TRUE.
varying_coords(1) = .FALSE.
call MPI_Cart_sub(grid%comm,varying_coords,
+ grid%col_comm,ierr)
end subroutine Setup_grid
subroutine Fox(n,grid,local_A,local_B,loca l_C,n_bar)
integer, intent(in) :: n, n_bar
TYPE(GRID_INFO_TYPE), intent(in) :: grid
real, intent(in) , dimension(:,:) :: local_A, local_B
real, intent(out), dimension (:,:) :: local_C
real temp_A( SIZE(A,DIM=1),SIZE(A,DIM=2) )
integer step, source, dest, request
integer status(MPI_STATUS_SIZE), bcast_root
local_C = 0.0
source = mod( (grid%my_row + 1), grid%q )
dest = mod( (grid%my_row + grid%q -1), (grid%q) )
temp_A = 0.0
do step = 0, grid%q -1
bcast_root = mod( (grid%my_row + step), (grid%q) )
if (bcast_root == grid%my_col) then
call MPI_BCAST(local_A,n_bar*n_bar,MPI_REAL,
+ bcast_root, grid%row_comm, ierr)
call sgemm('N','N',n_bar,n_bar,n_bar,1.0,
+ local_A,n_bar,local_B,n_bar,1.0,local_C,n_bar)
else
call MPI_BCAST(temp_A,n_bar*n_bar,MPI_REAL,
+ bcast_root, grid%row_comm, ierr)
call sgemm('N','N',n_bar,n_bar,n_bar,1.0,
+ temp_A,n_bar,local_B,n_bar,1.0,local_C,n_bar)
endif
call MPI_Send(local_B,n_bar*n_bar,MPI_REAL,dest, 0,
+ grid%col_comm, ierr)
call MPI_Recv(local_B,n_bar*n_bar,MPI_REAL,source,0,
+ grid%col_comm, status, ierr )
enddo
end subroutine Fox
end program myfox

 :KHUH7R*R)URP+HUH
:KDW:H+DYHQ
W'LVFXVVHG
03, LV D ODUJH OLEUDU\ 7KH 6WDQGDUG >@ LV RYHU  SDJHV ORQJ DQG LW GHILQHV PRUH WKDQ 
IXQFWLRQV$VDFRQVHTXHQFHWKLV *XLGHKDVFRYHUHGRQO\DVPDOOIUDFWLRQRI03,DQGPDQ\UHDGHUV
ZLOOIDLOWRILQGDGLVFXVVLRQRIIXQFWLRQVWKDWWKH\ZRXOGILQGYHU\XVHIXOLQWKHLUDSSOLFDWLRQV6RZH
EULHIO\OLVWVRPHRIWKHPRUHLPSRUWDQWLGHDVLQ03,WKDWZHKDYHQRWGLVFXVVHGKHUH

 &RPPXQLFDWLRQ0RGHV:HKDYHXVHGRQO\WKH VWDQGDUGFRPPXQLFDWLRQPRGHIRU send

7KLV PHDQV WKDW LW LV XS WR WKH V\VWHP WR GHFLGH ZKHWKHU WKH PHVVDJH LV EXIIHUHG
03, SURYLGHV WKUHH RWKHU FRPPXQLFDWLRQ PRGHV EXIIHUHG V\QFKURQRXV DQG UHDG\ ,Q
EXIIHUHG PRGH WKH XVHU H[SOLFLWO\ FRQWUROV WKH EXIIHULQJ RI RXWJRLQJ PHVVDJHV ,Q
V\QFKURQRXVPRGHDVHQGZLOOQRWFRPSOHWHXQWLODPDWFKLQJUHFHLYHLVSRVWHG,Q
 UHDG\ PRGH D VHQG PD\ EH VWDUWHG RQO\ LI D PDWFKLQJ UHFHLYH KDV DOUHDG\ EHHQ
SRVWHG03,SURYLGHVWKUHHDGGLWLRQDOVHQGIXQFWLRQVIRUWKHVHPRGHV
 1RQEORFNLQJ &RPPXQLFDWLRQ :H KDYH XVHG RQO\ EORFNLQJ VHQGV DQG UHFHLYHV
MPI_Send DQG MPI_Recv )RU WKH VHQG WKLV PHDQV WKDW WKH FDOO ZRQ
W UHWXUQ
XQWLO WKH PHVVDJH GDWD DQG HQYHORSH KDYH EHHQ EXIIHUHG RU VHQW  LH XQWLO WKH
PHPRU\UHIHUHQFHGLQWKHFDOOWR MPI_SendLVDYDLODEOHIRUUHXVH)RUWKHUHFHLYH
WKLV PHDQV WKDW WKH FDOO ZRQ
W UHWXUQ XQWLO WKH GDWD KDV EHHQ UHFHLYHG LQWR WKH
PHPRU\UHIHUHQFHGLQWKHFDOOWR MPI_Recv0DQ\DSSOLFDWLRQVFDQLPSURYHWKHLU
SHUIRUPDQFH E\ XVLQJ QRQEORFNLQJ FRPPXQLFDWLRQ 7KLV PHDQV WKDW WKH FDOOV WR
VHQGUHFHLYH PD\ UHWXUQ EHIRUH WKH RSHUDWLRQ FRPSOHWHV )RU H[DPSOH LI WKH
PDFKLQHKDVDVHSDUDWHFRPPXQLFDWLRQSURFHVVRUDQRQEORFNLQJVHQGFRXOGVLPSO\
QRWLI\WKHFRPPXQLFDWLRQSURFHVVRUWKDWLWVKRXOGEHJLQFRPSRVLQJDQGVHQGLQJWKH
PHVVDJH 03, SURYLGHV QRQEORFNLQJ VHQGV LQ HDFK RI WKH IRXU PRGHV DQG D
QRQEORFNLQJ UHFHLYH ,W DOVR SURYLGHV YDULRXV XWLOLW\ IXQFWLRQV IRU GHWHUPLQLQJ WKH
FRPSOHWLRQVWDWXVRIDQRQEORFNLQJRSHUDWLRQ
 ,QWHUFRPPXQLFDWRUV 5HFROOHFW WKDW 03, SURYLGHV WZR W\SHV RI FRPPXQLFDWRUV LQWUD
FRPPXQLFDWRUV DQG LQWHUFRPPXQLFDWRUV ,QWHUFRPPXQLFDWRUV FDQ EH XVHG IRU
SRLQWWRSRLQW FRPPXQLFDWLRQV EHWZHHQ SURFHVVHV EHORQJLQJ WR GLVWLQFW LQWUD
FRPPXQLFDWRUV
7KHUHDUHPDQ\RWKHUIXQFWLRQVDYDLODEOHWRXVHUVRI03,,IZHKDYHQ
WGLVFXVVHGDIDFLOLW\\RXQHHG
SOHDVHFRQVXOWWKH6WDQGDUG>@WRGHWHUPLQHZKHWKHULWLVSDUWRI03,

,PSOHPHQWDWLRQVRI03,
,I \RX GRQ
W KDYH DQ LPSOHPHQWDWLRQ RI 03, WKHUH DUH WKUHH YHUVLRQV WKDW DUH IUHHO\ DYDLODEOH E\
DQRQ\PRXVIWSIURPWKHIROORZLQJVLWHV

$UJRQQH 1DWLRQDO /DE0LVVLVVLSSL 6WDWH 8QLYHUVLW\ 7KH DGGUHVV LV

info.mcs.anl.govDQGWKHGLUHFWRU\LV pub/mpi

(GLQEXUJK 8QLYHUVLW\ 7KH DGGUHVV LV ftp.epcc.ed.ac.uk DQG WKH GLUHFWRU\ LV

pub/chimp/release

2KLR 6XSHUFRPSXWHU &HQWHU 7KH DGGUHVV LV tbag.osc.edu DQG WKH GLUHFWRU\ LV
SXEODP
$OORIWKHVHUXQRQQHWZRUNVRI81,;ZRUNVWDWLRQV7KH$UJRQQH0LVVLVVLSSL6WDWHDQG(GLQEXUJK
YHUVLRQV DOVR UXQ RQ YDULRXV SDUDOOHO SURFHVVRUV &KHFN WKH 5($'0( ILOHV WR VHH LI \RXU
PDFKLQHVDUHVXSSRUWHG

0RUH,QIRUPDWLRQRQ03,
7KHUHLVDQ03,)$4DYDLODEOHE\DQRQ\PRXVIWSDW

0LVVLVVLSSL 6WDWH 8QLYHUVLW\ 7KH DGGUHVV LV ftp.erc.msstate.edu DQG WKH ILOH LV
pub/mpi/faq
7KHUHDUHDOVRQXPHURXVZHESDJHVGHYRWHGWR03,$IHZRIWKHVHDUH

http://www.epm.ornl.gov/~walker/mpi 7KH 2DN 5LGJH 1DWLRQDO /DE 03, ZHE

 SDJH

http://www.erc.msstate.edu/mpi7KH0LVVLVVLSSL6WDWH03,ZHESDJH

http://www.mcs.anl.gov/mpi7KH$UJRQQH03,ZHESDJH
(DFKRIWKHVHVLWHVFRQWDLQVDZHDOWKRILQIRUPDWLRQDERXW03,2ISDUWLFXODUQRWHWKH0LVVLVVLSSL
6WDWHSDJHFRQWDLQVDELEOLRJUDSK\RISDSHUVRQ03,DQGWKH$UJRQQHSDJHFRQWDLQVDFROOHFWLRQRI
WHVW03,SURJUDPV
7KH 03, 6WDQGDUG >@ LV FXUUHQWO\ DYDLODEOH IURP HDFK RI WKH VLWHV DERYH 7KLV LV RI FRXUVH WKH
GHILQLWLYHVWDWHPHQWRIZKDW03,LV6RLI\RX
UHQRWFOHDURQVRPHWKLQJWKLVLVWKHILQDODUELWHU,W
DOVR FRQWDLQV D ODUJH QXPEHU RI QLFH H[DPSOHV RI XVHV RI WKH YDULRXV 03, IXQFWLRQV 6R LW LV
FRQVLGHUDEO\PRUHWKDQMXVWDUHIHUHQFH&XUUHQWO\VHYHUDOPHPEHUVRIWKH03,)RUXPDUHZRUNLQJ
RQDQDQQRWDWHGYHUVLRQRIWKH03,VWDQGDUG>@
7KH ERRN >@ LV D WXWRULDO LQWURGXFWLRQ WR 03, ,W SURYLGHV QXPHURXV FRPSOHWH H[DPSOHV RI 03,
SURJUDPV
7KHERRN>@FRQWDLQVDWXWRULDOLQWURGXFWLRQWR03,RQZKLFKWKLVJXLGHLVEDVHG,WDOVRFRQWDLQV
D PRUH JHQHUDO LQWURGXFWLRQ WR SDUDOOHO SURFHVVLQJ DQG WKH SURJUDPPLQJ RI PHVVDJHSDVVLQJ
PDFKLQHV

7KH 8VHQHW QHZVJURXS comp.parallel.mpi SURYLGHV LQIRUPDWLRQ RQ XSGDWHV WR DOO RI WKHVH
GRFXPHQWVDQGVRIWZDUH

7KH)XWXUHRI03,
$V LW LV FXUUHQWO\ GHILQHG 03, IDLOV WR VSHFLI\ WZR FULWLFDO FRQFHSWV ,2 DQG WKH
FUHDWLRQGHVWUXFWLRQRISURFHVVHV:RUNKDVDOUHDG\EHHQVWDUWHGRQWKHGHYHORSPHQWRIERWK,2
IDFLOLWLHV DQG G\QDPLF SURFHVV FUHDWLRQ ,QIRUPDWLRQ RQ WKH IRUPHU FDQ EH REWDLQHG IURP
http://lovelace.nas.nasa.gov/MPI-IO/mpi-io.html DQGLQIRUPDWLRQ RQ WKH ODWWHU FDQ EH IRXQG
RQ WKH $UJRQQH 03, ZHE SDJH 6LJQLILFDQW GHYHORSPHQWV DUH LQYDULDEO\ SRVWHG WR
comp.parallel.mpi


 &RPSLOLQJDQG5XQQLQJ03,
3URJUDPV
7KLVVHFWLRQLVLQWHQGHGWRJLYHWKHRXWOLQHRIKRZWRFRPSLOHDQGUXQDSURJUDPLQWKH,%0
63
03, SURJUDP ZULWWHQ LQ )RUWUDQ RU )RUWUDQ  FDQ EH FRPSLOHG XVLQJ WKH IROORZLQJ
FRPPDQG
mpif77 program.f
%\GHIDXOWWKH SURJUDP ZLOO EH UXQQLQJ RQ  SURFHVVRUV RI WKH 63 7KH SURJUDP FDQ EH
LQYRNHGE\WKHQDPHRIH[HFXWDEOH
a.out
 7KHQXPEHURISURFHVVHVLVFRQWUROHGE\WKHHQYLURQPHQWYDULDEOH03B352&67KHZHE
SDJH KWWSZZZKNXKNFFVSWHFKQLFDOVHWHQYKWPO KDV PDQXDOV IRU VHWWLQJ WKH
HQYLURQPHQWYDULDEOH

 5HIHUHQFH
>@*HRIIUH\)R[HWDO 6ROYLQJ3UREOHPVRQ&RQFXUUHQW3URFHVVRUV(QJOHZRRG&OLIIV1-3UHQWLFH+DOO

>@:LOOLDP*URSS(ZLQJ/XVNDQG$QWKRQ\6NMHOOXP 8VLQJ03,3RUWDEOH3DUDOOHO3URJUDPPLQJZLWK
WKH0HVVDJH3DVVLQJ,QWHUIDFH&DPEULGJH0$0,73UHVV
>@ %ULDQ : .HUQLJKDQ DQG 'HQQLV 0 5LWFKLH 7KH & 3URJUDPPLQJ /DQJXDJH QG HG (QJOHZRRG
&OLIIV1-3UHQWLFH+DOO
>@0HVVDJH3DVVLQJ,QWHUIDFH)RUXP 03,$0HVVDJH3DVVLQJ,QWHUIDFH6WDQGDUG,QWHUQDWLRQDO-RXUQDO
RI6XSHUFRPSXWHU$SSOLFDWLRQVYROQRV$OVRDYDLODEOHDV7HFKQLFDO5HSRUW&6
&RPSXWHU6FLHQFH'HSW8QLYHUVLW\RI7HQQHVVHH.QR[YLOOH71
>@6WHYH2WWRHWDO03,$QQRWDWHG5HIHUHQFH0DQXDO&DPEULGJH0$0,73UHVVWRDSSHDU
>@3HWHU63DFKHFR3URJUDPPLQJ3DUDOOHOZLWK03,6DQ)UDQFLVFR&$0RUJDQ.DXIPDQQ
>@3HWHU63DFKHFR$8VHUV*XLGHWR03,